aiiinotate 0.6.2 → 0.8.0

package/README.md

@@ -6,7 +6,7 @@ aiiinotate is a fast and lightweight annotation server for IIIF. It relies on `n
 
 ## PROD USAGE
 
-### Install 
+### Install
 
 1. **Install mongodb**.
     - see [dev installation script for help](./scripts/setup_mongodb.sh)
@@ -17,9 +17,37 @@ aiiinotate is a fast and lightweight annotation server for IIIF. It relies on `n
 npm install aiiinotate
 ```
 
-### Setup the app 
+### Env definition
 
-0. **Setup your `.env`** file after [.env.template](./config/.env.template).
+#### Basic definition
+
+Copy [`config/.env.template`](./config/.env.template) to `.env` and edit it.
+
+#### Runtime env sourcing
+
+Once the package is installed, it must access variables from the .env file. However, running `aiiinotate <commands>` creates a subscript which means **you can't `source` an `.env` file**.
+
+```bash
+# THIS WILL FAIL: `aiiinotate` is executed in a subscript that doesn't inherit from the variables fetched in `source`.
+source /path/to/.env && aiiinotate <command>
+```
+
+#### The solutions: do either:
+1. use `dotenvx` to inject variables:
+    ```bash
+    npx dotenvx run -f /path/to/.env -- aiiinotate <command>
+    ```
+2. manually export variables:
+    ```bash
+    set -a
+    source /path/to/.env
+    set +a
+    aiiinotate <command>
+    ```
+
+For clarity, we omit env sourcing from the below commands.
+
+### Setup the app
 
 1. **Start `mongod`**
 
@@ -30,38 +58,41 @@ sudo systemctl start mongod
 2. **Create and configure the database**
 
 ```bash
-aiiinotate --env <path-to-your-env-file> -- migrate apply
+aiiinotate migrate apply
 ```
 
 ### Usage
 
 All commands are accessible through a CLI (`./src/cli`).
 
-#### Run the app 
+#### Run the app
 
 ```bash
-aiiinotate --env <path-to-your-env-file> -- serve prod
-# or 
-aiiinotate --env <path-to-your-env-file> -- serve dev
+aiiinotate serve prod
 ```
 
-#### Run administration commands
+#### Run the CLI
 
 The base command is:
 
 ```bash
-aiiinotate --env <path-to-your-env-file> -- <command>
+aiiinotate -- <command>
 ```
 
 It will give full access to the CLI interface of Aiiinotate. Run `aiiinotate --help` for more info.
 
-1. Import data - TODO
+For more information, see [the CLI docs](https://github.com/Aikon-platform/aiiinotate/blob/main/docs/cli.md).
+
+#### Import data
+
+See [the CLI docs](https://github.com/Aikon-platform/aiiinotate/blob/main/docs/cli.md).
+
 
 ---
 
 ## DEV USAGE
 
-### Install 
+### Install
 
 ```bash
 # clone the repo
@@ -84,15 +115,15 @@ npm i
 
 After installing, some setup must be done
 
-0. **Setup your `.env`** file after [.env.template](./config/.env.template) and place it at `./config/.env`.
+1. **Setup your `.env`** file after [`config/.env.template`](./config/.env.template) and place it at `./config/.env`.
 
-1. **Start `mongod`**
+2. **Start `mongod`**
 
 ```bash
 sudo systemctl start mongod
 ```
 
-2. **Configure the database**
+3. **Configure the database**
 
 ```bash
 npm run migrate apply
@@ -112,32 +143,32 @@ npm run dev
 npm run prod
 ```
 
-- **Test the app**
+- **Test the app**. NOTE: the tests will probably fail if you set the env variable `AIIINOTATE_STRICT_MODE` to `true`.
 
 ```bash
 npm run test
 ```
 
-- **Run the CLI**
+- **Run the CLI**. (see [the CLI docs](https://github.com/Aikon-platform/aiiinotate/blob/dev/docs/cli.md) for more info)
+
 
 ```bash
 npm cli
 ```
 
-- **Process migrations**
+- **Process migrations**. (see [the CLI docs](https://github.com/Aikon-platform/aiiinotate/blob/dev/docs/cli.md) for more info)
 
 ```bash
-# create a new migration. NOTE: the `--` is necessary !
-npm run migrate make -- --migration-name <your migration name>
+# NOTE: the `--` is necessary !
+npm run migrate -- <command> <arguments?>
+```
 
-# apply all pending migrations
-npm run migrate apply
 
-# revert the last migration
-npm run migrate revert
+- **Import data**. (see [the CLI docs](https://github.com/Aikon-platform/aiiinotate/blob/main/docs/cli.md) for more info)
 
-# revert all migrations
-npm run migrate revert-all
+```bash
+# NOTE: the `--` is necessary !
+npm run cli -- import <arguments>
 ```
 
 ---

package/cli/import.js

@@ -1,40 +1,17 @@
 import { Command, Option, Argument } from "commander";
 
-import Annotations2 from "#annotations/annotations2.js";
-import Annotations3 from "#annotations/annotations3.js";
-import { getFilesToProcess, fileRead } from "#cli/utils/io.js";
-import loadMongoClient from "#cli/utils/mongoClient.js";
+import { fileRead, parseImportInputFile } from "#cli/utils/io.js";
+import FastifyClient from "#cli/utils/fastifyClient.js";
+import ProgressBar from "#cli/utils/progressbar.js";
+import logger from "#utils/logger.js";
+import { inspectObj } from "#utils/utils.js";
 
-////////////////////////////////////////
-
-// allowed imports
-const importTypes = [
-  "annotation",  // import a single annotation
-  "annotation-list",  // import a IIIF 2.x annotationList
-  "annotation-page",  // import a IIIF 3.x annotationPage
-  "manifest",  // import a single manifest
-  // "annotation-array",  // import a JSON array of IIIF annotations
-  // "manifest-array"  // import a json array of manifests
-]
-
-// allowed import types per IIIF version
-const allowedImportTypes = {
-  2: ["annotation", "annotation-list", "manifest"],
-  3: ["annotation", "annotation-page", "manifest"]
-}
-
-const checkAllowedImportType = (iiifVersion, dataType) => {
-  if (
-    ! allowedImportTypes[iiifVersion].includes(dataType)
-  ) {
-    console.error(`${checkAllowedImportType.name}: forbidden import type '${dataType}' for IIIF version '${iiifVersion}'. allowed import types are: ${allowedImportTypes[iiifVersion]}`);
-    process.exit(1);
-  };
+/** @typedef {import("#types").FastifyInstanceType} FastifyInstanceType */
 
-}
+////////////////////////////////////////
 
 const notImplementedExit = (method) => {
-  console.log(`\n\nERROR: import is not implemented '${method}'`);
+  logger.error(`\n\nERROR: import is not implemented '${method}'`);
   process.exit(1);
 }
 
@@ -42,28 +19,42 @@ const parseNumber = (x) => Number(x);
 
 ////////////////////////////////////////
 
-async function importAnnotationPage(annotations3, fileArr, iiifVersion) {
+/**
+ * @param {FastifyInstanceType} fastifyClient
+ * @param {string[]} fileArr - array of full paths to annotationLists to insert.
+ */
+async function importAnnotationPage(fastifyClient, fileArr) {
   notImplementedExit(`${importAnnotationPage.name} is not implemented`)
 }
 
 /**
- *
- * @param {Annotations2} annotations2
- * @param {string[]} fileArr
- * @param {2|3} iiifVersion
+ * @param {FastifyClient} fastifyClient
+ * @param {string[]} fileArr - array of full paths to annotationLists to insert.
  */
-async function importAnnotationList(annotations2, fileArr, iiifVersion) {
-  // RUN THE SCRIPT:
-  // > npm run migrate-revert && npm run migrate-apply && npm run cli import -- annotation-list -i 2 -f ./data/aikon_wit9_man11_anno165_annotation_list.jsonld
+async function importAnnotationList(fastifyClient, fileArr) {
+  // fileArr = fileArr.slice(0,10)
+
+  const pb = new ProgressBar({ desc: "importing annotations", total: fileArr.length});
+  const importErrors = [];
   let totalImports = 0
 
-  for (const file of fileArr) {
-    const annotationList = JSON.parse(await fileRead(file));
-    const result = await annotations2.insertAnnotationList(annotationList);
-    totalImports += Object.keys(result).length;
+  for ( const [i, file] of fileArr.entries() ) {
+    const
+      annotationList = JSON.parse(fileRead(file)),
+      [statusCode, resultPromise] = await fastifyClient.importAnnotationList(annotationList),
+      result = await resultPromise;
+    if ( statusCode <= 299 ) {
+      totalImports += result.insertedCount;
+    } else {
+      importErrors.push(file);
+    }
+    pb.update(i)
   }
 
-  console.log(`\n\nDONE: imported ${totalImports} annotations into Aiiinotate !`);
+  if ( importErrors.length ) {
+    logger.info(`There were problems importing annotations from the following ${importErrors.length} annotation lists: ${inspectObj(importErrors, -1)}`)
+  }
+  logger.info(`Imported ${totalImports} annotations into Aiiinotate !`);
   return
 }
 
@@ -72,51 +63,33 @@ async function importAnnotationList(annotations2, fileArr, iiifVersion) {
 /**
  * run the cli
  * @param {import('commander').Command} command
- * @param {string} dataType: one of importTypes
  * @param {object} options
  */
-async function action(command, dataType, options) {
-  const mongoClient = loadMongoClient();
+async function action(command, options) {
 
   /** @type {2 | 3} */
   const iiifVersion = options.iiifVersion;
   /** @type {string[]} */
-  const files = options.files;
-  /** @type {boolean} */
-  const listFiles = options.listFiles;
-
-  checkAllowedImportType(iiifVersion, dataType);
-
-  const filesToProcess = getFilesToProcess(files, listFiles);
+  const inputFile = options.file;
 
-  const annotations2 = new Annotations2(
-    mongoClient,
-    mongoClient.db()
-  );
+  const fastifyClient = new FastifyClient();
+  await fastifyClient.build();
 
   // run
-  switch (dataType) {
-    case ("annotation-list"):
-      await importAnnotationList(annotations2, filesToProcess, iiifVersion);
-      break;
-    default:
-      notImplementedExit(dataType);
+  const filesToProcess = await parseImportInputFile(inputFile);
+  if ( iiifVersion===2 ) {
+    await importAnnotationList(fastifyClient, filesToProcess);
+  } else {
+    await importAnnotationPage(fastifyClient, filesToProcess);
   }
-  mongoClient.close();
-
+  // exit
+  await fastifyClient.stop();
 }
 
 /////////////////////////////////////////
 
 /** define the cli */
-function makeImportCommand(mongoClient) {
-
-  // argument and option name syntax:
-  // --opt-name <requiredVal> => you mst provide a value after --opt-name
-  // --opt-name [optionalVal] => if `optionalVal` is not provided, --opt-name will be treated as boolean
-  const dataTypeArg =
-    new Argument("<data-type>", "type of data to import")
-      .choices(importTypes);
+function makeImportCommand() {
 
   const versionOpt =
     new Option("-i, --iiif-version <version>", "IIIF version")
@@ -125,19 +98,14 @@ function makeImportCommand(mongoClient) {
       .makeOptionMandatory();
 
   const filesOpt =
-    new Option("-f, --files <files...>", "files to process, either as: space-separated filepath(s) to the JSON(s) OR path to a file containing a list of paths to JSON files to process (1 line per path)")
+    new Option("-f, --file <file>", "file containing paths to AnnotationLists or AnnotationPages to process (1 line per path)")
       .makeOptionMandatory();
 
-  const listFilesOpt =
-    new Option("-l, --list-files", "flag indicating that --files points to a file containing a list of JSON files to process (1 line per path)")
-
   return new Command("import")
     .description("import data into aiiinotate")
-    .addArgument(dataTypeArg)
     .addOption(filesOpt)
     .addOption(versionOpt)
-    .addOption(listFilesOpt)
-    .action((dataType, options, command) => action(mongoClient, command, dataType, options))
+    .action((options, command) => action(command, options))
 }
 
 export default makeImportCommand;

package/cli/index.js

@@ -13,7 +13,6 @@ import { Command, Option } from "commander";
 
 // import dotenvx from "dotenvx";
 
-import makeMongoClient from "#cli/utils/mongoClient.js";
 import makeImportCommand from "#cli/import.js";
 import makeMigrateCommand from "#cli/migrate.js";
 import makeServeCommand from "#cli/serve.js";
@@ -29,25 +28,11 @@ function makeCli() {
     Run individual commands to see command-specific help.
     `.replace(/^\s+/gm, "");
 
-  const envFileOpt =
-    new Option("--env <env-file>", "path to .env file").makeOptionMandatory();
-
-  // NOTE: how do we load the env variables ? it's a bit unorthodox:
-  // - the CLI requires to use a global `--env` option with a path to the .env file.
-  // - we use the hook `preAction` that is called before any (sub-)command's `action` function is called.
-  // - in the `preAction` hook, we call `loadEnv` that will load all env files defined in the `.env` file.
-  // - this way, all env variables will be defined in the subcommand's `action` methods (and children).
-  // this is based on https://github.com/tj/commander.js/issues/563#issuecomment-520112985 but replaxes `.on` with `.action`, which works better.
-  // WARNING: this means that the env variables can't be used in (sub-)commands BEFORE `action()` has been called.
+  // NOTE: before running, it is necessary to load env variables.
   const cli = new Command();
   cli
     .name("aiiinotate")
     .description(desc)
-    .usage("--env <path-to-your-env-file> -- <command> [options]")
-    .addOption(envFileOpt)
-    .hook("preAction", (thisCommand, actionCommand) => {
-      loadEnv(thisCommand.opts().env);
-    })
     .addCommand(makeServeCommand())
     .addCommand(makeImportCommand())
     .addCommand(makeMigrateCommand());
@@ -56,4 +41,4 @@ function makeCli() {
   return cli;
 }
 
-await makeCli();
+makeCli();

package/cli/migrate.js

@@ -112,7 +112,8 @@ function makeMigrateCommand() {
     new Argument("<migration-op>", "name of migration operation").choices(allowedMigrateOp);
 
   const migrationNameOpt =
-    new Option("-n, --migration-name <name>", "name of migration (for 'make' argument)");
+    new Option("-n, --migration-name <name>", "name of migration (for 'make' argument)")
+      .makeOptionMandatory();
 
   return new Command("migrate")
     .description("run database migrations")

package/cli/serve.js

@@ -2,13 +2,13 @@ import serve from "#src/server.js";
 
 import { Command, Option, Argument } from "commander";
 
-/** @typedef {import("#types").RerveModeType} RerveModeType */
+/** @typedef {import("#types").ServeModeType} ServeModeType */
 
-const serveModeValues = ["test", "dev", "prod"];
+const serveModeValues = ["dev", "prod"];
 
 /**
  * @param {import('commander').Command} command
- * @param {RerveModeType} serveMode
+ * @param {ServeModeType} serveMode
  */
 async function action(command, serveMode) {
   await serve(serveMode);

package/cli/utils/fastifyClient.js

@@ -0,0 +1,74 @@
+import build from "#src/app.js";
+
+/** @typedef {import("#types").FastifyInstanceType} FastifyInstanceType */
+/** @typedef {import("#types").FastifyReplyType} FastifyReplyType */
+/** @typedef {import("#types").InsertResponseType} InsertResponseType */
+
+/**
+ * a client to interact with the fastify app.
+ *
+ * @example
+ *  import FastifyClient from "#cli/utils/fastifyClient.js";
+ *  const fastify = new FastifyClient();
+ *  // load the fastify instance asynchronously
+ *  await fastify.build();
+ *  // run operations
+ *  await fastify.importAnnotationList(...);
+ */
+class FastifyClient {
+  constructor () { }
+
+  // NOTE: instanciating this.fastify is asynchronous and a constructor must be synchronous,
+  // so we use a builder to set this.fastify, following this pattern: https://stackoverflow.com/a/43433773
+  async build() {
+    /** @type {FastifyInstanceType} */
+    this.fastify = await build("default");
+    // TODO find way to actually completely disable logging because it makes the CLI UI uglyyy !
+    // log only fastify errors
+    this.fastify.log.level = "error";
+  }
+
+  async stop() {
+    await this.fastify.close();
+    this.fastify.log.info("fastify instance successfully closed.");
+  }
+
+  /**
+   * @param {string} route
+   * @returns {Promise<FastifyReplyType>}
+   */
+  injectGet(route) {
+    return this.fastify.inject({
+      method: "GET",
+      url: route,
+    })
+  }
+
+  /**
+   * @param {string} route
+   * @param {object} payload
+   * @returns {Promise<FastifyReplyType>}
+   */
+  injectPost(route, payload) {
+    return this.fastify.inject({
+      method: "POST",
+      url: route,
+      payload: payload
+    })
+  }
+
+  async importAnnotationPage(annotationPage) {
+    // TODO
+  }
+
+  /**
+   * @param {object} annotationList - IIIF 2 annotationList
+   * @returns {[number, Promise<InsertResponseType>]} - [ statusCode, response ]
+   */
+  async importAnnotationList(annotationList) {
+    const r = await this.injectPost("/annotations/2/createMany", annotationList);
+    return [ r.statusCode, r.json() ];
+  }
+}
+
+export default FastifyClient;

package/cli/utils/io.js

@@ -54,53 +54,22 @@ function fileArrayValidate (fileArr) {
 
 /**
  * `file` is a path to a file containing paths to other files (1 file per line).
- * validate all paths and return them as absolute paths
+ * parse the file and return an array of absolute paths to files.
  * @param {str} file
- * @returns {string[]}
+ * @returns {Promise<string[]>}
  */
-async function getFilesInListFile(file) {
+async function parseImportInputFile(file) {
   // read `file` split it by lines, remove empty lines
   const fileArr =
     fileRead(file)
       .split("\n")
       .filter(l => !l.match(/^\s*$/g));
-  return fileArrayValidate(fileArr);
-}
-
-/**
- * get the files to import and return them as absolute paths
- *
- * `fileArr` is a list of paths to either:
- *  - (fileArr=false) JSON files to import
- *  - (fileArr=true)  text files containing paths to the JSONS to import
- * => take `fileArr`, validate that all files exist, extract all filepaths
- * from `fileArr`, and return the array of actual JSON paths to process.
- *
- * @param {string[]} fileArr
- * @param {boolean} listFiles
- * @returns {string[]} the list of existing files to process.
- */
-function getFilesToProcess(fileArr, listFiles=false) {
-  let filesToProcess = fileArrayValidate(fileArr);
-
-  // if `listFile`, open the files containing paths of files to proces, and redo the same validation process for each file in a list file.
-  if ( listFiles ) {
-    let filesInListFiles = []
-
-    filesToProcess.map((theListFile) => {
-      const files = getFilesInListFile(theListFile);
-      filesInListFiles = filesInListFiles.concat(files);
-    })
-
-    filesToProcess = [...new Set(filesInListFiles)];  // deduplicate
-  }
-
-  return filesToProcess
+  return [...new Set(fileArrayValidate(fileArr))];
 }
 
 export {
   fileRead,
   fileOk,
   fileArrayValidate,
-  getFilesToProcess
+  parseImportInputFile
 }

package/cli/utils/mongoClient.js

@@ -1,13 +1,15 @@
 import { MongoClient } from "mongodb";
 
+import { MONGODB_CONNSTRING, MONGODB_DB } from "#constants";
+
 /**
  * load a mongo client and connect it to the database. exists if there's an error
  * @returns {import("mongodb").MongoClient}
  */
 function loadMongoClient() {
   try {
-    const client = new MongoClient(process.env.MONGODB_CONNSTRING);
-    client.db(process.env.MONGODB_DB);
+    const client = new MongoClient(MONGODB_CONNSTRING);
+    client.db(MONGODB_DB);
     return client;
   } catch (err) {
     console.error(`mongoClient: could not connect to DB because of error ${err.message}`);

package/cli/utils/progressbar.js

@@ -0,0 +1,86 @@
+/**
+ * a progress bar
+ * @example
+ * const total = 100;
+ * const ms = 10;
+ * const pb = new ProgressBar({ desc: "Progressbar example", total: total });
+ * const sleep = (pb, i) => {
+ *     pb.update(i);
+ *     return new Promise((res, rej) => setTimeout(res, ms));
+ * };
+ * for (let i=0; i<=total; i++) {
+ *     await sleep(pb, i);
+ * }
+ */
+class ProgressBar {
+  constructor(options = {
+    desc: "Progress",
+    total: 100
+  }) {
+    this.desc = options.desc
+    this.total = options.total
+    this.current = 0;
+    this.startTime = Date.now();
+  }
+
+  update(current) {
+    this.current = Math.min(current, this.total);
+    this.render();
+  }
+
+  render() {
+    const terminalWidth = process.stdout.columns || 80;
+
+    const percent = this.total === 0 ? 100 : (this.current / this.total) * 100;
+    const percentStr = percent.toFixed(1) + "%";
+
+    // calculate elapsed and estimated time
+    const elapsedTime = Date.now() - this.startTime;
+    const elapsedSec = Math.floor(elapsedTime / 1000);
+    const rate = this.current / (elapsedTime / 1000);
+    const remainingTime = rate > 0 ? Math.floor((this.total - this.current) / rate) : 0;
+
+    const timeStr = `${this.formatTime(elapsedSec)}/${this.formatTime(remainingTime)}`;
+    const countStr = `${this.current}/${this.total}`;
+
+    // calculate available space for bar
+    const prefix = `${this.desc} [`;
+    const suffix = `] ${percentStr} ${countStr} ${timeStr}`;
+    const availableWidth = terminalWidth - prefix.length - suffix.length;
+
+    if (availableWidth < 10) {
+      process.stdout.write(`\r${this.desc}: ${percentStr}`);
+      return;
+    }
+
+    // build progress bar
+    const filledWidth = Math.floor((this.current / this.total) * availableWidth);
+    const emptyWidth = availableWidth - filledWidth;
+    const bar = "█".repeat(filledWidth) + "░".repeat(emptyWidth);
+
+    // build and display complete line
+    const line = prefix + bar + suffix;
+    process.stdout.write("\r" + line);
+
+    // new line when complete
+    if (this.current === this.total) {
+      process.stdout.write("\n");
+    }
+  }
+
+  formatTime(seconds) {
+    const h = Math.floor(seconds / 3600);
+    const m = Math.floor((seconds % 3600) / 60);
+    const s = seconds % 60;
+
+    if (h > 0) {
+      return `${h}h${m}m`;
+    } else if (m > 0) {
+      return `${m}m${s}s`;
+    } else {
+      return `${s}s`;
+    }
+  }
+}
+
+export default ProgressBar

package/config/.env.template

@@ -14,8 +14,15 @@ AIIINOTATE_HOST=127.0.0.1
 # HTTP scheme: HTTP or HTTPS. should be HTTP in dev and in docker
 AIIINOTATE_SCHEME=http
 
+# where to output logs. either: "stdout"|"file"|"stdout+file"|"off"
+AIIINOTATE_LOG_TARGET=stdout
+# directory to save logs to (if AIIINOTATE_LOG_TARGET enables logging to file)
+AIIINOTATE_LOG_DIR=
+
 # max number of items to display per result page
 AIIINOTATE_PAGE_SIZE=5000
+# "true"|"false". strict error throwing when inserting annotations. equivalent to `throwOnCanvasIndexError=true` (raise an error if an annotation's target manifest can't be found) and `throwOnXywhError=true` (raise an error if you can't extract a bounding box for an annotation's target). use in controlled environments where you know precisely the structure of your annotations and where you know that your manifests are accessible through HTTP.
+AIIINOTATE_STRICT_MODE=false
 
 # IGNORE
 AIIINOTATE_BASE_URL="$AIIINOTATE_SCHEME://$AIIINOTATE_HOST:$AIIINOTATE_PORT"