aiiinotate 0.2.0

package/README.md

@@ -0,0 +1,61 @@
+# aiiinotate
+
+aiiinotate is a fast and lightweight annotations server for IIIF. It relies on `nodejs/fastify` and `mongodb` and provides an API to read/write/update/delete IIIF annotations and index manifests.
+
+---
+
+## Install
+
+```bash
+bash setup.sh
+```
+
+--- 
+
+## Usage
+
+Start the app in dev:
+
+```bash
+npm start
+```
+
+Start the app in prod:
+
+```bash
+npm prod
+```
+
+Test the app:
+
+```bash
+npm test
+```
+
+Run the CLI:
+
+```bash
+npm cli
+```
+
+Process migrations:
+
+```bash
+# create a new migration. NOTE: the `--` is necessary !
+npm run migrate-make -- --migrate-name <your migration name>
+
+# apply all pending migrations
+npm run migrate-apply
+
+# revert the last migration
+npm run migrate-revert
+
+# revert all migrations
+npm run migrate-revert-all)
+```
+
+---
+
+## License
+
+MIT License

package/cli/import.js

@@ -0,0 +1,142 @@
+import { Command, Option, Argument } from "commander";
+
+import Annotations2 from "#annotations/annotations2.js";
+import Annotations3 from "#annotations/annotations3.js";
+import { getFilesToProcess, fileRead } from "#cli/io.js";
+
+////////////////////////////////////////
+
+// allowed imports
+const importTypes = [
+  "annotation",  // import a single annotation
+  "annotation-list",  // import a IIIF 2.x annotationList
+  "annotation-page",  // import a IIIF 3.x annotationPage
+  "manifest",  // import a single manifest
+  // "annotation-array",  // import a JSON array of IIIF annotations
+  // "manifest-array"  // import a json array of manifests
+]
+
+// allowed import types per IIIF version
+const allowedImportTypes = {
+  2: ["annotation", "annotation-list", "manifest"],
+  3: ["annotation", "annotation-page", "manifest"]
+}
+
+const checkAllowedImportType = (iiifVersion, dataType) => {
+  if (
+    ! allowedImportTypes[iiifVersion].includes(dataType)
+  ) {
+    console.error(`${checkAllowedImportType.name}: forbidden import type '${dataType}' for IIIF version '${iiifVersion}'. allowed import types are: ${allowedImportTypes[iiifVersion]}`);
+    process.exit(1);
+  };
+
+}
+
+const notImplementedExit = (method) => {
+  console.log(`\n\nERROR: import is not implemented '${method}'`);
+  process.exit(1);
+}
+
+const parseNumber = (x) => Number(x);
+
+////////////////////////////////////////
+
+async function importAnnotationPage(annotations3, fileArr, iiifVersion) {
+  notImplementedExit(`${importAnnotationPage.name} is not implemented`)
+}
+
+/**
+ *
+ * @param {Annotations2} annotations2
+ * @param {string[]} fileArr
+ * @param {2|3} iiifVersion
+ */
+async function importAnnotationList(annotations2, fileArr, iiifVersion) {
+  // RUN THE SCRIPT:
+  // > npm run migrate-revert && npm run migrate-apply && npm run cli import -- annotation-list -i 2 -f ./data/aikon_wit9_man11_anno165_annotation_list.jsonld
+  let totalImports = 0
+
+  for (const file of fileArr) {
+    const annotationList = JSON.parse(await fileRead(file));
+    const result = await annotations2.insertAnnotationList(annotationList);
+    totalImports += Object.keys(result).length;
+  }
+
+  console.log(`\n\nDONE: imported ${totalImports} annotations into Aiiinotate !`);
+  return
+}
+
+////////////////////////////////////////
+
+/**
+ * run the cli
+ * @param {string} dataType: one of importTypes
+ * @param {object} options
+ * @param {import('commander').Command} command
+ * @param {import('mongodb').MongoClient} mongoClient
+ */
+async function action(mongoClient, command, dataType, options) {
+
+  /** @type {2 | 3} */
+  const iiifVersion = options.iiifVersion;
+  /** @type {string[]} */
+  const files = options.files;
+  /** @type {boolean} */
+  const listFiles = options.listFiles;
+
+  checkAllowedImportType(iiifVersion, dataType);
+
+  const filesToProcess = await getFilesToProcess(files, listFiles);
+
+  const annotations2 = new Annotations2(
+    mongoClient,
+    mongoClient.db()
+  );
+
+  // run
+  switch (dataType) {
+    case ("annotation-list"):
+      await importAnnotationList(annotations2, filesToProcess, iiifVersion);
+      break;
+    default:
+      notImplementedExit(dataType);
+  }
+  mongoClient.close();
+
+}
+
+/////////////////////////////////////////
+
+/** define the cli */
+function makeImportCommand(mongoClient) {
+
+  // argument and option name syntax:
+  // --opt-name <requiredVal> => you mst provide a value after --opt-name
+  // --opt-name [optionalVal] => if `optionalVal` is not provided, --opt-name will be treated as boolean
+  const dataTypeArg =
+    new Argument("<data-type>", "type of data to import")
+      .choices(importTypes);
+
+  const versionOpt =
+    new Option("-i, --iiif-version <version>", "IIIF version")
+      .choices(["2", "3"])
+      .argParser(parseNumber)
+      .makeOptionMandatory();
+
+  const filesOpt =
+    new Option("-f, --files <files...>", "files to process, either as: space-separated filepath(s) to the JSON(s) OR path to a file containing a list of paths to JSON files to process (1 line per path)")
+      .makeOptionMandatory();
+
+  const listFilesOpt =
+    new Option("-l, --list-files", "flag indicating that --files points to a file containing a list of JSON files to process (1 line per path)")
+
+  return new Command("import")
+    .description("import data into aiiinotate")
+    .addArgument(dataTypeArg)
+    .addOption(filesOpt)
+    .addOption(versionOpt)
+    .addOption(listFilesOpt)
+    .action((dataType, options, command) => action(mongoClient, command, dataType, options))
+}
+
+export default makeImportCommand;

package/cli/index.js

@@ -0,0 +1,26 @@
+/**
+ * command line interface. run through the package.json.
+ * usage: npm run cli import -- [args] [opts]
+ *
+ * NOTE: node recommends only creating 1 mongoclient per app
+ * when possible for performance reasons => we create a global mongoClient
+ * here, and then pass it down to all the other scripts.
+ */
+
+import { Command } from "commander";
+
+import makeMongoClient from "#cli/mongoClient.js";
+import makeImportCommand from "#cli/import.js";
+import makeMigrateCommand from "#cli/migrate.js";
+
+const cli = new Command();
+
+const mongoClient = await makeMongoClient();
+
+cli
+  .name("aiiinotate-cli")
+  .description("utility command line interfaces for aiiinotate")
+  .addCommand(makeImportCommand(mongoClient))
+  .addCommand(makeMigrateCommand(mongoClient));
+
+cli.parse(process.argv);

package/cli/io.js

@@ -0,0 +1,105 @@
+import path from "path";
+import fs from "fs";
+
+const cwd = process.cwd();  // directory the script is run from
+
+/** @returns {Promise<boolean>} true if file `f` exists, false otherwise */
+const fileOk = (f) =>
+  fs.promises.access(f, fs.constants.R_OK)
+    .then(() =>  true)
+    .catch(() => {
+      console.log("file does not exist or could not be read: ", f);
+      return false
+    });
+
+/**
+ * @param {string} f
+ * @return {Promise<string>}
+ */
+const fileRead = (f) =>
+  fs.promises.readFile(f, { encoding: "utf8" })
+    .then(data => data)
+    .catch(() => {
+      console.log("error reading file: ", f);
+    });
+
+/**
+ * take an input array of filepaths. convert the paths to absolute, and check that the files exist
+ * the cli exits if any of the files don't exist
+ * @param {string[]} fileArr
+ * @returns { Promise<string[]> } array of absolute filepaths
+ */
+async function fileArrayValidate (fileArr) {
+  // convert to absolute filepaths
+  fileArr = fileArr.map(f =>
+    path.isAbsolute(f) ? f : path.join(cwd, f));
+
+  // validate paths
+  // the `fileOk` map is wrapped in a `Promise.all`  because `fileOk` returns a promise,
+  // so we `await` for all file checks to be performed before ensuring that all files have been found.
+  const success = await Promise.all(
+    fileArr.map(async (f) => await fileOk(f))
+  ).then(filesExistArr =>
+    filesExistArr.every(x => x===true)
+  );
+
+  if (!success) {
+    console.log("\n\nERROR: some files could not be accessed. exiting...");
+    process.exit(1);
+  }
+  return fileArr
+}
+
+/**
+ * `file` is a path to a file containing paths to other files (1 file per line).
+ * validate all paths and return them as absolute paths
+ * @param {str} file
+ * @returns {Promise<string[]>}
+ */
+async function getFilesInListFile(file) {
+  return fileRead(file)
+    .then(content =>
+      content.split("\n").filter(l => !l.match(/^\s*$/g)) )
+    .then(fileArrayValidate);
+}
+
+/**
+ * get the files to import and return them as absolute paths
+ *
+ * `fileArr` is a list of paths to either:
+ *  - (fileArr=false) JSON files to import
+ *  - (fileArr=true)  text files containing paths to the JSONS to import
+ * => take `fileArr`, validate that all files exist, extract all filepaths
+ * from `fileArr`, and return the array of actual JSON paths to process.
+ *
+ * NOTE: file order is NOT PRESERVED since files are opened using async pools
+ *
+ * @param {string[]} fileArr
+ * @param {boolean} listFiles
+ * @returns {string[]}
+ */
+async function getFilesToProcess(fileArr, listFiles=false) {
+  let filesToProcess = await fileArrayValidate(fileArr);
+
+  // if `listFile`, open the files containing paths of files to proces, and redo the same validation process for each file in a list file.
+  if ( listFiles ) {
+    let filesInListFiles = []
+
+    await Promise.all(filesToProcess.map(async (theListFile) => {
+      const files = await getFilesInListFile(theListFile);
+      filesInListFiles = filesInListFiles.concat(files);
+    }))
+
+    filesToProcess = [...new Set(filesInListFiles)];  // deduplicate
+  }
+
+  return filesToProcess
+}
+
+
+export {
+  fileRead,
+  fileOk,
+  fileArrayValidate,
+  getFilesToProcess
+}

package/cli/migrate.js

@@ -0,0 +1,123 @@
+/**
+ * run and apply migrations.
+ *
+ * the commands here are wrappers for migrate-mongo.
+ * the big particularity is that we handle 2 databases in parrallel:
+ * a dev/prod database and a test database (that will be populated
+ * by running tests, emptied after running the tests)
+ * in turn, we need to apply migrations in parrallel to both databases.
+ *
+ */
+import path from "node:path";
+import fs from "node:fs";
+import { fileURLToPath } from "node:url";
+import { execSync } from "node:child_process"
+
+import { Command, Option, Argument } from "commander";
+
+
+/** @typedef {"make"|"apply"|"revert"|"revert-all"} MigrateOpType */
+const allowedMigrateOp = ["make", "apply", "revert", "revert-all"];
+
+const
+  // path to current dirctory
+  dirCli = path.dirname(fileURLToPath(import.meta.url)),
+  dirRoot = path.resolve(dirCli, ".."),
+  dirMigrations = path.resolve(dirRoot, "migrations"),
+  dirMigrationsScripts = path.resolve(dirMigrations, "migrationScripts"),
+  migrationsConfigMain = path.resolve(dirMigrations, "migrate-mongo-config-main.js"),
+  migrationsConfigTest = path.resolve(dirMigrations, "migrate-mongo-config-test.js"),
+  migrationConfigs = [migrationsConfigMain, migrationsConfigTest];
+
+/** return a date in YYYYMMDDhhmmss format */
+function formatDate(date) {
+  function pad2(n) {  // always returns a string
+    return (n < 10 ? "0" : "") + n;
+  }
+
+  return date.getFullYear() +
+    pad2(date.getMonth() + 1) +
+    pad2(date.getDate()) +
+    pad2(date.getHours()) +
+    pad2(date.getMinutes()) +
+    pad2(date.getSeconds());
+}
+
+/**
+ * create a single migration file
+ * @param {string} migrationName
+ */
+function migrateMake(migrationName) {
+  if ( migrationName == null ) {
+    throw new Error(`migration name must be a string. got ${migrationName}`);
+  }
+  fs.copyFileSync(
+    path.resolve(dirMigrations, "migrationTemplate.js"),
+    path.resolve(dirMigrationsScripts, `${formatDate(new Date())}-${migrationName}.js`)
+  );
+}
+
+/** apply all pending migrations */
+function migrateApply() {
+  migrationConfigs.map((mc) => execSync(`npx migrate-mongo up -f ${mc}`));
+}
+
+/** revert the last migration */
+function migrateRevert() {
+  migrationConfigs.map((mc) => execSync(`npx migrate-mongo down -f ${mc}`));
+}
+
+/** revert all migrations */
+function migrateRevertAll() {
+  // there are as many migrations as there are files in `dirMigrationsScripts`
+  // => revert one migration per migration file
+  // do this for each migration file (prod and test database).
+  migrationConfigs.map((mc) =>
+    fs.readdirSync(dirMigrationsScripts).map((_) =>
+      execSync(`npx migrate-mongo down -f ${mc}`)
+    )
+  )
+}
+
+/**
+ * run the cli
+ * @param {import('mongodb').MongoClient} mongoClient
+ * @param {import('commander').Command} command
+ * @param {MigrateOpType} mongoClient
+ * @param {object} options
+ */
+function action(mongoClient, command, migrationOp, options) {
+  const { migrationName } = options;
+  console.log(">>>", migrationName, options)
+
+  switch (migrationOp) {
+    case ("make"):
+      migrateMake(migrationName);
+      break;
+    case ("apply"):
+      migrateApply();
+      break;
+    case ("revert"):
+      migrateRevert();
+      break;
+    case ("revert-all"):
+      migrateRevertAll();
+      break;
+  }
+}
+
+function makeMigrateCommand(mongoClient) {
+  const migrationOpArg =
+    new Argument("<migration-op>", "name of migration operation").choices(allowedMigrateOp);
+
+  const migrationNameOpt =
+    new Option("-n, --migration-name <name>", "name of migration (for 'make' argument)");
+
+  return new Command("migrate")
+    .description("run database migrations")
+    .addArgument(migrationOpArg)
+    .addOption(migrationNameOpt)
+    .action((migrationOp, options, command) => action(mongoClient, command, migrationOp, options))
+}
+
+export default makeMigrateCommand;

package/cli/mongoClient.js

@@ -0,0 +1,11 @@
+import { MongoClient } from "mongodb";
+
+export default async function() {
+  const client = new MongoClient(process.env.MONGODB_CONNSTRING);
+  try {
+    client.db(process.env.MONGODB_DB);  // client.db(config.mongodbName);
+    return client;
+  } catch (err) {
+    console.log("cli/mongoClient: error connecting", err);
+  }
+}

package/docs/architecture.md

@@ -0,0 +1,88 @@
+# Architecture
+
+This is a high-level overview of the Aiiinotate and a good place to start if you want to work on the app
+
+---
+
+## Top level plugins
+
+Fastify uses a [**plugin system**](https://fastify.dev/docs/latest/Guides/Plugins-Guide/). In short, it means that, 
+- to access the global `fastify` instance, your code needs to be a plugin
+- all plugins must export a single function that registers the plugin on the global fastify instance
+
+This can be quite convoluted and conflict with a "normal" module architecture, so we do a mix of the two: 
+- **all modules at the root** of `src/` are plugins (except `config`)
+- **nested plugins** within each of the above are defined when they need to access the fastify instance. 
+
+```
+./src/fileServer/ - module storing and serving data files for test fixtures
+./src/schemas/    - JsonSchemas definition and validation
+./src/db/         - connects the app to the mongojs database
+./src/data/       - routes and modules to read/write data 
+
+```
+
+---
+
+## The `data` plugin
+
+### In general
+
+As you can see, the `data` plugin stores the vast majority of the app's logic. It defines:
+- **`route.js` files** HTTP routes for user interactions
+- **`collection classes`** (`abstractCollection.js`, `manifests(2|3).js`, `annotations(2|3).js`) for each MongoJS collection that handles all internal functionnalities for all collections (read/write/update/delete data)
+
+Each `route` and `class` is a fastify plugin. Since the `data` plugin is registered last, it can access functionalities from all other root plugins defined above.
+
+### Structure and inheritence
+
+```
+├── index.js               // `data` plugin root
+├── routes.js              // generic routes 
+├── annotations
+│   ├── annotations2.js    // plugin for IIIF presentation 2 annotations
+│   ├── annotations3.js    // plugin for IIIF presentation 3 annotations 
+│   ├── routes.js          // routes for both plugins
+├── collectionAbstract.js  // abstract class with functionnalities for all plugins
+├── manifests              
+    ├── manifests2.js      // plugin for IIIF presentation 2 manifests
+    ├── manifests3.js      // plugin for IIIF presentation 3 manifests
+    ├── routes.js          // routes for both plugins
+```
+
+At a high level, `route` files receive data and relegate internal mongoJS interaction to the `collection classes`. Once these processes are finished, the `collection classes` return data to the `routes`, which send the response to the users. So:
+
+- the work of `routes` is to define input/output JsonSchemas and delegate to the proper `collection classes`
+- the work of `classes` is to do the database interaction, data formatting etc.
+
+Our 5 `collection classes` (`collectionAbstract`, `annotations2`, `annotations3`, `manifests2`, `manifests3`) use class inheritence: all classes inherit from `collectionAbstract`
+- `collectionAbstract` defines collection-agnostic processes that can be used by all other classes.
+- other classes implement functionnalities for a specific data type and collection. 
+
+--- 
+
+## Details: why plugins are complicated ?
+
+If using a plugin-only architecture, **you should not do "local" imports** (imports from one part of your app to another): everything that you want to communicate between apps must be registered as a plugin on the global fastify instance, and then other files need to access the fastify instance:
+
+```js
+// here, we define a root plugin with 2 subplugins (1st is a decorator, 2nd a normal plugin).
+fastify.register((instance, opts, done) => {
+
+  // decorator 
+  instance.decorate('util', (a, b) => a + b)
+  console.log(instance.util('that is ', 'awesome'))
+
+  // plugin
+  fastify.register((instance, opts, done) => {
+    console.log(instance.util('that is ', 'awesome')) 
+    done()
+  })
+
+  done()
+})
+```
+
+In the above example, functionnalities defined in each plugin (except decorators) are encapsulated: a plugin is a scope and everything that's defined in a plugin must be accessed through the fastify instance. There is also the question of plugins vs. decorators, plugin definition order...
+
+In short, if using a plugin-only architecture, every single one of your files would have to be a plugin, registered in the proper order... It would be hell.

package/docs/db.md

@@ -0,0 +1,38 @@
+# DATABASE AND MIGRATIONS
+
+---
+
+## Collections
+
+---
+
+## Validation rules
+
+---
+
+## Migrations
+
+Migrations are used to specify changes to the database's structure (creation and deletion of collections and indexes, changes to collection options such as validation rules etc.). Migration is done using [`migrate-mongo`](https://github.com/seppevs/migrate-mongo), a command-line interface for npm. 
+
+*Note that here, migrations only concern changes to the structure, not changes to the data.*
+
+Where things get a bit complicated is that, while in use, our app uses a single database, in **dev mode, our app uses 2 databses**:
+- a `main` database (default database that will be put in production)
+- a `test` database (empty database to run tests, add dummy data to and so on).
+
+For consistency, `test` must mirror the structure of `main`: same collections, indexes, validation rules etc. So, we must manage our migrations so that **all changes to the structure of `main` are reflected in the structure of `test`**. This is not possible natively through `migrate-mongo`. Managing both databases in parrallel manually would risk a lot of inconsistencies.
+
+
+Our solution is to automate the management of both databases in parrallel:
+- there are **2 migration config files**, one for each database
+- both migration config files point to **the same migration scripts folder**, so that both database can apply the same migrations
+- execute all migrations by **wrapping `migrate-mongo` in a homemade script**: [`scripts/migrations.sh`](../scripts/migrations.sh).
+
+```
+root/
+  |__migrations/
+       |__migrationScripts              // folder containing all migration scripts. consumed by both migration config files.
+       |__baseConfig.js                 // base config file that both config files are derived from
+       |__migrate-mongo-config-main.js  // config file for the main db
+       |__migrate-mongo-config-test.js  // config file for the test db
+```

package/docs/dev_iiif_compatibility.md

@@ -0,0 +1,43 @@
+# Development notes: IIIF compatibility and variations between IIIF prsentation APIxs 3.x and IIIF 2.x
+
+The annotation server uses an internal data model that can convert to and from annotation models defined in IIIF 2.x and 3.x presentation API. Respectively:
+- IIIF 3.x follows the [W3C Web Annotations standard (WA)](https://www.w3.org/TR/annotation-model/)
+- IIIF 2.x follows the [Open Annotations standard (OA)] (http://www.commonsemantics.com/oa/Open%20Annotation%20Data%20Model%20Primer.html#examples)
+
+In general, there are [breaking changes](https://iiif.io/api/presentation/3.0/change-log/#1-breaking-changes) between IIIF 2.x and 3.x. For annotations, the OA model allows things that are not possible in WA, and vice versa. Our internal data model tries to store a maximum amount of data from WA and OA annotations, and then do the necessary conversion to produce valid OA/WA annotations.
+
+To make things funnier, the original Open Annotations standard website is no longer available.
+
+## Annotations and the `motivation` attribute
+
+### The problem
+
+Both OA and WA have a `motivation` attribute that describes the function of the annotation in relation to the Manifest. Mainly, the use of `painting` (WA) / `sc:painting` (OA) indicates that the annotation is the primary content of the canvas and should be rendered.
+- WA allows 2 values: `painting`, `supplementing`
+- OA allows a more values and defines an ontology. [IIIF 2.1 specifies](https://iiif.io/api/presentation/2.1/#comment-annotations) that the value given to a non-painting annotation should be `oa:commenting`
+
+In practice, in SAS/Aikon, non-painting annotations have the `motivation`: `[ "oa:tagging", "oa:supplementing" ]`.
+
+### What we do
+
+In an annotation, the `motivation` field is a `string[]` array that accepts all values it receives.
+- **when writing data from IIIF to DB**, `motivation` values are connected to string arrays but are not verified.
+- **when converting data from DB to IIIF**, `motivation` values stored in the database are processed:
+    - converting to IIIF 2.x:
+        - if WA values are stored in the DB, they are converted to 2.x: `painting => sc:painting`, `supplementing => oa:commenting`.
+        - if OA values are stored in the DB, they are kept as is
+    - converting to IIIF 3.x:
+        - if WA values are in the DB, they are kept as is
+        - if OA values are in the DB, they are converted to `painting` (if `sc:painting` is in the array of motivations stored) or `commenting` otherwise.
+
+## DCTypes and the `body.type` attribute
+
+### Problem
+
+- WA allows : `Dataset`, `Image`, `Video`, `Sound`, `Text`
+- OA allows : `text`, `image`, `sound`, `dataset`, `software`, `interactive`, `event`, `physical`, `object`
+- In IIIF 2.x, values are prefixed by `dctypes:`: `dctypes:text`.
+
+### Solution
+
+Since there is a close mapping between the two, we convert to WA allowed types before saving to database.