aiiinotate 0.2.2 → 0.2.6

package/README.md

@@ -1,61 +1,117 @@
 # aiiinotate
 
-aiiinotate is a fast and lightweight annotations server for IIIF. It relies on `nodejs/fastify` and `mongodb` and provides an API to read/write/update/delete IIIF annotations and index manifests.
+aiiinotate is a fast and lightweight annotation server for IIIF. It relies on `nodejs/fastify` and `mongodb` and provides an API to read/write/update/delete IIIF annotations and index manifests.
 
 ---
 
-## Install
+## PROD USAGE
+
+### Install 
+
+TODO: install mongodb
+
+```bash
+npm install aiiinotate
+```
+
+### Usage
+
+The base command is:
+
+```bash
+aiiinotate --env <path-to-your-env-file> -- <command>
+```
+
+It will give full access to the CLI interface of Aiiinotate.
+
+####  Run the app 
+
+0. **Setup your `.env`** file after [.env.template](./config/.env.template).
+
+1. **Start `mongod`**
+
+```bash
+sudo systemctl start mongod
+```
+
+2. **Create and configure the database**
+
+```bash
+aiiinotate --env <path-to-your-env-file> -- migrate apply
+```
+
+3. **Run**
+
+```bash
+aiiinotate --env <path-to-your-env-file> -- serve prod
+# or 
+aiiinotate --env <path-to-your-env-file> -- serve dev
+```
+
+#### Import data
+
+TODO
+
+---
+
+## DEV USAGE
+
+### Install 
 
 ```bash
 bash setup.sh
 ```
 
---- 
+### Usage
 
-## Usage
+#### First steps
 
-Start the app in dev:
+0. **Setup your `.env`** file after [.env.template](./config/.env.template) and place it at `./config/.env`.
+
+1. **Start `mongod`**
 
 ```bash
-npm start
+sudo systemctl start mongod
 ```
 
-Start the app in prod:
+#### Run commands
+
+- **Start the app**
 
 ```bash
-npm prod
+npm run start
 ```
 
-Test the app:
+- **Test the app**
 
 ```bash
-npm test
+npm run test
 ```
 
-Run the CLI:
+- **Run the CLI**
 
 ```bash
 npm cli
 ```
 
-Process migrations:
+- **Process migrations**
 
 ```bash
 # create a new migration. NOTE: the `--` is necessary !
-npm run migrate-make -- --migrate-name <your migration name>
+npm run migrate make -- --migrate-name <your migration name>
 
 # apply all pending migrations
-npm run migrate-apply
+npm run migrate apply
 
 # revert the last migration
-npm run migrate-revert
+npm run migrate revert
 
 # revert all migrations
-npm run migrate-revert-all)
+npm run migrate revert-all)
 ```
 
 ---
 
 ## License
 
-MIT License
+GNU GPL 3.0.

package/{run.sh → _run.sh}

@@ -3,7 +3,7 @@
 source "./scripts/utils.sh";
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-ENV_FILE="$SCRIPT_DIR/src/config/.env";
+ENV_FILE="$SCRIPT_DIR/config/.env";
 
 print_usage() {
     cat<<EOF
@@ -31,10 +31,7 @@ start () {
 
     start_mongod
 
-    if [ "$mode" = "setup" ]; then
-        dotenvx run -f "$ENV_FILE" -- \
-        node "$SCRIPT_DIR/scripts/setup.js";
-    elif [ "$mode" = "dev" ]; then
+    if [ "$mode" = "dev" ]; then
         dotenvx run -f "$ENV_FILE" -- \
         node --watch "$SCRIPT_DIR/src/server.js";
     elif [ "$mode" = "test" ]; then

package/cli/import.js

@@ -2,7 +2,8 @@ import { Command, Option, Argument } from "commander";
 
 import Annotations2 from "#annotations/annotations2.js";
 import Annotations3 from "#annotations/annotations3.js";
-import { getFilesToProcess, fileRead } from "#cli/io.js";
+import { getFilesToProcess, fileRead } from "#cli/utils/io.js";
+import loadMongoClient from "#cli/utils/mongoClient.js";
 
 ////////////////////////////////////////
 
@@ -70,12 +71,12 @@ async function importAnnotationList(annotations2, fileArr, iiifVersion) {
 
 /**
  * run the cli
+ * @param {import('commander').Command} command
  * @param {string} dataType: one of importTypes
  * @param {object} options
- * @param {import('commander').Command} command
- * @param {import('mongodb').MongoClient} mongoClient
  */
-async function action(mongoClient, command, dataType, options) {
+async function action(command, dataType, options) {
+  const mongoClient = loadMongoClient();
 
   /** @type {2 | 3} */
   const iiifVersion = options.iiifVersion;
@@ -86,7 +87,7 @@ async function action(mongoClient, command, dataType, options) {
 
   checkAllowedImportType(iiifVersion, dataType);
 
-  const filesToProcess = await getFilesToProcess(files, listFiles);
+  const filesToProcess = getFilesToProcess(files, listFiles);
 
   const annotations2 = new Annotations2(
     mongoClient,

package/cli/index.js

@@ -1,3 +1,5 @@
+#!/usr/bin/env node
+
 /**
  * command line interface. run through the package.json.
  * usage: npm run cli import -- [args] [opts]
@@ -7,20 +9,51 @@
  * here, and then pass it down to all the other scripts.
  */
 
-import { Command } from "commander";
+import { Command, Option } from "commander";
+
+// import dotenvx from "dotenvx";
 
-import makeMongoClient from "#cli/mongoClient.js";
+import makeMongoClient from "#cli/utils/mongoClient.js";
 import makeImportCommand from "#cli/import.js";
 import makeMigrateCommand from "#cli/migrate.js";
+import makeServeCommand from "#cli/serve.js";
+import loadEnv from "#cli/utils/env.js";
+
+
+function makeCli() {
+
+  const desc =
+    "Command line interface for aiiinotate.\n\n"
+    + `All commands are accessible through this CLI: starting the app,
+    managing and running migrations, importing and exporting data.
+    Run individual commands to see command-specific help.
+    `.replace(/^\s+/gm, "");
 
-const cli = new Command();
+  const envFileOpt =
+    new Option("--env <env-file>", "path to .env file").makeOptionMandatory();
 
-const mongoClient = await makeMongoClient();
+  // NOTE: how do we load the env variables ? it's a bit unorthodox:
+  // - the CLI requires to use a global `--env` option with a path to the .env file.
+  // - we use the hook `preAction` that is called before any (sub-)command's `action` function is called.
+  // - in the `preAction` hook, we call `loadEnv` that will load all env files defined in the `.env` file.
+  // - this way, all env variables will be defined in the subcommand's `action` methods (and children).
+  // this is based on https://github.com/tj/commander.js/issues/563#issuecomment-520112985 but replaxes `.on` with `.action`, which works better.
+  // WARNING: this means that the env variables can't be used in (sub-)commands BEFORE `action()` has been called.
+  const cli = new Command();
+  cli
+    .name("aiiinotate")
+    .description(desc)
+    .usage("--env <path-to-your-env-file> -- <command> [options]")
+    .addOption(envFileOpt)
+    .hook("preAction", (thisCommand, actionCommand) => {
+      loadEnv(thisCommand.opts().env);
+    })
+    .addCommand(makeServeCommand())
+    .addCommand(makeImportCommand())
+    .addCommand(makeMigrateCommand());
 
-cli
-  .name("aiiinotate-cli")
-  .description("utility command line interfaces for aiiinotate")
-  .addCommand(makeImportCommand(mongoClient))
-  .addCommand(makeMigrateCommand(mongoClient));
+  cli.parse(process.argv);
+  return cli;
+}
 
-cli.parse(process.argv);
+await makeCli();

package/cli/migrate.js

@@ -15,6 +15,8 @@ import { execSync } from "node:child_process"
 
 import { Command, Option, Argument } from "commander";
 
+import loadMongoClient from "#cli/utils/mongoClient.js";
+
 
 /** @typedef {"make"|"apply"|"revert"|"revert-all"} MigrateOpType */
 const allowedMigrateOp = ["make", "apply", "revert", "revert-all"];
@@ -81,14 +83,12 @@ function migrateRevertAll() {
 
 /**
  * run the cli
- * @param {import('mongodb').MongoClient} mongoClient
  * @param {import('commander').Command} command
  * @param {MigrateOpType} mongoClient
  * @param {object} options
  */
-function action(mongoClient, command, migrationOp, options) {
+async function action(command, migrationOp, options) {
   const { migrationName } = options;
-  console.log(">>>", migrationName, options)
 
   switch (migrationOp) {
     case ("make"):
@@ -106,7 +106,7 @@ function action(mongoClient, command, migrationOp, options) {
   }
 }
 
-function makeMigrateCommand(mongoClient) {
+function makeMigrateCommand() {
   const migrationOpArg =
     new Argument("<migration-op>", "name of migration operation").choices(allowedMigrateOp);
 
@@ -117,7 +117,7 @@ function makeMigrateCommand(mongoClient) {
     .description("run database migrations")
     .addArgument(migrationOpArg)
     .addOption(migrationNameOpt)
-    .action((migrationOp, options, command) => action(mongoClient, command, migrationOp, options))
+    .action((migrationOp, options, command) => action(command, migrationOp, options))
 }
 
 export default makeMigrateCommand;

package/cli/serve.js

@@ -0,0 +1,29 @@
+import serve from "#src/server.js";
+
+import { Command, Option, Argument } from "commander";
+
+/** @typedef {import("#types").RerveModeType} RerveModeType */
+
+const serveModeValues = ["test", "dev", "prod"];
+
+/**
+ * @param {import('commander').Command} command
+ * @param {RerveModeType} serveMode
+ */
+async function action(command, serveMode) {
+  await serve(serveMode);
+}
+
+function makeServeCommand() {
+
+  const serveModeArg =
+    new Argument("<run-mode>", "mode with which to run the app")
+      .choices(serveModeValues);
+
+  return new Command("serve")
+    .description("run Aiiinotate. <run-mode>")
+    .addArgument(serveModeArg)
+    .action((serveMode, command) => action(command, serveMode));
+}
+
+export default makeServeCommand;

package/cli/utils/env.js

@@ -0,0 +1,17 @@
+import dotenvx from "@dotenvx/dotenvx";
+
+import { fileOk } from "#cli/utils/io.js";
+
+/**
+ * check that the `--env` provided by the user exists. if it exists, load variables, otherwise exit.
+ * @param {string} envPath
+ */
+async function loadEnv(envPath) {
+  if ( !fileOk(envPath) ){
+    console.error(`env.loadEnv: envPath provided by '--env' not found at '${envPath}'. exiting...`);
+    process.exit(1);
+  };
+  dotenvx.config({ path: envPath })
+}
+
+export default loadEnv

package/cli/utils/io.js

@@ -0,0 +1,106 @@
+import path from "path";
+import fs from "fs";
+
+
+const cwd = process.cwd();  // directory the script is run from
+
+/**
+ * convert a filepath to absolute if it is relative.
+ * @param {string} f
+ * @returns {string}
+ */
+const toAbsPath = (f) => path.isAbsolute(f) ? f : path.join(cwd, f);
+
+/** @returns {boolean} true if file `f` exists, false otherwise */
+const fileOk = (f) => {
+  f = toAbsPath(f);
+  try {
+    fs.accessSync(f, fs.constants.R_OK);
+    return true;
+  } catch (e) {
+    console.error(`io.fileOk: file does not exist or could not be read: ${f}`);
+    return false
+  }
+}
+
+/**
+ * @param {string} f
+ * @return {string?}
+ */
+const fileRead = (f) => {
+  f = toAbsPath(f);
+  try {
+    return fs.readFileSync(f, { encoding: "utf8" })
+  } catch (err) {
+    console.error(`io.fileRead: could not read file: ${f}`);
+  }
+}
+
+/**
+ * take an input array of filepaths. convert the paths to absolute, and check that the files exist
+ * the cli exits if any of the files don't exist
+ * @param {string[]} fileArr
+ * @returns { string[] } array of absolute filepaths
+ */
+function fileArrayValidate (fileArr) {
+  // convert to absolute filepaths
+  const success = fileArr.every(fileOk);
+  if (!success) {
+    console.error("io.fileArrayValidate: some files could not be accessed. exiting...");
+    process.exit(1);
+  }
+  return fileArr
+}
+
+/**
+ * `file` is a path to a file containing paths to other files (1 file per line).
+ * validate all paths and return them as absolute paths
+ * @param {str} file
+ * @returns {string[]}
+ */
+async function getFilesInListFile(file) {
+  // read `file` split it by lines, remove empty lines
+  const fileArr =
+    fileRead(file)
+      .split("\n")
+      .filter(l => !l.match(/^\s*$/g));
+  return fileArrayValidate(fileArr);
+}
+
+/**
+ * get the files to import and return them as absolute paths
+ *
+ * `fileArr` is a list of paths to either:
+ *  - (fileArr=false) JSON files to import
+ *  - (fileArr=true)  text files containing paths to the JSONS to import
+ * => take `fileArr`, validate that all files exist, extract all filepaths
+ * from `fileArr`, and return the array of actual JSON paths to process.
+ *
+ * @param {string[]} fileArr
+ * @param {boolean} listFiles
+ * @returns {string[]} the list of existing files to process.
+ */
+function getFilesToProcess(fileArr, listFiles=false) {
+  let filesToProcess = fileArrayValidate(fileArr);
+
+  // if `listFile`, open the files containing paths of files to proces, and redo the same validation process for each file in a list file.
+  if ( listFiles ) {
+    let filesInListFiles = []
+
+    filesToProcess.map((theListFile) => {
+      const files = getFilesInListFile(theListFile);
+      filesInListFiles = filesInListFiles.concat(files);
+    })
+
+    filesToProcess = [...new Set(filesInListFiles)];  // deduplicate
+  }
+
+  return filesToProcess
+}
+
+export {
+  fileRead,
+  fileOk,
+  fileArrayValidate,
+  getFilesToProcess
+}

package/cli/utils/mongoClient.js

@@ -0,0 +1,18 @@
+import { MongoClient } from "mongodb";
+
+/**
+ * load a mongo client and connect it to the database. exists if there's an error
+ * @returns {import("mongodb").MongoClient}
+ */
+function loadMongoClient() {
+  try {
+    const client = new MongoClient(process.env.MONGODB_CONNSTRING);
+    client.db(process.env.MONGODB_DB);
+    return client;
+  } catch (err) {
+    console.error(`mongoClient: could not connect to DB because of error ${err.message}`);
+    process.exit(1);
+  }
+}
+
+export default loadMongoClient;

package/docs/env.md

@@ -0,0 +1,25 @@
+# Managing and using env variables
+
+## In general
+
+Env variables must be stored in a `.env` file that contains the same variables as the one in `config/.env.template`. 
+
+All commands except testing are launched through the CLI. We leverage that to load env files at CLI level, making them accessible to the app and all processes initiated from the CLI:
+- the CLI requires a global option `--env` with a path to your env file. 
+- when defining the CLI, a lifecycle hook is used: `preAction`. After the CLI has been run, but before any action has been run, `cli.hook("preAction")` is used to load all env variables using `dotenvx`.
+- within the subcommand's `action` hook, and the rest of app, the env variables will now be defined.
+- this circumvents a limitation of `commander` (our CLI library) that does not allow passing global options to subcommands. It also allows to load directly all env variables at the root of the script, instead of having to redo it in each subcommand.
+
+Testing is the only command that doesn't use the CLI. We use `dotenvx` to load the env variables and pass them to the tested functions:
+
+```bash
+dotenvx run -f ./config/.env -- node --test --test-isolation=none
+```
+
+## In dev
+
+In dev, your `.env` file must be located at `$root/config/.env`.
+
+## In prod
+
+In prod, use the `--env` option to specify the path to your env variable.

package/package.json

@@ -1,28 +1,22 @@
 {
   "name": "aiiinotate",
-  "version": "0.2.2",
+  "version": "0.2.6",
   "description": "a fast IIIF-compliant annotation server",
-  "main": "index.js",
+  "main": "./cli/index.js",
   "type": "module",
   "directories": {
     "doc": "docs"
   },
   "bin": {
-    "migrate": "npm run cli -- migrate",
-    "run": "bash run.sh -p",
-    "aiiinotate": "npm run cli"
+    "aiiinotate": "./cli/index.js"
   },
   "scripts": {
-    "setup": "bash run.sh -s",
-    "start": "bash run.sh -d",
-    "prod": "bash run.sh -p",
-    "test": "bash run.sh -t",
-    "cli": "dotenvx run -f ./src/config/.env -- node ./cli/index.js",
+    "cli": "node ./cli/index.js --env=./config/.env",
+    "setup": "npm run cli migrate apply",
+    "start": "npm run cli serve dev",
+    "test": "dotenvx run -f ./config/.env -- node --test --test-isolation=none",
     "lint": "npx eslint --fix",
-    "migrate-make": "npm run cli -- migrate make",
-    "migrate-apply": "npm run cli migrate apply",
-    "migrate-revert": "npm run cli migrate revert",
-    "migrate-revert-all": "npm run cli migrate revert-all"
+    "migrate": "npm run cli -- migrate"
   },
   "pre-commit": [
     "lint"
@@ -50,7 +44,7 @@
     "#migrations/*.js": "./migrations/*.js",
     "#db/*.js": "./src/db/*.js",
     "#schemas/*.js": "./src/schemas/*.js",
-    "#config/*.js": "./src/config/*.js",
+    "#config/*.js": "./config/*.js",
     "#data/*.js": "./src/data/*.js",
     "#utils/*.js": "./src/data/utils/*.js",
     "#fileServer/*.js": "./src/fileServer/*.js",

package/src/server.js

@@ -5,11 +5,14 @@
 import build from "#src/app.js";
 
 /**
- * @param {object} options
+ * @param {import("#types").RerveModeType} serveMode
  */
-async function start (options) {
+async function server (serveMode) {
+  if (["dev", "prod"].includes(serveMode)) {
+    serveMode = "default";
+  }
 
-  const fastify = await build();
+  const fastify = await build(serveMode);
 
   try {
     fastify.listen({ port: process.env.APP_PORT });
@@ -19,4 +22,4 @@ async function start (options) {
   }
 };
 
-start();
+export default server;

package/src/types.js

@@ -15,6 +15,8 @@
 
 /** @typedef {import("ajv").ValidateFunction} AjvValidateFunctionType */
 
+/** @typedef {"test"|"dev"|"prod"} RerveModeType */
+
 /** @typedef {"uri"|"manifestShortId"|"canvasUri"} AnnotationsDeleteKeyType */
 
 /** @typedef {2|3} IiifPresentationVersionType */

package/cli/io.js

@@ -1,105 +0,0 @@
-import path from "path";
-import fs from "fs";
-
-const cwd = process.cwd();  // directory the script is run from
-
-/** @returns {Promise<boolean>} true if file `f` exists, false otherwise */
-const fileOk = (f) =>
-  fs.promises.access(f, fs.constants.R_OK)
-    .then(() =>  true)
-    .catch(() => {
-      console.log("file does not exist or could not be read: ", f);
-      return false
-    });
-
-/**
- * @param {string} f
- * @return {Promise<string>}
- */
-const fileRead = (f) =>
-  fs.promises.readFile(f, { encoding: "utf8" })
-    .then(data => data)
-    .catch(() => {
-      console.log("error reading file: ", f);
-    });
-
-/**
- * take an input array of filepaths. convert the paths to absolute, and check that the files exist
- * the cli exits if any of the files don't exist
- * @param {string[]} fileArr
- * @returns { Promise<string[]> } array of absolute filepaths
- */
-async function fileArrayValidate (fileArr) {
-  // convert to absolute filepaths
-  fileArr = fileArr.map(f =>
-    path.isAbsolute(f) ? f : path.join(cwd, f));
-
-  // validate paths
-  // the `fileOk` map is wrapped in a `Promise.all`  because `fileOk` returns a promise,
-  // so we `await` for all file checks to be performed before ensuring that all files have been found.
-  const success = await Promise.all(
-    fileArr.map(async (f) => await fileOk(f))
-  ).then(filesExistArr =>
-    filesExistArr.every(x => x===true)
-  );
-
-  if (!success) {
-    console.log("\n\nERROR: some files could not be accessed. exiting...");
-    process.exit(1);
-  }
-  return fileArr
-}
-
-/**
- * `file` is a path to a file containing paths to other files (1 file per line).
- * validate all paths and return them as absolute paths
- * @param {str} file
- * @returns {Promise<string[]>}
- */
-async function getFilesInListFile(file) {
-  return fileRead(file)
-    .then(content =>
-      content.split("\n").filter(l => !l.match(/^\s*$/g)) )
-    .then(fileArrayValidate);
-}
-
-/**
- * get the files to import and return them as absolute paths
- *
- * `fileArr` is a list of paths to either:
- *  - (fileArr=false) JSON files to import
- *  - (fileArr=true)  text files containing paths to the JSONS to import
- * => take `fileArr`, validate that all files exist, extract all filepaths
- * from `fileArr`, and return the array of actual JSON paths to process.
- *
- * NOTE: file order is NOT PRESERVED since files are opened using async pools
- *
- * @param {string[]} fileArr
- * @param {boolean} listFiles
- * @returns {string[]}
- */
-async function getFilesToProcess(fileArr, listFiles=false) {
-  let filesToProcess = await fileArrayValidate(fileArr);
-
-  // if `listFile`, open the files containing paths of files to proces, and redo the same validation process for each file in a list file.
-  if ( listFiles ) {
-    let filesInListFiles = []
-
-    await Promise.all(filesToProcess.map(async (theListFile) => {
-      const files = await getFilesInListFile(theListFile);
-      filesInListFiles = filesInListFiles.concat(files);
-    }))
-
-    filesToProcess = [...new Set(filesInListFiles)];  // deduplicate
-  }
-
-  return filesToProcess
-}
-
-
-export {
-  fileRead,
-  fileOk,
-  fileArrayValidate,
-  getFilesToProcess
-}

package/cli/mongoClient.js

@@ -1,11 +0,0 @@
-import { MongoClient } from "mongodb";
-
-export default async function() {
-  const client = new MongoClient(process.env.MONGODB_CONNSTRING);
-  try {
-    client.db(process.env.MONGODB_DB);  // client.db(config.mongodbName);
-    return client;
-  } catch (err) {
-    console.log("cli/mongoClient: error connecting", err);
-  }
-}