@sera4/essentia 1.1.13 → 1.1.15

package/README.md

@@ -7,3 +7,4 @@ A collection of small modules providing essential utilities for TWS services:
 * A logging tool
 * An HTTP response tool for consistenty formatting error responses
 * A small utility to dump the last commit of the project
+* [Paper Trail](paper-trail/README.md)

package/index.js

@@ -8,5 +8,6 @@ export { default as halDecorator } from "./hal/index.js";
 export { default as formatter } from "./formatter/index.js";
 export { HealthCheck as healthCheck } from "./health/index.js";
 export { default as queue } from "./queue/index.js";
+export { default as paperTrail } from "./paper-trail/index.js";
 export * from "./utils/index.js";
 export * from "./safe_proxy/index.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sera4/essentia",
-  "version": "1.1.13",
+  "version": "1.1.15",
   "description": "A library of utilities for Teleporte Web Services",
   "main": "index.js",
   "type": "module",
@@ -30,7 +30,8 @@
   "dependencies": {
     "amqplib": "^0.8.0",
     "async": "^2.6.3",
-    "axios": "^0.21.1",
+    "axios": "^0.21.4",
+    "deep-diff": "^1.0.2",
     "fast-safe-stringify": "^2.0.7",
     "git-rev": "^0.2.1",
     "lodash": "^4.17.21",

package/paper-trail/README.md

@@ -0,0 +1,67 @@
+# Sequelize Paper Trail
+
+A paper trail/sequelize hook plugin.  This plugin has been modified to use the node14 CLS API called AsyncLocalStorage as well as fix some of the postgres related bugs.  We have kept our own implemnetation and various changes were needed to support the express framework and pushing this to the public would hinder the general usability of this library for all.
+
+## Usage
+
+In your database setup, setup your models and then attach the paperTrail to them.  Many default options need to be changed to support underscore notation and the use of Account instead of User as the default table, thus suggestions options for TWS projects are shown below.
+
+```
+import { paperTrail } from "@sera4/essentia"
+import Sequelize from 'sequelize';
+
+...
+
+const sequelize = new Sequelize(...);
+
+PaperTrail.init(sequelize, Sequelize, { UUID: true,
+  underscored: true, enableMigration: true, underscoredAttributes: true,
+  debug: false, userModel: 'Account', enableCompression: true });
+};
+
+const db = {};
+// for any model you want enabled
+db["Tenant"]            = TenantModel(sequelize, Sequelize);
+
+db.PaperTrail = PaperTrail.init(sequelize, Sequelize, { UUID: true, underscored: true,
+  enableMigration: true, underscoredAttributes: true, continuationKey: "accountId",
+  debug: false, userModel: 'Account', userModelAttribute: 'account_id',
+  enableCompression: true });
+
+db.PaperTrail.defineModels();
+
+.. and for any model that needs a papertrail, call hasPaperTrail() on it ..
+db["Tenant"].hasPaperTrail();
+db["TenantLicense"].hasPaperTrail();
+```
+
+and similarly, to use the automatic ALS (asyncLocalStorage) for automatic setting of the account/userID - add this into your middleware
+
+```
+import db from "../database/models/index.js";
+
+let alsEnabled = db.PaperTrail.alsEnabled();
+
+const localStorageMiddleware = async (req, res, next) => {
+  await asyncLocalStorage.run(new Map(), () => {
+    if (!alsEnabled) {
+      alsEnabled = true;
+      db.PaperTrail.setupALS(asyncLocalStorage);
+    }
+
+    next();
+  });
+};
+```
+as this will ensure that the ALS context is initialized AFTER the first `run()` invocation.
+
+`Please note`: For automatic accountId fetching, it is imperitive that you add
+
+```
+  asyncLocalStorage.getStore().set("accountId", accountId);
+```
+somewhere into your express middleware so the database hooks can pickup the accountId when changes are made.
+
+## History
+
+This is a private fork/implementation of https://github.com/nielsgl/sequelize-paper-trail which has been released under MIT license.  This was privitized due to the number of commented out items/untested in their project as well as the change from CLS (continuation-local-storage) which is experimental to the supported AsyncLocalStorage which was stabilized in Node 14 as the CLS API 101.

package/paper-trail/helpers.js

@@ -0,0 +1,99 @@
+import pkg from 'deep-diff';
+import _ from 'lodash';
+
+const { diff } = pkg;
+const capitalizeFirstLetter = string =>
+	string.charAt(0).toUpperCase() + string.slice(1);
+
+const toUnderscored = obj => {
+	_.forEach(obj, (k, v) => {
+		obj[k] = v
+			.replace(/(?:^|\.?)([A-Z])/g, (x, y) => `_${y.toLowerCase()}`)
+			.replace(/^_/, '');
+	});
+	return obj;
+};
+
+const calcDelta = (current, next, exclude, strict, DEBUG = false) => {
+	if (DEBUG) {
+		console.log('current', current);
+		console.log('next', next);
+		console.log('exclude', exclude);
+	}
+
+	const di = diff(current, next);
+
+	if (DEBUG) {
+		console.log('di', di);
+
+		let diffs2 = [];
+		if (di) {
+			diffs2 = di.map(i =>
+				JSON.parse(JSON.stringify(i).replace('"__data",', '')),
+			);
+		}
+
+		console.log('diffs2', diffs2);
+
+		console.log(
+			'filter1',
+			diffs2.filter(i => i.path.join(',').indexOf('_') === -1),
+		);
+
+		console.log(
+			'filter2',
+			diffs2.filter(i => exclude.every(x => i.path.indexOf(x) === -1)),
+		);
+	}
+
+	let diffs = [];
+	if (di) {
+		diffs = di
+			.map(i => JSON.parse(JSON.stringify(i).replace('"__data",', '')))
+			.filter(i => {
+				if (!strict && i.kind === 'E') {
+					// eslint-disable-next-line eqeqeq
+					if (i.lhs != i.rhs) return i;
+				} else return i;
+				return false;
+			})
+			.filter(i => exclude.every(x => i.path.indexOf(x) === -1));
+	}
+
+	if (diffs.length > 0) {
+		return diffs;
+	}
+	return null;
+};
+
+const diffToString = val => {
+	if (typeof val === 'undefined' || val === null) 
+		return '';
+
+	if (val === true)
+		return '1';
+
+	if (val === false) 
+		return '0';
+
+	if (typeof val === 'string') 
+		return val;
+
+	if (!Number.isNaN(Number(val))) 
+		return `${String(val)}`;
+
+	if ((typeof val === 'undefined' ? 'undefined' : typeof val) === 'object') 
+		return `${JSON.stringify(val)}`;
+
+	if (Array.isArray(val)) 
+		return `${JSON.stringify(val)}`;
+
+	return '';
+};
+
+export default {
+	capitalizeFirstLetter,
+	toUnderscored,
+	calcDelta,
+	diffToString,
+};

package/paper-trail/index.js

@@ -0,0 +1,636 @@
+import * as jsdiff from 'diff';
+import _ from 'lodash';
+import helpers from './helpers.js';
+import paginator from "../paginator/s4-pagination.js";
+
+let failHard = false;
+let als;
+let Sequelize = null
+
+const paperTrail = {};
+
+paperTrail.init = (sequelize, sequelizePackage, optionsArg) => {
+	Sequelize = sequelizePackage;
+	// In case that options is being parsed as a readonly attribute.
+	// Or it is not passed at all
+	const optsArg = _.cloneDeep(optionsArg || {});
+
+	const defaultOptions = {
+		debug: false,
+		log: null,
+		exclude: [
+			'id',
+			'createdAt',
+			'updatedAt',
+			'deletedAt',
+			'created_at',
+			'updated_at',
+			'deleted_at',
+			'revision',
+		],
+		revisionAttribute: 'revision',
+		revisionModel: 'Revision',
+		revisionChangeModel: 'RevisionChange',
+		enableRevisionChangeModel: false,
+		UUID: false,
+		underscored: false,
+		underscoredAttributes: false,
+		defaultAttributes: {
+			documentId: 'documentId',
+			revisionId: 'revisionId',
+		},
+		als: null,
+		userModel: false,
+		userModelAttribute: 'userId',
+		enableCompression: false,
+		enableMigration: false,
+		enableStrictDiff: true,
+		continuationNamespace: null,
+		continuationKey: 'userId',
+		metaDataFields: null,
+		metaDataContinuationKey: 'metaData',
+		mysql: false,
+	};
+
+	if (optsArg.underscoredAttributes) {
+		helpers.toUnderscored(defaultOptions.defaultAttributes);
+	}
+
+	const options = _.defaults(optsArg, defaultOptions);
+
+	const log = options.log || console.log;
+
+	function createBeforeHook(operation) {
+
+		const beforeHook = function beforeHook(instance, opt) {
+			if (options.debug) {
+				log('beforeHook called');
+				log('instance:', instance);
+				log('opt:', opt);
+			}
+
+			if (opt.noPaperTrail) {
+				if (options.debug) {
+					log('noPaperTrail opt: is true, not logging');
+				}
+				return;
+			}
+
+			const destroyOperation = operation === 'destroy';
+
+			let previousVersion = {};
+			let currentVersion = {};
+			if (!destroyOperation && options.enableCompression) {
+				_.forEach(opt.defaultFields, a => {
+					previousVersion[a] = instance._previousDataValues[a];
+					currentVersion[a] = instance.dataValues[a];
+				});
+			} else {
+				previousVersion = instance._previousDataValues;
+				currentVersion = instance.dataValues;
+			}
+			// Supported nested models.
+			previousVersion = _.omitBy(
+				previousVersion,
+				i => i != null && typeof i === 'object' && !(i instanceof Date),
+			);
+			previousVersion = _.omit(previousVersion, options.exclude);
+
+			currentVersion = _.omitBy(
+				currentVersion,
+				i => i != null && typeof i === 'object' && !(i instanceof Date),
+			);
+			currentVersion = _.omit(currentVersion, options.exclude);
+
+			// Disallow change of revision
+			instance.set(
+				options.revisionAttribute,
+				instance._previousDataValues[options.revisionAttribute],
+			);
+
+			// Get diffs
+			const delta = helpers.calcDelta(
+				previousVersion,
+				currentVersion,
+				options.exclude,
+				options.enableStrictDiff,
+			);
+
+			const currentRevisionId = instance.get(options.revisionAttribute);
+
+			if (failHard && !currentRevisionId && opt.type === 'UPDATE') {
+				throw new Error('Revision Id was undefined');
+			}
+
+			if (options.debug) {
+				log('delta:', delta);
+				log('revisionId', currentRevisionId);
+			}
+			// Check if all required fields have been provided to the opts / ALS
+			if (options.metaDataFields) {
+				// get all required field keys as an array
+				const requiredFields = _.keys(
+					_.pickBy(
+						options.metaDataFields,
+						function isMetaDataFieldRequired(required) {
+							return required;
+						},
+					),
+				);
+				if (requiredFields && requiredFields.length) {
+          const store = options.als && options.als.getStore();
+					const metaData = (store && store.get(metaDataContinuationKey)) || opt.metaData
+					const requiredFieldsProvided = _.filter(
+						requiredFields,
+						function isMetaDataFieldNonUndefined(field) {
+							return metaData[field] !== undefined;
+						},
+					);
+					if (
+						requiredFieldsProvided.length !== requiredFields.length
+					) {
+						log(
+							'Required fields: ',
+							options.metaDataFields,
+							requiredFields,
+						);
+						log(
+							'Required fields provided: ',
+							metaData,
+							requiredFieldsProvided,
+						);
+						throw new Error(
+							'Not all required fields are provided to paper trail!',
+						);
+					}
+				}
+			}
+
+			if (destroyOperation || (delta && delta.length > 0)) {
+				const revisionId = (currentRevisionId || 0) + 1;
+				instance.set(options.revisionAttribute, revisionId);
+
+				if (!instance.context) {
+					instance.context = {};
+				}
+				instance.context.delta = delta;
+			}
+
+			if (options.debug) {
+				log('end of beforeHook');
+			}
+		};
+		return beforeHook;
+	}
+
+	function createAfterHook(operation) {
+		const afterHook = function afterHook(instance, opt) {
+			const store = options.als && options.als.getStore()
+			if (options.debug) {
+				log('afterHook called');
+				log('instance:', instance);
+				log('opt:', opt);
+
+				if (store) {
+					log(
+						`ALS ${options.continuationKey}:`,
+						store.get(options.continuationKey),
+					);
+				}
+			}
+
+			const destroyOperation = operation === 'destroy';
+
+			if (
+				instance.context &&
+				((instance.context.delta &&
+					instance.context.delta.length > 0) ||
+					destroyOperation)
+			) {
+				const Revision = sequelize.model(options.revisionModel);
+				let RevisionChange;
+
+				if (options.enableRevisionChangeModel) {
+					RevisionChange = sequelize.model(
+						options.revisionChangeModel,
+					);
+				}
+
+				const { delta } = instance.context;
+
+				let previousVersion = {};
+				let currentVersion = {};
+				if (!destroyOperation && options.enableCompression) {
+					_.forEach(opt.defaultFields, a => {
+						previousVersion[a] = instance._previousDataValues[a];
+						currentVersion[a] = instance.dataValues[a];
+					});
+				} else {
+					previousVersion = instance._previousDataValues;
+					currentVersion = instance.dataValues;
+				}
+
+				// Supported nested models.
+				previousVersion = _.omitBy(
+					previousVersion,
+					i =>
+						i != null &&
+						typeof i === 'object' &&
+						!(i instanceof Date),
+				);
+				previousVersion = _.omit(previousVersion, options.exclude);
+
+				currentVersion = _.omitBy(
+					currentVersion,
+					i =>
+						i != null &&
+						typeof i === 'object' &&
+						!(i instanceof Date),
+				);
+				currentVersion = _.omit(currentVersion, options.exclude);
+
+				if (!options.als)
+					options.als = asyncLocalStorage;
+
+        const store = options.als && options.als.getStore()
+				if (store && !store.has(options.continuationKey)) {
+					if (failHard)
+						throw new Error(
+							`The ALS continuationKey ${options.continuationKey} was not defined.`,
+						);
+				}
+
+				let document = currentVersion;
+
+				if (options.mysql) {
+					document = JSON.stringify(document);
+				}
+
+				// Build revision
+				const query = {
+					model: this.name,
+					document,
+					operation,
+				};
+
+				// Add all extra data fields to the query object
+				if (options.metaDataFields) {
+					const metaData =
+						(store && store.get(options.metaDataContinuationKey)) ||
+						opt.metaData;
+					if (metaData) {
+						_.forEach(
+							options.metaDataFields,
+							function getMetaDataValues(required, field) {
+								const value = metaData[field];
+								if (options.debug) {
+									log(
+										`Adding metaData field to Revision - ${field} => ${value}`,
+									);
+								}
+								if (!(field in query)) {
+									query[field] = value;
+								} else if (options.debug) {
+									log(
+										`Revision object already has a value at ${field} => ${query[field]}`,
+									);
+									log('Not overwriting the original value');
+								}
+							},
+						);
+					}
+				}
+
+				// in case of custom user models that are not 'userId'
+				query[options.userModelAttribute] =
+					(store && store.get(options.continuationKey)) || opt.userId;
+
+				query[options.defaultAttributes.documentId] = instance.id;
+
+				const revision = Revision.build(query);
+
+				revision[options.revisionAttribute] = instance.get(
+					options.revisionAttribute,
+				);
+
+				// Save revision(
+				return revision
+					.save({ transaction: opt.transaction })
+					.then(objectRevision => {
+						// Loop diffs and create a revision-diff for each
+						if (options.enableRevisionChangeModel) {
+							_.forEach(delta, difference => {
+								const o = helpers.diffToString(
+									difference.item
+										? difference.item.lhs
+										: difference.lhs,
+								);
+								const n = helpers.diffToString(
+									difference.item
+										? difference.item.rhs
+										: difference.rhs,
+								);
+
+								// let document = difference;
+								document = difference;
+								let diff = o || n ? jsdiff.diffChars(o, n) : [];
+
+								if (options.mysql) {
+									document = JSON.stringify(document);
+									diff = JSON.stringify(diff);
+								}
+
+								const d = RevisionChange.build({
+									path: difference.path[0],
+									document,
+									diff,
+									revisionId: objectRevision.id,
+								});
+
+								d.save({ transaction: opt.transaction })
+									.then(savedD => {
+										// Add diff to revision
+										objectRevision[
+											`add${helpers.capitalizeFirstLetter(
+												options.revisionChangeModel,
+											)}`
+										](savedD);
+
+										return null;
+									})
+									.catch(err => {
+										log('RevisionChange save error', err);
+										throw err;
+									});
+							});
+						}
+						return null;
+					})
+					.catch(err => {
+						log('Revision save error', err);
+						throw err;
+					});
+			}
+
+			if (options.debug) {
+				log('end of afterHook');
+			}
+
+			return null;
+		};
+		return afterHook;
+	}
+
+	// order in which sequelize processes the hooks
+	// (1)
+	// beforeBulkCreate(instances, options, fn)
+	// beforeBulkDestroy(instances, options, fn)
+	// beforeBulkUpdate(instances, options, fn)
+	// (2)
+	// beforeValidate(instance, options, fn)
+	// (-)
+	// validate
+	// (3)
+	// afterValidate(instance, options, fn)
+	// - or -
+	// validationFailed(instance, options, error, fn)
+	// (4)
+	// beforeCreate(instance, options, fn)
+	// beforeDestroy(instance, options, fn)
+	// beforeUpdate(instance, options, fn)
+	// (-)
+	// create
+	// destroy
+	// update
+	// (5)
+	// afterCreate(instance, options, fn)
+	// afterDestroy(instance, options, fn)
+	// afterUpdate(instance, options, fn)
+	// (6)
+	// afterBulkCreate(instances, options, fn)
+	// afterBulkDestroy(instances, options, fn)
+	// afterBulkUpdate(instances, options, fn)
+
+	// Extend model prototype with "hasPaperTrail" function
+	// Call model.hasPaperTrail() to enable revisions for model
+	_.assignIn(Sequelize.Model, {
+		hasPaperTrail: function hasPaperTrail() {
+			if (options.debug) {
+				log('Enabling paper trail on', this.name);
+			}
+
+			this.rawAttributes[options.revisionAttribute] = {
+				type: Sequelize.INTEGER,
+				defaultValue: 0,
+			};
+			this.revisionable = true;
+
+			// not sure if we need this
+			this.refreshAttributes();
+
+			if (options.enableMigration) {
+				const tableName = this.getTableName();
+
+				const queryInterface = sequelize.getQueryInterface();
+
+				queryInterface.describeTable(tableName).then(attributes => {
+					if (!attributes[options.revisionAttribute]) {
+						if (options.debug) {
+							log('adding revision attribute to the database');
+						}
+
+						queryInterface
+							.addColumn(tableName, options.revisionAttribute, {
+								type: Sequelize.INTEGER,
+								defaultValue: 0,
+							})
+							.then(() => null)
+							.catch(err => {
+								log('something went really wrong..', err);
+								return null;
+							});
+					}
+					return null;
+				});
+			}
+
+			this.addHook('beforeCreate', createBeforeHook('create'));
+			this.addHook('beforeDestroy', createBeforeHook('destroy'));
+			this.addHook('beforeUpdate', createBeforeHook('update'));
+			this.addHook('afterCreate', createAfterHook('create'));
+			this.addHook('afterDestroy', createAfterHook('destroy'));
+			this.addHook('afterUpdate', createAfterHook('update'));
+
+			// create association
+			return this.hasMany(sequelize.models[options.revisionModel], {
+				foreignKey: options.defaultAttributes.documentId,
+				constraints: false,
+				scope: {
+					model: this.name,
+				},
+			});
+		},
+	});
+
+	return {
+		// TWS-1032 since ALS must be set after a .run()
+		//   invocation this CANNOT be set on init().  If you would
+		//   like to use ALS, set it after the first invocation of run()
+		setupALS: function(asyncLocalStorage) {
+			options.als = asyncLocalStorage;
+		},
+		alsEnabled: function() {
+			return (options.als === true)
+		},
+		defineModels: function defineModels(db) {
+			// Attributes for RevisionModel
+			let attributes = {
+				model: {
+					type: Sequelize.TEXT,
+					allowNull: false,
+				},
+				document: {
+					type: Sequelize.JSONB,
+					allowNull: false,
+				},
+				operation: Sequelize.STRING(7),
+			};
+
+			if (options.mysql) {
+				attributes.document.type = Sequelize.TEXT('MEDIUMTEXT');
+			}
+
+			attributes[options.defaultAttributes.documentId] = {
+				type: Sequelize.INTEGER,
+				allowNull: false,
+			};
+
+			attributes[options.revisionAttribute] = {
+				type: Sequelize.INTEGER,
+				allowNull: false,
+			};
+
+			attributes[options.userModelAttribute] = {
+				type: Sequelize.INTEGER,
+				allowNull: true
+			};
+
+			if (options.UUID) {
+				attributes.id = {
+					primaryKey: true,
+					type: Sequelize.UUID,
+					defaultValue: Sequelize.UUIDV4,
+				};
+				attributes[options.defaultAttributes.documentId].type =
+					Sequelize.UUID;
+
+					console.log("userModelatribute", options.userModelAttribute)
+				attributes[options.userModelAttribute].type = Sequelize.UUID;
+			}
+
+			if (options.debug) {
+				log('attributes', attributes);
+			}
+
+			// Revision model
+			const Revision = sequelize.define(
+				options.revisionModel,
+				attributes,
+				{
+					underscored: options.underscored,
+					tableName: options.tableName,
+				},
+			);
+			Revision.associate = function associate(models) {
+				if (options.debug) {
+					log('models', models);
+				}
+
+				Revision.belongsTo(
+					sequelize.model(options.userModel),
+					options.belongsToUserOptions,
+				);
+			};
+
+			if (options.enableRevisionChangeModel) {
+				// Attributes for RevisionChangeModel
+				attributes = {
+					path: {
+						type: Sequelize.TEXT,
+						allowNull: false,
+					},
+					document: {
+						type: Sequelize.JSONB,
+						allowNull: false,
+					},
+					diff: {
+						type: Sequelize.JSONB,
+						allowNull: false,
+					},
+				};
+
+				if (options.mysql) {
+					attributes.document.type = Sequelize.TEXT('MEDIUMTEXT');
+					attributes.diff.type = Sequelize.TEXT('MEDIUMTEXT');
+				}
+
+				if (options.UUID) {
+					attributes.id = {
+						primaryKey: true,
+						type: Sequelize.UUID,
+						defaultValue: Sequelize.UUIDV4,
+					};
+				}
+				// RevisionChange model
+				const RevisionChange = sequelize.define(
+					options.revisionChangeModel,
+					attributes,
+					{
+						underscored: options.underscored,
+					},
+				);
+
+				// Set associations
+				Revision.hasMany(RevisionChange, {
+					foreignKey: options.defaultAttributes.revisionId,
+					constraints: false,
+				});
+				paginator.paginate(Revision);
+
+				RevisionChange.belongsTo(Revision);
+
+				if (db)
+					db[RevisionChange.name] = RevisionChange;
+			}
+
+			if (db)
+				db[Revision.name] = Revision;
+
+			/*
+			 * We could extract this to a separate function so that having a
+			 * user model doesn't require different loading
+			 *
+			 * or perhaps we could omit this because we are creating the
+			 * association through the associate call above.
+			 */
+			if (options.userModel) {
+				Revision.belongsTo(
+					sequelize.model(options.userModel),
+					options.belongsToUserOptions,
+				);
+			}
+
+			return Revision;
+		},
+	};
+};
+
+/**
+ * Throw exceptions when the user identifier from ALS is not set or if the
+ * revisionAttribute was not loaded on the model.
+ */
+paperTrail.enableFailHard = () => {
+	failHard = true;
+};
+
+export default paperTrail;