@universis/webhooks 1.1.1

package/jest.config.js

@@ -0,0 +1,202 @@
+/**
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+/** @type {import('jest').Config} */
+const config = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/private/var/folders/bs/w3b1f9k16kb7mm17d66vdmwc0000gn/T/jest_dx",
+
+  // Automatically clear mock calls, instances, contexts and results before every test
+  // clearMocks: false,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  // collectCoverage: false,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  // coverageDirectory: undefined,
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: "v8",
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // The default configuration for fake timers
+  // fakeTimers: {
+  //   "enableGlobally": false
+  // },
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "mjs",
+  //   "cjs",
+  //   "jsx",
+  //   "ts",
+  //   "mts",
+  //   "cts",
+  //   "tsx",
+  //   "json",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  moduleNameMapper: {
+      '@universis/webhooks': '<rootDir>/src',
+  },
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  // preset: undefined,
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state before every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state and implementation before every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  // setupFilesAfterEnv: [],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  // testEnvironment: "jest-environment-node",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  // testMatch: [
+  //   "**/__tests__/**/*.?([mc])[jt]s?(x)",
+  //   "**/?(*.)+(spec|test).?([mc])[jt]s?(x)"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // A map from regular expressions to paths to transformers
+  // transform: undefined,
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "/node_modules/",
+  //   "\\.pnp\\.[^\\/]+$"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};
+
+module.exports = config;

package/jsconfig.json

@@ -0,0 +1,8 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "@universis/webhooks": ["./src"]
+    }
+  }
+}

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@universis/webhooks",
+  "version": "1.1.1",
+  "description": "Universis Webhooks API Library",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "engines": {
+    "node": ">=14"
+  },
+  "scripts": {
+    "test": "jest",
+    "eslint": "eslint",
+    "build": "rimraf dist && babel src --out-dir dist --copy-files --source-maps"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/universis/webhooks.git"
+  },
+  "keywords": [
+    "universis",
+    "api",
+    "extension",
+    "plugin",
+    "library",
+    "microservice"
+  ],
+  "author": "",
+  "license": "LGPL-3.0-or-later",
+  "bugs": {
+    "url": "https://gitlab.com/universis/webhooks/issues"
+  },
+  "peerDependencies": {
+    "@themost/common": "^2",
+    "@themost/data": "^2",
+    "@themost/express": "^1",
+    "@themost/query": "^2",
+    "@themost/xml": "^2"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.21.0",
+    "@babel/core": "^7.21.4",
+    "@babel/eslint-parser": "^7.21.3",
+    "@babel/plugin-proposal-decorators": "^7.21.0",
+    "@babel/preset-env": "^7.21.4",
+    "@babel/register": "^7.21.0",
+    "@rollup/plugin-babel": "^6.0.3",
+    "@themost/data": "^2.21.1",
+    "@themost/test": "^2.11.4",
+    "@types/express": "^4.17.6",
+    "@types/jasmine": "^3.5.10",
+    "@types/superagent": "^8.1.9",
+    "dotenv": "^8.2.0",
+    "eslint": "^8.57.1",
+    "jest": "^30.1.3",
+    "rimraf": "3.0.2",
+    "typescript": "^5.0.4"
+  },
+  "homepage": "https://gitlab.com/universis/webhooks#readme",
+  "dependencies": {
+    "@themost/json-logger": "^1.3.0",
+    "rotating-file-stream": "^3.2.7",
+    "rxjs": "^7.8.2",
+    "superagent": "^10.2.3"
+  }
+}

package/src/SchemaLoader.d.ts

@@ -0,0 +1,4 @@
+import { FileSchemaLoaderStrategy } from '@themost/data';
+export declare class SchemaLoader extends FileSchemaLoaderStrategy {
+
+}

package/src/SchemaLoader.js

@@ -0,0 +1,46 @@
+import {FileSchemaLoaderStrategy} from '@themost/data';
+import path from 'path';
+import { ValueDialect } from '@themost/data';
+
+if (typeof ValueDialect.prototype.$literal === 'undefined') {
+    /**
+     * Add $literal support to ValueDialect
+     */
+    ValueDialect.prototype.$literal = async function (value) {
+        return value;
+    }
+}
+
+/**
+ * A schema loader for loading extra data models
+ */
+export class SchemaLoader extends FileSchemaLoaderStrategy {
+
+    /**
+     * @param {import('@themost/data').ConfigurationBase} config
+     */
+    constructor(config) {
+        super(config);
+        // set model path
+        this.setModelPath(path.resolve(__dirname, 'config/models'));
+    }
+
+    getModelDefinition(name) {
+        const model = super.getModelDefinition.bind(this)(name);
+        if (model) {
+            if (Array.isArray(model.eventListeners) ) {
+                model.eventListeners.forEach(eventListener => {
+                    if (eventListener.type.indexOf('.') === 0) {
+                        eventListener.type = path.resolve(__dirname,  eventListener.type);
+                    }
+                });
+            }
+            if (model.classPath && model.classPath.indexOf('.')===0) {
+                model.classPath= path.resolve(__dirname, model.classPath);
+            }
+        }
+        return model;
+    }
+
+}
+

package/src/WebHookService.d.ts

@@ -0,0 +1,11 @@
+import { ApplicationService } from '@themost/common';
+import {DataEventArgs} from '@themost/data';
+import {JsonLogger} from '@themost/json-logger';
+
+export declare class WebHookService extends ApplicationService {
+    readonly logger: JsonLogger;
+    refresh(): Promise<void>;
+    emit(event: DataEventArgs, extraArgs?: {
+        action?: string
+    }): void;
+}

package/src/WebHookService.js

@@ -0,0 +1,275 @@
+// noinspection NpmUsedModulesInstalled
+
+import {ApplicationService} from '@themost/common';
+import {WebHookScopeAccessElements} from './config/scope.access';
+import {
+    DataObjectState,
+    DataQueryable,
+    ODataModelBuilder,
+    SchemaLoaderStrategy
+} from '@themost/data';
+import {SchemaLoader} from './SchemaLoader';
+import request from 'superagent';
+import {QueryExpression} from '@themost/query';
+import { JsonLogger } from '@themost/json-logger';
+import path from 'path';
+import { createStream } from 'rotating-file-stream';
+import {DataModelFilterParser} from '@themost/data/data-model-filter.parser';
+import { afterSave } from './listeners/OnWebHookAction';
+
+class WebHookService extends ApplicationService {
+    constructor(app) {
+        super(app);
+        /**
+         * @type {import('@themost/data').DefaultSchemaLoaderStrategy}
+         */
+        const schemaLoader = app.getConfiguration().getStrategy(SchemaLoaderStrategy);
+        schemaLoader.loaders.push(new SchemaLoader(app.getConfiguration()));
+        /**
+         * @type {import('@themost/data').ODataModelBuilder}
+         */
+        const builder = app.getService(ODataModelBuilder);
+        if (builder) {
+            builder.clean(true);
+        }
+        // extend universis api scope access configuration
+        if (app && app.container) {
+            app.container.subscribe((container) => {
+                if (container) {
+                    // add extra scope access elements
+                    const scopeAccess = app.getConfiguration().getStrategy(function ScopeAccessConfiguration() {
+                    });
+                    if (scopeAccess != null) {
+                        scopeAccess.elements.unshift(...WebHookScopeAccessElements);
+                    }
+                }
+            });
+        }
+
+        // configure logger
+        // get options from configuration
+        const options = Object.assign({
+            logLevel: 'info',
+            logFormat: 'raw',
+            logDir: path.join(process.cwd(), 'log')
+        }, app.getConfiguration().getSourceAt('settings/universis/webhooks'));
+        const logDir = options.logDir ? path.resolve(options.logDir) : path.join(process.cwd(), 'log');
+        const logFile = 'webhooks.log';
+        // create a rotating write stream
+        // noinspection JSValidateTypes
+        const logStream = createStream(logFile, {
+            interval: '1d', // rotate daily
+            path: logDir
+        });
+        this.logger = new JsonLogger({
+            level: options.logLevel,
+            format: options.logFormat,
+            stdout: logStream,
+            stderr: logStream
+        });
+        this.logger.info(`Universis WebHookService initialized with log level "${options.logLevel}"`);
+
+        if (app) {
+            const self = this;
+            setTimeout(() => {
+                (async () => {
+                    await self.refresh();
+                })();
+            }, 1000);
+        }
+
+    }
+
+    /**
+     * Refresh web hook actions
+     * @returns {Promise<void>}
+     */
+    async refresh() {
+        /**
+         * @type {import('@themost/data').DataContext}
+         */
+        let context;
+        try {
+            // noinspection JSUnresolvedReference
+            context = this.getApplication().createContext();
+            this.logger.info('Registering existing web hook actions');
+            const items = await context.model('WebHookAction').asQueryable().take(-1).silent().getAllItems();
+            if (items.length > 0) {
+                this.logger.info(`Found ${items.length} web hook action(s). The operation will automatically register event listeners to the underlying data models.`);
+            }
+            for (const item of items) {
+                const model = context.model(item.about);
+                if (model) {
+                    await new Promise((resolve, reject) => {
+                        afterSave({ state: DataObjectState.Insert, target: item, model}, (err) => {
+                            if (err) {
+                                return reject(err);
+                            }
+                            return resolve();
+                        });
+                    });
+                } else {
+                    this.logger.warn(`Cannot register web hook action with name "${item.name || 'unknown'}" because model "${item.about}" does not exist.`);
+                }
+            }
+        } catch(err) {
+            this.logger.error('An error occurred when registering existing web hooks');
+            this.logger.error(err);
+        } finally {
+            if (context) {
+                await context.finalizeAsync()
+            }
+        }
+    }
+
+    /**
+     *
+     * @param {import('@themost/data').DataModel} model
+     * @param {*} target
+     * @param {string} filter
+     * @param {number} state
+     * @returns {Promise<boolean>}
+     */
+    async filterObjectAsync(model, target, filter, state) {
+        const parser = new DataModelFilterParser(model);
+        const { $expand, $where } = await parser.parseAsync(filter)
+        const q = new DataQueryable(model);
+        Object.assign(q.query, {
+            $where,
+            $expand
+        });
+        const { primaryKey } = model;
+        const id = target[primaryKey];
+        q.prepare();
+        if (state !== DataObjectState.Delete) {
+            q.where(primaryKey).equal(id);
+        }
+        const exists = await q.count();
+        return (exists > 0);
+    }
+
+    /**
+     * Emit web hook events
+     * @param {import('@themost/data').DataEventArgs} event
+     * @param {{action?: string, emit?: boolean}} extraArgs
+     */
+    emit(event, extraArgs) {
+        const context = event.model.context;
+        // get previous and current state of the entity
+        const { previous, target, state, model } = event;
+        /**
+         * @type {import('@themost/express').ExpressDataApplication|import('@themost/common').ApplicationBase}
+         */
+        const application = context.application;
+        // noinspection JSUnresolvedReference
+        /**
+         * @type {import('@themost/express').ExpressDataContext|import('@themost/data').DataContext}
+         */
+        const newContext = application.createContext();
+        const about = model.name;
+        const self = this;
+        (async function() {
+            // noinspection JSUnresolvedReference
+            const q = newContext.model('WebHook').asQueryable().where((x, about, state) => {
+                // noinspection JSBitwiseOperatorUsage
+                return (x.action.about === about) && (x.action.state === state) && (x.active === true);
+            }, about, state).select((x) => {
+                return {
+                    id: x.id,
+                    config: x.config,
+                    action: x.action,
+                    about: x.action.about,
+                    emit: x.emit,
+                    username: x.owner.name,
+                    filter: x.action.filter,
+                    when: x.action.when
+                }
+            }).silent().take(25).skip(0);
+            // apply extra arguments for finding web hooks
+            const findArgs = Object.assign({ emit: true }, extraArgs || {});
+            q.prepare();
+            Object.keys(findArgs).forEach((key) => {
+                const value = findArgs[key];
+                q.query.$where ? q.and(key).equal(value) : q.where(key).equal(value);
+            });
+
+            const { viewAdapter: WebHooks } = newContext.model('WebHook');
+            const targetModel = newContext.model(about);
+            // get entity set
+            const builder = application.getService(ODataModelBuilder);
+            const entitySet = builder.getEntityTypeEntitySet(targetModel.name)
+            const { primaryKey: key } = targetModel;
+            // get origin
+            const origin = context.getConfiguration().getSourceAt('settings/universis/api/origin');
+            /**
+             * @type {{total: number, skip: number, value: Array<{ id: string,config: { url: string, insecure: boolean, secret?: string, contentType: string }, emit: boolean, username: string,action: string,about: string,filter: string,when: string}>}} list
+             */
+            let list = await q.getList();
+            if (list.total > 0) {
+                while (list.total >= (list.skip + list.value.length)) {
+                    for (const item of list.value) {
+                        // set context user
+                        newContext.user  = {
+                            name: item.username || 'anonymous'
+                        }
+                        try {
+                            let send = true;
+                            if (state === DataObjectState.Insert) {
+                                // validate the current state of a new object
+                                const filter = item.when || item.filter;
+                                send = await self.filterObjectAsync(targetModel, target, filter, state);
+                            } else {
+                                // validate previous state
+                                send = await self.filterObjectAsync(targetModel, previous, item.when, state);
+                                // validate current state
+                                if (send && item.filter) {
+                                    if (state === DataObjectState.Delete) {
+                                        send = await self.filterObjectAsync(targetModel, previous, item.filter, state);
+                                    } else {
+                                        send = await self.filterObjectAsync(targetModel, target, item.filter, state);
+                                    }
+                                }
+                            }
+                            if (send) {
+                                self.logger.info(`Sending web hook for model "${about}" to url "${item.config.url}"`);
+                                // send web hook request
+                                const result = await request('post', item.config.url)
+                                    .set('Content-Type', item.config.contentType)
+                                    .send({
+                                        event: item.action,
+                                        entityType: targetModel.name,
+                                        entitySet: entitySet && entitySet.name,
+                                        origin: origin,
+                                        target: {
+                                            id: (state === DataObjectState.Insert) ? target[key] : previous[key],
+                                        }
+                                    });
+                                const lastResponse = {
+                                    status: result.status
+                                };
+                                self.logger.info(`Web hook for model "${about}" to url ${item.config.url} returned status ${result.status}`);
+                                // update last response with an ad-hoc query
+                                await newContext.db.executeAsync(
+                                    new QueryExpression().update(WebHooks).set({
+                                        lastResponse: JSON.stringify(lastResponse),
+                                    }).where('id').equal(item.id)
+                                , []);
+                            } else {
+                                self.logger.debug(`Web hook for model "${about}" and object with identifier ${target[key]} to url "${item.config.url}" was filtered and not sent`);
+                            }
+                        } catch (err) {
+                            self.logger.error(`Error sending web hook for model "${about}" to url "${item.config.url}"`);
+                            self.logger.error(err);
+                        }
+                    }
+                    list = await q.take(25).skip(list.skip + list.value.length);
+                }
+            }
+        })();
+    }
+
+}
+
+export {
+    WebHookService
+}