@webda/core 2.1.5 → 2.3.0

package/lib/utils/throttler.js

@@ -0,0 +1,136 @@
+/**
+ * Represent an item within the Throttler
+ */
+class ThrottlerItem {
+    constructor(method, callbacks, name) {
+        this.method = method;
+        this.callbacks = callbacks;
+        this.name = name;
+    }
+    toString() {
+        return `${this.name}: ${this.promise ? "IN-PROGRESS" : "QUEUED"}`;
+    }
+}
+/**
+ * Promise Throttler
+ *
+ * Allow you to queue promise and execute them concurrently
+ *
+ * Several libraries does that
+ *
+ * The queue method still gives you a simple Promise linked to the resolution
+ * of your queued item, so you can still wait for the execution of the specific
+ * item
+ */
+export class Throttler {
+    /**
+     *
+     * @param concurrency max concurrent promise to execute
+     */
+    constructor(concurrency = 10) {
+        this.concurrency = concurrency;
+        /**
+         * Current queue
+         */
+        this._queue = [];
+        /**
+         * Number of promises in-progress
+         */
+        this.current = 0;
+        /**
+         * Resolver for each call to waitForCompletion
+         */
+        this._waiters = [];
+    }
+    /**
+     * Queue a new promise
+     *
+     * @param method executor that return the promise to queue
+     * @param name of the task, usefull when calling getInProgress
+     * @returns
+     */
+    queue(method, name = `Promise_${this.getSize()}`) {
+        return new Promise((...callbacks) => {
+            this._queue.push(new ThrottlerItem(method, callbacks, name));
+            this.add();
+        });
+    }
+    /**
+     * Set the concurrency
+     * @param concurrency newValue
+     *
+     * If decreased, it will be in effect only when current promises resolve
+     * If increased, it will have immediate effect
+     */
+    setConcurrency(concurrency) {
+        let oldValue = this.concurrency;
+        this.concurrency = concurrency;
+        // Ensure we launch new promise if concurrency has increased
+        while (oldValue < this.concurrency) {
+            this.add();
+            oldValue++;
+        }
+    }
+    /**
+     * Get inprogress items
+     * @returns
+     */
+    getInProgress() {
+        return this._queue.filter(p => p.promise);
+    }
+    /**
+     * Get global queue size
+     * @returns
+     */
+    getSize() {
+        return this._queue.length;
+    }
+    /**
+     * Wait until every promise resolve
+     * @returns
+     */
+    async waitForCompletion() {
+        if (this.current === 0) {
+            return;
+        }
+        return new Promise(resolve => {
+            this._waiters.push(resolve);
+        });
+    }
+    /**
+     * Internal manage the promise concurrency
+     * @returns
+     */
+    add() {
+        if (this.current >= this.concurrency) {
+            return;
+        }
+        const next = this._queue.find(p => !p.promise);
+        if (!next) {
+            if (this.current === 0) {
+                // Call all waiters
+                this._waiters.forEach(p => p());
+                this._waiters = [];
+            }
+            return;
+        }
+        this.current++;
+        next.promise = next
+            .method()
+            .then(res => {
+            // Call the resolve
+            next.callbacks[0](res);
+        })
+            .catch(err => {
+            // Call the reject
+            next.callbacks[1](err);
+        })
+            .finally(() => {
+            this.current--;
+            // Remove from queue
+            this._queue.splice(this._queue.indexOf(next), 1);
+            this.add();
+        });
+    }
+}
+//# sourceMappingURL=throttler.js.map

package/lib/utils/throttler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"throttler.js","sourceRoot":"","sources":["../../src/utils/throttler.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,aAAa;IAKjB,YAAmB,MAA0B,EAAS,SAAkC,EAAS,IAAY;QAA1F,WAAM,GAAN,MAAM,CAAoB;QAAS,cAAS,GAAT,SAAS,CAAyB;QAAS,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IACjH,QAAQ;QACN,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACpE,CAAC;CACF;AAED;;;;;;;;;;GAUG;AACH,MAAM,OAAO,SAAS;IAcpB;;;OAGG;IACH,YAAsB,cAAsB,EAAE;QAAxB,gBAAW,GAAX,WAAW,CAAa;QAjB9C;;WAEG;QACO,WAAM,GAAoB,EAAE,CAAC;QACvC;;WAEG;QACH,YAAO,GAAW,CAAC,CAAC;QACpB;;WAEG;QACO,aAAQ,GAAmB,EAAE,CAAC;IAMS,CAAC;IAElD;;;;;;OAMG;IACH,KAAK,CAAC,MAA0B,EAAE,OAAe,WAAW,IAAI,CAAC,OAAO,EAAE,EAAE;QAC1E,OAAO,IAAI,OAAO,CAAM,CAAC,GAAG,SAAS,EAAE,EAAE;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACH,cAAc,CAAC,WAAmB;QAChC,IAAI,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,4DAA4D;QAC5D,OAAO,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE;YAClC,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,QAAQ,EAAE,CAAC;SACZ;IACH,CAAC;IAED;;;OAGG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACH,OAAO;QACL,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE;YACtB,OAAO;SACR;QACD,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;YAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACO,GAAG;QACX,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE;YACpC,OAAO;SACR;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE;YACT,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE;gBACtB,mBAAmB;gBACnB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAChC,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;aACpB;YACD,OAAO;SACR;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,OAAO,GAAG,IAAI;aAChB,MAAM,EAAE;aACR,IAAI,CAAC,GAAG,CAAC,EAAE;YACV,mBAAmB;YACnB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,CAAC,EAAE;YACX,kBAAkB;YAClB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,oBAAoB;YACpB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACjD,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,CAAC,CAAC,CAAC;IACP,CAAC;CACF"}

package/lib/utils/throttler.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/utils/throttler.spec.js

@@ -0,0 +1,47 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { suite, test } from "@testdeck/mocha";
+import * as assert from "assert";
+import { Throttler } from "./throttler.js";
+let ThrottlerTest = class ThrottlerTest {
+    async test() {
+        let resolvers = [];
+        let t = new Throttler(2);
+        t.queue(() => new Promise(resolve => resolvers.push(resolve)), "P1");
+        t.queue(() => new Promise(resolve => resolvers.push(resolve)), "P2");
+        t.queue(() => new Promise(resolve => resolvers.push(resolve)), "P3");
+        t.queue(() => new Promise(resolve => resolvers.push(resolve)), "P4");
+        t.queue(() => new Promise(resolve => resolvers.push(resolve)), "P5");
+        t.queue(() => new Promise((_, reject) => resolvers.push(reject)), "P6");
+        t.queue(() => new Promise(resolve => resolvers.push(resolve)), "P7");
+        assert.strictEqual(t.getSize(), 7);
+        // @ts-ignore
+        resolvers.shift()();
+        assert.strictEqual(t.getInProgress().length, 2);
+        t.setConcurrency(1);
+        // @ts-ignore
+        resolvers.shift()();
+        await new Promise(resolve => setImmediate(resolve));
+        let curs = t.getInProgress();
+        assert.strictEqual(curs.length, 1, `Currents ${curs}`);
+        t.setConcurrency(3);
+        assert.strictEqual(t.getInProgress().length, 3);
+        let p = t.waitForCompletion();
+        resolvers.forEach(r => r());
+        await new Promise(resolve => setImmediate(resolve));
+        resolvers.forEach(r => r());
+        await p;
+        await t.waitForCompletion();
+    }
+};
+__decorate([
+    test
+], ThrottlerTest.prototype, "test", null);
+ThrottlerTest = __decorate([
+    suite
+], ThrottlerTest);
+//# sourceMappingURL=throttler.spec.js.map

package/lib/utils/throttler.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"throttler.spec.js","sourceRoot":"","sources":["../../src/utils/throttler.spec.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,IAAM,aAAa,GAAnB,MAAM,aAAa;IAEX,AAAN,KAAK,CAAC,IAAI;QACR,IAAI,SAAS,GAAmB,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC3E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC3E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC3E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC3E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC3E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC9E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC3E,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QACnC,aAAa;QACb,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;QACpB,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QACpB,aAAa;QACb,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;QACpB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QACpD,IAAI,IAAI,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC;QAC7B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC;QACvD,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,CAAC;QAC9B,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QACpD,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,CAAC,CAAC;QACR,MAAM,CAAC,CAAC,iBAAiB,EAAE,CAAC;IAC9B,CAAC;CACF,CAAA;AA7BO;IADL,IAAI;yCA6BJ;AA9BG,aAAa;IADlB,KAAK;GACA,aAAa,CA+BlB"}

package/lib/utils/waiter.d.ts

@@ -2,7 +2,7 @@ import { Logger } from "../index.js";
 /**
  * Function that define the amount of time between calls
  */
-export declare type WaitDelayer = (retries: number) => number;
+export type WaitDelayer = (retries: number) => number;
 /**
  * Return a delayer that always wait for the same amount of time
  * @param pause amount of time to wait
@@ -18,7 +18,7 @@ export declare function WaitExponentialDelay(pause: number): WaitDelayer;
 /**
  * Delayer factory definition
  */
-export declare type WaitDelayerFactory = (interval: number) => WaitDelayer;
+export type WaitDelayerFactory = (interval: number) => WaitDelayer;
 /**
  * WaitDelayer definition
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webda/core",
-  "version": "2.1.5",
+  "version": "2.3.0",
   "description": "Expose API with Lambda",
   "keywords": [
     "aws",
@@ -14,6 +14,7 @@
   "scripts": {
     "build": "tsc-esm",
     "build:module": "webda build",
+    "build:watch": "webda build --watch",
     "grammar": "antlr4ts -visitor src/stores/webdaql/WebdaQLLexer.g4 src/stores/webdaql/WebdaQLParser.g4 && yarn run lint:fix",
     "pretest": "npm run build",
     "lint": "prettier --check src/**/*",
@@ -22,7 +23,7 @@
   },
   "dependencies": {
     "@types/json-schema": "^7.0.8",
-    "@webda/workout": "^2.1.2",
+    "@webda/workout": "^2.2.0",
     "accept-language": "^3.0.18",
     "ajv": "^8.6.2",
     "ajv-formats": "^2.1.1",
@@ -37,7 +38,7 @@
     "global": "^4.4.0",
     "jsonc-parser": "^3.0.0",
     "jsonpath": "^1.1.1",
-    "jsonwebtoken": "^8.5.1",
+    "jsonwebtoken": "^9.0.0",
     "merge": "^2.1.1",
     "mime-types": "^2.1.31",
     "mustache": "^4.2.0",
@@ -55,20 +56,20 @@
     "yaml": "^2.0.0"
   },
   "devDependencies": {
-    "@testdeck/mocha": "^0.2.0",
+    "@testdeck/mocha": "^0.3.2",
     "@types/node": "14.6.0",
     "@types/nodemailer": "^6.4.4",
-    "@webda/tsc-esm": "^1.0.1",
+    "@webda/tsc-esm": "^1.0.2",
     "antlr4ts-cli": "^0.5.0-alpha.4",
     "axios": "^1.1.3",
     "c8": "^7.11.3",
-    "fs-extra": "^10.0.0",
+    "fs-extra": "^11.0.0",
     "mocha": "^10.0.0",
     "prettier": "^2.3.2",
-    "sinon": "^14.0.0",
+    "sinon": "^15.0.0",
     "source-map-support": "^0.5.19",
     "ts-node": "^10.1.0",
-    "typescript": "~4.8.2"
+    "typescript": "~4.9.3"
   },
   "files": [
     "lib",
@@ -102,5 +103,5 @@
   "engines": {
     "node": ">=14.6.0"
   },
-  "gitHead": "f8603d252baf83769b4906eff55c4d38e0934a1d"
+  "gitHead": "fcdb5181ae9353dd8618017fae07ca75203bb1ad"
 }

package/webda.module.json

@@ -1617,6 +1617,13 @@
           "description": "Webda model to use within the Store",
           "default": "Webda/CoreModel"
         },
+        "models": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Different models managed by this store"
+        },
         "asyncDelete": {
           "type": "boolean",
           "description": "async delete"
@@ -1627,14 +1634,44 @@
         },
         "strict": {
           "type": "boolean",
-          "description": "Allow to load object that does not have the type data",
+          "description": "Allow to load object that does not have the type data\n\nIf set to true, then the Store will only managed the defined _model and no model extending this one",
+          "default": false
+        },
+        "defaultModel": {
+          "type": "boolean",
+          "description": "When __type model not found, use the model If strict is setup this parameter is not used",
           "default": true
         },
+        "forceModel": {
+          "type": "boolean",
+          "description": "If set, Store will ignore the __type",
+          "default": false
+        },
         "slowQueryThreshold": {
           "type": "number",
           "description": "Slow query threshold",
           "default": 30000
         },
+        "parent": {
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string"
+            },
+            "condition": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "name",
+            "condition"
+          ],
+          "description": "Parent of the store\n\nIf parent is specified the url route is relative to it"
+        },
+        "graphql": {
+          "type": "boolean",
+          "description": "For future use in our GraphQL api\n\nExpose this store in the graphql"
+        },
         "persistence": {
           "type": "object",
           "properties": {
@@ -1739,6 +1776,13 @@
           "description": "Webda model to use within the Store",
           "default": "Webda/CoreModel"
         },
+        "models": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Different models managed by this store"
+        },
         "asyncDelete": {
           "type": "boolean",
           "description": "async delete"
@@ -1749,14 +1793,44 @@
         },
         "strict": {
           "type": "boolean",
-          "description": "Allow to load object that does not have the type data",
+          "description": "Allow to load object that does not have the type data\n\nIf set to true, then the Store will only managed the defined _model and no model extending this one",
+          "default": false
+        },
+        "defaultModel": {
+          "type": "boolean",
+          "description": "When __type model not found, use the model If strict is setup this parameter is not used",
           "default": true
         },
+        "forceModel": {
+          "type": "boolean",
+          "description": "If set, Store will ignore the __type",
+          "default": false
+        },
         "slowQueryThreshold": {
           "type": "number",
           "description": "Slow query threshold",
           "default": 30000
         },
+        "parent": {
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string"
+            },
+            "condition": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "name",
+            "condition"
+          ],
+          "description": "Parent of the store\n\nIf parent is specified the url route is relative to it"
+        },
+        "graphql": {
+          "type": "boolean",
+          "description": "For future use in our GraphQL api\n\nExpose this store in the graphql"
+        },
         "folder": {
           "type": "string",
           "description": "Local path where to store all `json` files"