kuzzle 2.19.2 → 2.19.5

package/lib/core/shared/store.js

@@ -19,11 +19,12 @@
  * limitations under the License.
  */
 
-'use strict';
+"use strict";
 
-const Bluebird = require('bluebird');
-
-const { Mutex } = require('../../util/mutex');
+const { Mutex } = require("../../util/mutex");
+const { promiseAllN } = require("../../util/async");
+const kerror = require("../../kerror");
+const { getESIndexDynamicSettings } = require("../../util/esRequest");
 
 /**
  * Wrapper around the document store.
@@ -36,7 +37,7 @@ class Store {
    * @param {String} index
    * @param {storeScopeEnum} scope
    */
-  constructor (index, scope) {
+  constructor(index, scope) {
     this.index = index;
     this.scope = scope;
 
@@ -53,6 +54,7 @@ class Store {
       exists: `core:storage:${scope}:document:exist`,
       get: `core:storage:${scope}:document:get`,
       getMapping: `core:storage:${scope}:mappings:get`,
+      getSettings: `core:storage:${scope}:collection:settings:get`,
       mExecute: `core:storage:${scope}:document:mExecute`,
       mGet: `core:storage:${scope}:document:mGet`,
       refreshCollection: `core:storage:${scope}:collection:refresh`,
@@ -71,17 +73,20 @@ class Store {
 
     // the scroll and multiSearch method are special: they doesn't need an index parameter
     // we keep them for ease of use
-    this.scroll = (scrollId, opts) => global.kuzzle.ask(
-      `core:storage:${scope}:document:scroll`,
-      scrollId,
-      opts);
-
-    this.multiSearch = (targets, searchBody, opts) => global.kuzzle.ask(
-      `core:storage:${scope}:document:multiSearch`,
-      targets,
-      searchBody,
-      opts
-    );
+    this.scroll = (scrollId, opts) =>
+      global.kuzzle.ask(
+        `core:storage:${scope}:document:scroll`,
+        scrollId,
+        opts
+      );
+
+    this.multiSearch = (targets, searchBody, opts) =>
+      global.kuzzle.ask(
+        `core:storage:${scope}:document:multiSearch`,
+        targets,
+        searchBody,
+        opts
+      );
   }
 
   /**
@@ -91,19 +96,26 @@ class Store {
    *
    * @returns {Promise}
    */
-  async init (collections = {}) {
-    const mutex = new Mutex(`Store.init(${this.index})`, {
-      timeout: -1,
+  async init(collections = {}) {
+    const creatingMutex = new Mutex(`Store.init(${this.index})`, {
+      timeout: 0,
       ttl: 30000,
     });
 
-    await mutex.lock();
+    if (await creatingMutex.lock()) {
+      try {
+        await this.createCollections(collections, {
+          indexCacheOnly: false,
+        });
+      } finally {
+        await creatingMutex.unlock();
+      }
+    } else {
+      await creatingMutex.wait({ timeout: -1 });
 
-    try {
-      await this.createCollections(collections);
-    }
-    finally {
-      await mutex.unlock();
+      await this.createCollections(collections, {
+        indexCacheOnly: true,
+      });
     }
   }
 
@@ -114,17 +126,100 @@ class Store {
    *
    * @returns {Promise}
    */
-  createCollections (collections) {
-    return Bluebird.map(
-      Object.entries(collections),
-      ([collection, mappings]) => {
+  createCollections(collections, { indexCacheOnly = false } = {}) {
+    return promiseAllN(
+      Object.entries(collections).map(([collection, config]) => async () => {
+        // @deprecated
+        if (!(config.mappings !== undefined && config.settings !== undefined)) {
+          // @deprecated
+          return global.kuzzle.ask(
+            `core:storage:${this.scope}:collection:create`,
+            this.index,
+            collection,
+            { mappings: config },
+            { indexCacheOnly }
+          );
+        }
+
+        // @deprecated
+        const isConfigDeprecated =
+          config.settings.number_of_shards === undefined &&
+          config.settings.number_of_replicas === undefined;
+
+        if (indexCacheOnly) {
+          return global.kuzzle.ask(
+            `core:storage:${this.scope}:collection:create`,
+            this.index,
+            collection,
+            // @deprecated
+            isConfigDeprecated ? { mappings: config.mappings } : config,
+            { indexCacheOnly }
+          );
+        }
+
+        const exist = await global.kuzzle.ask(
+          `core:storage:${this.scope}:collection:exist`,
+          this.index,
+          collection
+        );
+
+        if (exist) {
+          // @deprecated
+          const dynamicSettings = isConfigDeprecated
+            ? null
+            : getESIndexDynamicSettings(config.settings);
+
+          const existingSettings = await global.kuzzle.ask(
+            `core:storage:${this.scope}:collection:settings:get`,
+            this.index,
+            collection
+          );
+
+          if (
+            !isConfigDeprecated &&
+            parseInt(existingSettings.number_of_shards) !==
+              config.settings.number_of_shards
+          ) {
+            if (global.NODE_ENV === "development") {
+              throw kerror.get(
+                "storage",
+                "wrong_collection_number_of_shards",
+                collection,
+                this.index,
+                this.scope,
+                "number_of_shards",
+                config.settings.number_of_shards,
+                existingSettings.number_of_shards
+              );
+            }
+            global.kuzzle.log.warn(
+              `Attempt to recreate an existing collection ${collection} of index ${this.index} of scope ${this.scope} with non matching static setting : number_of_shards at ${config.settings.number_of_shards} while existing one is at ${existingSettings.number_of_shards}`
+            );
+          }
+
+          return global.kuzzle.ask(
+            `core:storage:${this.scope}:collection:create`,
+            this.index,
+            collection,
+            // @deprecated
+            isConfigDeprecated
+              ? { mappings: config.mappings }
+              : { mappings: config.mappings, settings: dynamicSettings },
+            { indexCacheOnly: true }
+          );
+        }
+
         return global.kuzzle.ask(
           `core:storage:${this.scope}:collection:create`,
           this.index,
           collection,
-          { mappings });
-      },
-      { concurrency: 10 });
+          // @deprecated
+          isConfigDeprecated ? { mappings: config.mappings } : config,
+          { indexCacheOnly }
+        );
+      }),
+      10
+    );
   }
 }
 

package/lib/core/statistics/index.js

@@ -19,6 +19,6 @@
  * limitations under the License.
  */
 
-'use strict';
+"use strict";
 
-module.exports = require('./statistics');
+module.exports = require("./statistics");

package/lib/core/statistics/statistics.js

@@ -19,21 +19,21 @@
  * limitations under the License.
  */
 
-'use strict';
+"use strict";
 
-const errorApiAssert = require('../../kerror').wrap('api', 'assert');
-const errorStats = require('../../kerror').wrap('services', 'stats');
+const errorApiAssert = require("../../kerror").wrap("api", "assert");
+const errorStats = require("../../kerror").wrap("services", "stats");
 
 /**
  * @class Statistics
  * @param {Kuzzle} kuzzle
  */
 class Statistics {
-  constructor () {
+  constructor() {
     // uses '{' and '}' to force all statistics frames to be stored on 1 redis
     // node
     // (see https://redis.io/topics/cluster-spec#keys-distribution-model)
-    this.cacheKeyPrefix = '{stats/}';
+    this.cacheKeyPrefix = "{stats/}";
 
     this.enabled = global.kuzzle.config.stats.enabled;
     this.ttl = global.kuzzle.config.stats.ttl * 1000;
@@ -45,7 +45,7 @@ class Statistics {
       completedRequests: new Map(),
       connections: new Map(),
       failedRequests: new Map(),
-      ongoingRequests: new Map()
+      ongoingRequests: new Map(),
     };
   }
 
@@ -54,24 +54,24 @@ class Statistics {
    *
    * @param {Request} request
    */
-  startRequest (request) {
-    if (! this.enabled) {
+  startRequest(request) {
+    if (!this.enabled) {
       return;
     }
 
     const protocol = request && request.context.connection.protocol;
 
-    if (! protocol) {
+    if (!protocol) {
       return;
     }
 
-    if (! this.currentStats.ongoingRequests.has(protocol)) {
+    if (!this.currentStats.ongoingRequests.has(protocol)) {
       this.currentStats.ongoingRequests.set(protocol, 1);
-    }
-    else {
+    } else {
       this.currentStats.ongoingRequests.set(
         protocol,
-        this.currentStats.ongoingRequests.get(protocol) + 1);
+        this.currentStats.ongoingRequests.get(protocol) + 1
+      );
     }
   }
 
@@ -80,28 +80,29 @@ class Statistics {
    *
    * @param {Request} request
    */
-  completedRequest (request) {
-    if (! this.enabled) {
+  completedRequest(request) {
+    if (!this.enabled) {
       return;
     }
 
     const protocol = request && request.context.connection.protocol;
 
-    if (! protocol) {
+    if (!protocol) {
       return;
     }
 
     this.currentStats.ongoingRequests.set(
       protocol,
-      this.currentStats.ongoingRequests.get(protocol) - 1);
+      this.currentStats.ongoingRequests.get(protocol) - 1
+    );
 
-    if (! this.currentStats.completedRequests.has(protocol)) {
+    if (!this.currentStats.completedRequests.has(protocol)) {
       this.currentStats.completedRequests.set(protocol, 1);
-    }
-    else {
+    } else {
       this.currentStats.completedRequests.set(
         protocol,
-        this.currentStats.completedRequests.get(protocol) + 1);
+        this.currentStats.completedRequests.get(protocol) + 1
+      );
     }
   }
 
@@ -110,28 +111,29 @@ class Statistics {
    *
    * @param {Request} request
    */
-  failedRequest (request) {
-    if (! this.enabled) {
+  failedRequest(request) {
+    if (!this.enabled) {
       return;
     }
 
     const protocol = request && request.context.connection.protocol;
 
-    if (! protocol) {
+    if (!protocol) {
       return;
     }
 
     this.currentStats.ongoingRequests.set(
       protocol,
-      this.currentStats.ongoingRequests.get(protocol) - 1);
+      this.currentStats.ongoingRequests.get(protocol) - 1
+    );
 
-    if (! this.currentStats.failedRequests.has(protocol)) {
+    if (!this.currentStats.failedRequests.has(protocol)) {
       this.currentStats.failedRequests.set(protocol, 1);
-    }
-    else {
+    } else {
       this.currentStats.failedRequests.set(
         protocol,
-        this.currentStats.failedRequests.get(protocol) + 1);
+        this.currentStats.failedRequests.get(protocol) + 1
+      );
     }
   }
 
@@ -140,22 +142,25 @@ class Statistics {
    *
    * @param {RequestContext} requestContext
    */
-  newConnection (requestContext) {
-    if (! this.enabled) {
+  newConnection(requestContext) {
+    if (!this.enabled) {
       return;
     }
 
-    if (! requestContext.connection.protocol) {
+    if (!requestContext.connection.protocol) {
       return;
     }
 
-    if (! this.currentStats.connections.has(requestContext.connection.protocol)) {
+    if (
+      !this.currentStats.connections.has(requestContext.connection.protocol)
+    ) {
       this.currentStats.connections.set(requestContext.connection.protocol, 1);
-    }
-    else {
+    } else {
       this.currentStats.connections.set(
         requestContext.connection.protocol,
-        this.currentStats.connections.get(requestContext.connection.protocol) + 1);
+        this.currentStats.connections.get(requestContext.connection.protocol) +
+          1
+      );
     }
   }
 
@@ -164,22 +169,26 @@ class Statistics {
    *
    * @param {RequestContext} requestContext
    */
-  dropConnection (requestContext) {
-    if (! this.enabled) {
+  dropConnection(requestContext) {
+    if (!this.enabled) {
       return;
     }
 
-    if (! requestContext.connection.protocol) {
+    if (!requestContext.connection.protocol) {
       return;
     }
 
-    if (this.currentStats.connections.get(requestContext.connection.protocol) === 1) {
+    if (
+      this.currentStats.connections.get(requestContext.connection.protocol) ===
+      1
+    ) {
       this.currentStats.connections.delete(requestContext.connection.protocol);
-    }
-    else {
+    } else {
       this.currentStats.connections.set(
         requestContext.connection.protocol,
-        this.currentStats.connections.get(requestContext.connection.protocol) - 1);
+        this.currentStats.connections.get(requestContext.connection.protocol) -
+          1
+      );
     }
   }
 
@@ -188,22 +197,24 @@ class Statistics {
    *
    * @returns {Promise}
    */
-  async getLastStats () {
-    if (! this.enabled) {
-      throw errorStats.get('not_available');
+  async getLastStats() {
+    if (!this.enabled) {
+      throw errorStats.get("not_available");
     }
 
     const frame = Object.assign(
-      { timestamp: (new Date()).getTime() },
-      this.currentStats);
+      { timestamp: new Date().getTime() },
+      this.currentStats
+    );
 
-    if (! this.lastFrame) {
+    if (!this.lastFrame) {
       return frame;
     }
 
     const last = await global.kuzzle.ask(
-      'core:cache:internal:get',
-      this.cacheKeyPrefix + this.lastFrame);
+      "core:cache:internal:get",
+      this.cacheKeyPrefix + this.lastFrame
+    );
 
     return Object.assign(frame, JSON.parse(last));
   }
@@ -214,20 +225,19 @@ class Statistics {
    * @param {Request} request
    * @returns {Promise<Object>}
    */
-  async getStats (request) {
-    if (! this.enabled) {
-      throw errorStats.get('not_available');
+  async getStats(request) {
+    if (!this.enabled) {
+      throw errorStats.get("not_available");
     }
 
     const response = {
       hits: [],
-      total: null
+      total: null,
     };
     const currentDate = new Date().getTime();
     let startTime;
     let stopTime;
 
-
     if (request && request.input.args && request.input.args.startTime) {
       startTime = isNaN(request.input.args.startTime)
         ? new Date(request.input.args.startTime).getTime()
@@ -241,11 +251,11 @@ class Statistics {
     }
 
     if (startTime !== undefined && isNaN(startTime)) {
-      throw errorApiAssert.get('invalid_argument', 'startTime', 'number');
+      throw errorApiAssert.get("invalid_argument", "startTime", "number");
     }
 
     if (stopTime !== undefined && isNaN(stopTime)) {
-      throw errorApiAssert.get('invalid_argument', 'stopTime', 'number');
+      throw errorApiAssert.get("invalid_argument", "stopTime", "number");
     }
 
     if (startTime !== undefined && startTime >= currentDate) {
@@ -254,18 +264,19 @@ class Statistics {
     }
 
     const stats = {
-      completedRequests: Object.fromEntries(this.currentStats.completedRequests),
+      completedRequests: Object.fromEntries(
+        this.currentStats.completedRequests
+      ),
       connections: Object.fromEntries(this.currentStats.connections),
       failedRequests: Object.fromEntries(this.currentStats.failedRequests),
       ongoingRequests: Object.fromEntries(this.currentStats.ongoingRequests),
     };
 
-    if (! this.lastFrame) {
-      if (! stopTime || stopTime >= currentDate) {
+    if (!this.lastFrame) {
+      if (!stopTime || stopTime >= currentDate) {
         response.hits.push(
-          Object.assign(
-            { timestamp: (new Date(currentDate)).getTime() },
-            stats));
+          Object.assign({ timestamp: new Date(currentDate).getTime() }, stats)
+        );
       }
 
       response.total = response.hits.length;
@@ -274,27 +285,31 @@ class Statistics {
     }
 
     const frames = await global.kuzzle.ask(
-      'core:cache:internal:searchKeys',
-      `${this.cacheKeyPrefix}*`);
+      "core:cache:internal:searchKeys",
+      `${this.cacheKeyPrefix}*`
+    );
 
     // Statistics keys are timestamp.
     // Ordering them guarantees stats frames to be returned in the right order
     const values = await global.kuzzle.ask(
-      'core:cache:internal:mget',
-      frames.sort());
+      "core:cache:internal:mget",
+      frames.sort()
+    );
 
     values.forEach((v, idx) => {
       const regex = new RegExp(`^${this.cacheKeyPrefix}`);
-      const frameDate = new Date(Number(frames[idx].replace(regex, '')));
+      const frameDate = new Date(Number(frames[idx].replace(regex, "")));
       const frameDateTime = frameDate.getTime();
 
-      if ( (! startTime || startTime <= frameDateTime)
-        && (! stopTime || stopTime >= frameDateTime)
+      if (
+        (!startTime || startTime <= frameDateTime) &&
+        (!stopTime || stopTime >= frameDateTime)
       ) {
         response.hits.push(
-          Object.assign(
-            JSON.parse(v),
-            { timestamp: (new Date(frameDateTime)).getTime() }));
+          Object.assign(JSON.parse(v), {
+            timestamp: new Date(frameDateTime).getTime(),
+          })
+        );
       }
     });
 
@@ -308,34 +323,33 @@ class Statistics {
    *
    * @returns {Promise<Object>}
    */
-  getAllStats () {
+  getAllStats() {
     return this.getStats();
   }
 
   /**
    * Init statistics component
    */
-  init () {
-    if (! this.enabled) {
+  init() {
+    if (!this.enabled) {
       return;
     }
 
     this.timer = setInterval(async () => {
       try {
         await this.writeStats();
-      }
-      catch (error) {
+      } catch (error) {
         global.kuzzle.log.error(`Cannot write stats frame: ${error}`);
       }
     }, this.interval);
 
-    global.kuzzle.on('core:cache:internal:flushdb', () => {
+    global.kuzzle.on("core:cache:internal:flushdb", () => {
       this.lastFrame = null;
     });
   }
 
-  async writeStats () {
-    if (! this.enabled) {
+  async writeStats() {
+    if (!this.enabled) {
       return;
     }
 
@@ -346,12 +360,12 @@ class Statistics {
     this.currentStats.failedRequests = new Map();
 
     await global.kuzzle.ask(
-      'core:cache:internal:store',
+      "core:cache:internal:store",
       this.cacheKeyPrefix + this.lastFrame,
       stats,
-      { ttl: this.ttl });
+      { ttl: this.ttl }
+    );
   }
 }
 
-
 module.exports = Statistics;