kuzzle 2.60.0-beta.3 → 2.60.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/api/controllers/adminController.d.ts +7 -1
- package/dist/lib/api/controllers/adminController.js +9 -2
- package/dist/lib/api/controllers/documentController.d.ts +1 -1
- package/dist/lib/api/funnel.js +12 -1
- package/dist/lib/api/httpRoutes.js +52 -20
- package/dist/lib/api/request/kuzzleRequest.d.ts +7 -2
- package/dist/lib/api/request/kuzzleRequest.js +20 -10
- package/dist/lib/api/request/requestContext.d.ts +1 -1
- package/dist/lib/api/request/requestResponse.js +2 -1
- package/dist/lib/cluster/command.d.ts +7 -0
- package/dist/lib/cluster/command.js +67 -28
- package/dist/lib/cluster/node.d.ts +13 -2
- package/dist/lib/cluster/node.js +20 -4
- package/dist/lib/cluster/subscriber.d.ts +2 -0
- package/dist/lib/cluster/subscriber.js +6 -0
- package/dist/lib/config/index.js +60 -3
- package/dist/lib/core/backend/backend.d.ts +4 -3
- package/dist/lib/core/backend/backend.js +15 -15
- package/dist/lib/core/backend/backendCluster.d.ts +1 -1
- package/dist/lib/core/backend/internalLogger.d.ts +6 -0
- package/dist/lib/core/backend/internalLogger.js +6 -0
- package/dist/lib/core/network/router.d.ts +18 -0
- package/dist/lib/core/network/router.js +44 -10
- package/dist/lib/core/security/authTokenSettings.d.ts +21 -0
- package/dist/lib/core/security/authTokenSettings.js +61 -0
- package/dist/lib/core/security/tokenRepository.js +8 -14
- package/dist/lib/core/shared/KoncordeWrapper.d.ts +9 -0
- package/dist/lib/core/shared/KoncordeWrapper.js +8 -8
- package/dist/lib/core/shared/store.js +4 -0
- package/dist/lib/core/storage/clientAdapter.d.ts +6 -0
- package/dist/lib/core/storage/clientAdapter.js +40 -5
- package/dist/lib/kuzzle/Logger.d.ts +4 -0
- package/dist/lib/kuzzle/Logger.js +9 -2
- package/dist/lib/kuzzle/internalIndexHandler.js +28 -11
- package/dist/lib/kuzzle/kuzzle.d.ts +17 -0
- package/dist/lib/kuzzle/kuzzle.js +62 -0
- package/dist/lib/kuzzle/vault.d.ts +13 -0
- package/dist/lib/kuzzle/vault.js +25 -7
- package/dist/lib/model/security/profile.d.ts +1 -1
- package/dist/lib/model/security/role.d.ts +1 -1
- package/dist/lib/model/security/user.d.ts +1 -1
- package/dist/lib/service/storage/7/elasticsearch.d.ts +18 -0
- package/dist/lib/service/storage/7/elasticsearch.js +50 -10
- package/dist/lib/service/storage/7/esWrapper.js +31 -2
- package/dist/lib/service/storage/8/elasticsearch.d.ts +18 -0
- package/dist/lib/service/storage/8/elasticsearch.js +50 -10
- package/dist/lib/service/storage/8/esWrapper.js +31 -2
- package/dist/lib/types/config/KuzzleConfiguration.d.ts +7 -0
- package/dist/lib/types/config/SecurityConfiguration.d.ts +4 -3
- package/dist/lib/types/controllers/ControllerDefinition.d.ts +4 -4
- package/dist/lib/types/events/EventGenericDocument.d.ts +2 -1
- package/dist/lib/types/events/EventProtocol.d.ts +1 -1
- package/dist/lib/util/mutex.d.ts +5 -4
- package/dist/lib/util/mutex.js +8 -2
- package/dist/package.json +1 -1
- package/package.json +1 -1
|
@@ -99,6 +99,17 @@ let _kuzzle = null;
|
|
|
99
99
|
* not keep a failed node alive, which a bare `process.exit(1)` never did
|
|
100
100
|
*/
|
|
101
101
|
const FAILURE_SHUTDOWN_TIMEOUT = 30000;
|
|
102
|
+
/**
|
|
103
|
+
* Stops the startup sequence once a shutdown has begun (#3045): `start()`
|
|
104
|
+
* catches it and leaves the shutdown to end the process. Internal: never
|
|
105
|
+
* handed to the caller of `start()`.
|
|
106
|
+
*/
|
|
107
|
+
class StartupInterruptedError extends Error {
|
|
108
|
+
constructor() {
|
|
109
|
+
super("Kuzzle is shutting down");
|
|
110
|
+
this.name = "StartupInterruptedError";
|
|
111
|
+
}
|
|
112
|
+
}
|
|
102
113
|
Reflect.defineProperty(global, "kuzzle", {
|
|
103
114
|
configurable: true,
|
|
104
115
|
enumerable: false,
|
|
@@ -175,9 +186,11 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
|
|
|
175
186
|
await new cacheEngine_1.default().init();
|
|
176
187
|
await new storageEngine_1.default().init();
|
|
177
188
|
await new realtime_1.default().init();
|
|
189
|
+
this.stopIfShuttingDown();
|
|
178
190
|
await this.internalIndex.init();
|
|
179
191
|
await new security_1.default().init();
|
|
180
192
|
// This will init the cluster module if enabled
|
|
193
|
+
this.stopIfShuttingDown();
|
|
181
194
|
await this.initKuzzleNode();
|
|
182
195
|
this.vault = vault_1.default.load(options?.vaultKey, options?.secretsFile, this.config.vault?.newAlgorithm);
|
|
183
196
|
this.validation.init();
|
|
@@ -189,21 +202,25 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
|
|
|
189
202
|
// before opening connections to external users
|
|
190
203
|
await this.entryPoint.init();
|
|
191
204
|
await this.debugger.init();
|
|
205
|
+
this.stopIfShuttingDown();
|
|
192
206
|
this.pluginsManager.application = application;
|
|
193
207
|
const pluginImports = await this.pluginsManager.init(options?.plugins);
|
|
194
208
|
const imports = lodash_1.default.merge({}, pluginImports, options?.import);
|
|
195
209
|
// Authentification plugins must be loaded before users import to avoid
|
|
196
210
|
// credentials related error which would prevent Kuzzle from starting
|
|
211
|
+
this.stopIfShuttingDown();
|
|
197
212
|
await this.loadInitialState(imports, options?.support);
|
|
198
213
|
await this.ask("core:security:verify");
|
|
199
214
|
this.router.init();
|
|
200
215
|
this.log.info("[✔] Core components loaded");
|
|
216
|
+
this.stopIfShuttingDown();
|
|
201
217
|
await this.install(options?.installations);
|
|
202
218
|
// `application` is the very wrapper handed to the plugins manager six
|
|
203
219
|
// lines up, so reading the name back off the manager was a round trip
|
|
204
220
|
// through a getter that can answer `undefined`.
|
|
205
221
|
this.log.info(`[✔] Start "${application.name}" application`);
|
|
206
222
|
this.openApiManager = new openapi_1.OpenApiManager(application.openApi, this.config.http.routes, this.pluginsManager.routes);
|
|
223
|
+
this.stopIfShuttingDown();
|
|
207
224
|
// @deprecated
|
|
208
225
|
await this.pipe("kuzzle:start");
|
|
209
226
|
await this.pipe("kuzzle:state:live");
|
|
@@ -215,6 +232,9 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
|
|
|
215
232
|
this._state = kuzzleStateEnum_1.default.RUNNING;
|
|
216
233
|
}
|
|
217
234
|
catch (error) {
|
|
235
|
+
if (this._state === kuzzleStateEnum_1.default.SHUTTING_DOWN) {
|
|
236
|
+
return this.leaveStartupToShutdown(error);
|
|
237
|
+
}
|
|
218
238
|
this.log.error(`[X] Cannot start Kuzzle ${this.version}: ${error instanceof Error ? error.message : (0, node_util_1.inspect)(error)}`);
|
|
219
239
|
// A failed start cannot be retried (a second Kuzzle instance cannot be
|
|
220
240
|
// built), and what it did start keeps the process alive: in production,
|
|
@@ -229,6 +249,36 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
|
|
|
229
249
|
throw error;
|
|
230
250
|
}
|
|
231
251
|
}
|
|
252
|
+
/**
|
|
253
|
+
* A node asked to stop while it starts must not go on taking cluster-wide
|
|
254
|
+
* locks or running the application's installations: if it is killed before
|
|
255
|
+
* its shutdown releases them, the other nodes wait for their TTL (#3045).
|
|
256
|
+
*/
|
|
257
|
+
stopIfShuttingDown() {
|
|
258
|
+
if (this._state === kuzzleStateEnum_1.default.SHUTTING_DOWN) {
|
|
259
|
+
throw new StartupInterruptedError();
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* The shutdown began during the startup, which stopped (or failed because
|
|
264
|
+
* of it): no "Cannot start" error and no second shutdown, the one running
|
|
265
|
+
* keeps its exit code and ends the process.
|
|
266
|
+
*
|
|
267
|
+
* The returned promise never settles, as `withLock` once the process is
|
|
268
|
+
* exiting: rejecting would send the caller down its failed-start path
|
|
269
|
+
* (often a `process.exit(1)`, which would cut the shutdown short), and
|
|
270
|
+
* resolving would tell it that Kuzzle is running.
|
|
271
|
+
*/
|
|
272
|
+
leaveStartupToShutdown(error) {
|
|
273
|
+
let message = "[ℹ] Startup stopped: Kuzzle is shutting down";
|
|
274
|
+
// A step that failed because of the shutdown, rather than stopped by it
|
|
275
|
+
if (!(error instanceof StartupInterruptedError)) {
|
|
276
|
+
const reason = error instanceof Error ? error.message : (0, node_util_1.inspect)(error);
|
|
277
|
+
message += ` (${reason})`;
|
|
278
|
+
}
|
|
279
|
+
this.log.info(message);
|
|
280
|
+
return new Promise(() => undefined);
|
|
281
|
+
}
|
|
232
282
|
/**
|
|
233
283
|
* Generates the node ID.
|
|
234
284
|
*
|
|
@@ -238,6 +288,13 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
|
|
|
238
288
|
if (this.config.cluster.enabled) {
|
|
239
289
|
await new cluster_1.default().init();
|
|
240
290
|
this.log.info("[✔] Cluster initialized");
|
|
291
|
+
// The storage caches were loaded before the join: what a peer created
|
|
292
|
+
// in between was broadcast before this node could hear it. Read them
|
|
293
|
+
// again before the plugins' `init()` relies on them (#3044).
|
|
294
|
+
await Promise.all([
|
|
295
|
+
this.ask("core:storage:public:cache:reload"),
|
|
296
|
+
this.ask("core:storage:private:cache:reload"),
|
|
297
|
+
]);
|
|
241
298
|
}
|
|
242
299
|
else {
|
|
243
300
|
this.log.info("[X] Cluster disabled: single node mode.");
|
|
@@ -317,6 +374,9 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
|
|
|
317
374
|
// one must not start before they are recorded
|
|
318
375
|
await (0, distributedLock_1.withLock)("backend:installations", async () => {
|
|
319
376
|
for (const installation of installations) {
|
|
377
|
+
// The lock may come long after it was asked for, once the node
|
|
378
|
+
// holding it is done: by then, this one may be shutting down
|
|
379
|
+
this.stopIfShuttingDown();
|
|
320
380
|
const isAlreadyInstalled = await this.ask("core:storage:private:document:exist", "kuzzle", "installations", installation.id);
|
|
321
381
|
if (!isAlreadyInstalled) {
|
|
322
382
|
try {
|
|
@@ -508,6 +568,8 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
|
|
|
508
568
|
// cannot serialise, and the payload is built right above.
|
|
509
569
|
const importPayloadHash = (0, crypto_1.sha256)((0, json_stable_stringify_1.default)(importPayload) ?? "");
|
|
510
570
|
const importType = async () => {
|
|
571
|
+
// Same as for the installations: the lock may come after a SIGTERM
|
|
572
|
+
this.stopIfShuttingDown();
|
|
511
573
|
// Read the bookkeeping *after* the lock, never before: a read taken
|
|
512
574
|
// before acquiring it is not protected by it, and the holder that
|
|
513
575
|
// acted on a stale "not initialized" used to create a document the
|
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
import { Vault } from "kuzzle-vault";
|
|
2
|
+
/**
|
|
3
|
+
* The default secrets file, `config/secrets.enc.json` at the project root.
|
|
4
|
+
*
|
|
5
|
+
* Installed as an npm package (KaaF), Kuzzle lives under `node_modules/kuzzle`
|
|
6
|
+
* and the project root is above it: from `node_modules/kuzzle/dist/lib/kuzzle`
|
|
7
|
+
* since the build outputs to `dist/` (2.50.0), from
|
|
8
|
+
* `node_modules/kuzzle/lib/kuzzle` before. Anywhere else (a Kuzzle checkout),
|
|
9
|
+
* the file is the checkout's own `config/secrets.enc.json`.
|
|
10
|
+
*
|
|
11
|
+
* @param dirname - the directory of this module
|
|
12
|
+
*/
|
|
13
|
+
declare function defaultSecretsFile(dirname: string): string;
|
|
2
14
|
declare function load(vaultKey?: string, secretsFile?: string, useNewAlgorithm?: boolean): Vault;
|
|
3
15
|
declare const _default: {
|
|
16
|
+
defaultSecretsFile: typeof defaultSecretsFile;
|
|
4
17
|
load: typeof load;
|
|
5
18
|
};
|
|
6
19
|
export = _default;
|
package/dist/lib/kuzzle/vault.js
CHANGED
|
@@ -35,15 +35,33 @@ const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
|
|
|
35
35
|
// strict while being false for every read before that first assignment
|
|
36
36
|
// (TD-57, #2760).
|
|
37
37
|
let ENV_VAULT_KEY;
|
|
38
|
+
/**
|
|
39
|
+
* The default secrets file, `config/secrets.enc.json` at the project root.
|
|
40
|
+
*
|
|
41
|
+
* Installed as an npm package (KaaF), Kuzzle lives under `node_modules/kuzzle`
|
|
42
|
+
* and the project root is above it: from `node_modules/kuzzle/dist/lib/kuzzle`
|
|
43
|
+
* since the build outputs to `dist/` (2.50.0), from
|
|
44
|
+
* `node_modules/kuzzle/lib/kuzzle` before. Anywhere else (a Kuzzle checkout),
|
|
45
|
+
* the file is the checkout's own `config/secrets.enc.json`.
|
|
46
|
+
*
|
|
47
|
+
* @param dirname - the directory of this module
|
|
48
|
+
*/
|
|
49
|
+
function defaultSecretsFile(dirname) {
|
|
50
|
+
const dir = dirname.split(node_path_1.default.sep).join("/");
|
|
51
|
+
for (const installed of [
|
|
52
|
+
"/node_modules/kuzzle/dist/lib/kuzzle",
|
|
53
|
+
"/node_modules/kuzzle/lib/kuzzle",
|
|
54
|
+
]) {
|
|
55
|
+
if (dir.endsWith(installed)) {
|
|
56
|
+
return node_path_1.default.resolve(dirname.slice(0, dirname.length - installed.length + 1), "config/secrets.enc.json");
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return node_path_1.default.resolve(dirname, "../../config/secrets.enc.json");
|
|
60
|
+
}
|
|
38
61
|
function load(vaultKey, secretsFile, useNewAlgorithm = false) {
|
|
39
|
-
// Using KaaF kuzzle is an npm package and is located under node_modules folder
|
|
40
|
-
// We need to get back to root folder of the project to get the secret file
|
|
41
|
-
const defaultEncryptedSecretsFile = __dirname.endsWith("/node_modules/kuzzle/lib/kuzzle")
|
|
42
|
-
? node_path_1.default.resolve(`${__dirname}/../../../../config/secrets.enc.json`)
|
|
43
|
-
: node_path_1.default.resolve(`${__dirname}/../../config/secrets.enc.json`);
|
|
44
62
|
const encryptedSecretsFile = secretsFile ||
|
|
45
63
|
process.env.KUZZLE_SECRETS_FILE ||
|
|
46
|
-
|
|
64
|
+
defaultSecretsFile(__dirname);
|
|
47
65
|
let key = vaultKey;
|
|
48
66
|
if ((0, isEmpty_1.default)(vaultKey) &&
|
|
49
67
|
(!(0, isEmpty_1.default)(process.env.KUZZLE_VAULT_KEY) || !(0, isEmpty_1.default)(ENV_VAULT_KEY))) {
|
|
@@ -71,5 +89,5 @@ function load(vaultKey, secretsFile, useNewAlgorithm = false) {
|
|
|
71
89
|
}
|
|
72
90
|
return vault;
|
|
73
91
|
}
|
|
74
|
-
module.exports = { load };
|
|
92
|
+
module.exports = { defaultSecretsFile, load };
|
|
75
93
|
//# sourceMappingURL=vault.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Policy, OptimizedPolicy, OptimizedPolicyRestrictions } from "../../types/index";
|
|
2
2
|
import type { Role } from "./role";
|
|
3
|
-
import type { KuzzleRequest } from "
|
|
3
|
+
import type { KuzzleRequest } from "../../api/request";
|
|
4
4
|
/** @internal */
|
|
5
5
|
type InternalProfilePolicy = {
|
|
6
6
|
role: Role;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ControllerRight, ControllerRights } from "../../types/controllers/ControllerRights";
|
|
2
|
-
import type { KuzzleRequest } from "
|
|
2
|
+
import type { KuzzleRequest } from "../../api/request";
|
|
3
3
|
import type { OptimizedPolicyRestrictions } from "../../types/PolicyRestrictions";
|
|
4
4
|
/**
|
|
5
5
|
* @class Role
|
|
@@ -452,6 +452,24 @@ export declare class ES7 {
|
|
|
452
452
|
mappings?: TypeMapping;
|
|
453
453
|
settings?: JSONObject;
|
|
454
454
|
}): Promise<null>;
|
|
455
|
+
/**
|
|
456
|
+
* Applies the mappings and settings of a `createCollection` call to a
|
|
457
|
+
* collection that already exists because another request created it,
|
|
458
|
+
* maybe so recently that its primary shards are still initializing.
|
|
459
|
+
*/
|
|
460
|
+
private _updateConcurrentlyCreatedCollection;
|
|
461
|
+
/**
|
|
462
|
+
* Waits until the primary shards of a collection are active, for at most
|
|
463
|
+
* `timeout`, and rejects if they are not by then.
|
|
464
|
+
*
|
|
465
|
+
* A collection another node has just created is in the cluster state, so
|
|
466
|
+
* it exists, before its primary shard is active: until then, Elasticsearch
|
|
467
|
+
* answers 503 to any document request on it. The node that creates a
|
|
468
|
+
* collection waits for its shards (`wait_for_active_shards`); a node that
|
|
469
|
+
* finds it already there calls this instead. It answers at once when the
|
|
470
|
+
* shards are already active.
|
|
471
|
+
*/
|
|
472
|
+
waitForCollectionShards(index: string, collection: string, timeout?: string): Promise<void>;
|
|
455
473
|
/**
|
|
456
474
|
* Retrieves settings definition for index/type
|
|
457
475
|
*
|
|
@@ -89,6 +89,9 @@ const HIDDEN_COLLECTION = "_kuzzle_keep";
|
|
|
89
89
|
// is refreshed while held, so a live holder is never overtaken; a crashed
|
|
90
90
|
// one leaves it for at most `ttl` (30 s).
|
|
91
91
|
const HIDDEN_COLLECTION_LOCK = { retryAttempts: Number.POSITIVE_INFINITY };
|
|
92
|
+
// How long `waitForCollectionShards` lets Elasticsearch wait for a
|
|
93
|
+
// collection's primary shards, which is also Elasticsearch's own default
|
|
94
|
+
const COLLECTION_SHARDS_TIMEOUT = "30s";
|
|
92
95
|
const ALIAS_PREFIX = "@"; // @todo next major release: Add ALIAS_PREFIX in FORBIDDEN_CHARS
|
|
93
96
|
const PRIVATE_PREFIX = "%";
|
|
94
97
|
const PUBLIC_PREFIX = "&";
|
|
@@ -1146,20 +1149,23 @@ class ES7 {
|
|
|
1146
1149
|
this._checkDynamicProperty(mappings);
|
|
1147
1150
|
const exists = await this.hasCollection(index, collection);
|
|
1148
1151
|
if (exists) {
|
|
1149
|
-
return this.
|
|
1152
|
+
return this._updateConcurrentlyCreatedCollection(index, collection, mappings, settings);
|
|
1150
1153
|
}
|
|
1151
1154
|
this._checkMappings(mappings);
|
|
1155
|
+
// Copies, not in-place changes: the race branch below must hand the
|
|
1156
|
+
// caller's own mappings and settings to `updateCollection`
|
|
1152
1157
|
requestBody.mappings = {
|
|
1153
1158
|
_meta: mappings._meta || this._config.commonMapping._meta,
|
|
1154
1159
|
dynamic: mappings.dynamic || this._config.commonMapping.dynamic,
|
|
1155
|
-
properties: lodash_1.default.merge(mappings.properties, this._config.commonMapping.properties),
|
|
1160
|
+
properties: lodash_1.default.merge({}, mappings.properties, this._config.commonMapping.properties),
|
|
1161
|
+
};
|
|
1162
|
+
requestBody.settings = {
|
|
1163
|
+
...settings,
|
|
1164
|
+
number_of_replicas: settings.number_of_replicas ||
|
|
1165
|
+
this._config.defaultSettings.number_of_replicas,
|
|
1166
|
+
number_of_shards: settings.number_of_shards ||
|
|
1167
|
+
this._config.defaultSettings.number_of_shards,
|
|
1156
1168
|
};
|
|
1157
|
-
settings.number_of_replicas =
|
|
1158
|
-
settings.number_of_replicas ||
|
|
1159
|
-
this._config.defaultSettings.number_of_replicas;
|
|
1160
|
-
settings.number_of_shards =
|
|
1161
|
-
settings.number_of_shards ||
|
|
1162
|
-
this._config.defaultSettings.number_of_shards;
|
|
1163
1169
|
try {
|
|
1164
1170
|
await this._client.indices.create(esRequest);
|
|
1165
1171
|
}
|
|
@@ -1167,8 +1173,10 @@ class ES7 {
|
|
|
1167
1173
|
if (lodash_1.default.get(error, "meta.body.error.type") ===
|
|
1168
1174
|
"resource_already_exists_exception") {
|
|
1169
1175
|
// race condition: the indice has been created between the "exists"
|
|
1170
|
-
// check above and this "create" attempt
|
|
1171
|
-
|
|
1176
|
+
// check above and this "create" attempt, by a request that may not
|
|
1177
|
+
// have waited for its shards yet. This request's mappings and
|
|
1178
|
+
// settings must still be applied, as if it had seen it exist
|
|
1179
|
+
return this._updateConcurrentlyCreatedCollection(index, collection, mappings, settings);
|
|
1172
1180
|
}
|
|
1173
1181
|
throw this._esWrapper.formatESError(error);
|
|
1174
1182
|
}
|
|
@@ -1187,6 +1195,38 @@ class ES7 {
|
|
|
1187
1195
|
}
|
|
1188
1196
|
return null;
|
|
1189
1197
|
}
|
|
1198
|
+
/**
|
|
1199
|
+
* Applies the mappings and settings of a `createCollection` call to a
|
|
1200
|
+
* collection that already exists because another request created it,
|
|
1201
|
+
* maybe so recently that its primary shards are still initializing.
|
|
1202
|
+
*/
|
|
1203
|
+
async _updateConcurrentlyCreatedCollection(index, collection, mappings, settings) {
|
|
1204
|
+
await this.waitForCollectionShards(index, collection);
|
|
1205
|
+
return this.updateCollection(index, collection, { mappings, settings });
|
|
1206
|
+
}
|
|
1207
|
+
/**
|
|
1208
|
+
* Waits until the primary shards of a collection are active, for at most
|
|
1209
|
+
* `timeout`, and rejects if they are not by then.
|
|
1210
|
+
*
|
|
1211
|
+
* A collection another node has just created is in the cluster state, so
|
|
1212
|
+
* it exists, before its primary shard is active: until then, Elasticsearch
|
|
1213
|
+
* answers 503 to any document request on it. The node that creates a
|
|
1214
|
+
* collection waits for its shards (`wait_for_active_shards`); a node that
|
|
1215
|
+
* finds it already there calls this instead. It answers at once when the
|
|
1216
|
+
* shards are already active.
|
|
1217
|
+
*/
|
|
1218
|
+
async waitForCollectionShards(index, collection, timeout = COLLECTION_SHARDS_TIMEOUT) {
|
|
1219
|
+
try {
|
|
1220
|
+
await this._client.cluster.health({
|
|
1221
|
+
index: this._getAlias(index, collection),
|
|
1222
|
+
timeout,
|
|
1223
|
+
wait_for_status: "yellow",
|
|
1224
|
+
});
|
|
1225
|
+
}
|
|
1226
|
+
catch (error) {
|
|
1227
|
+
throw this._esWrapper.formatESError(error);
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1190
1230
|
/**
|
|
1191
1231
|
* Retrieves settings definition for index/type
|
|
1192
1232
|
*
|
|
@@ -35,6 +35,35 @@ const kerror = (0, kerror_1.wrap)("services", "storage");
|
|
|
35
35
|
function isResponseError(error) {
|
|
36
36
|
return Boolean(error.meta);
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* What the client puts in `message` when the response carried nothing it could
|
|
40
|
+
* read: this placeholder, or an empty string for a bodyless response.
|
|
41
|
+
*/
|
|
42
|
+
const CLIENT_PLACEHOLDER_MESSAGE = "Response Error";
|
|
43
|
+
function isUsableMessage(message) {
|
|
44
|
+
return (typeof message === "string" &&
|
|
45
|
+
message.trim() !== "" &&
|
|
46
|
+
message !== CLIENT_PLACEHOLDER_MESSAGE);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* A message for an error that came with none worth keeping (typically a 503
|
|
50
|
+
* with no body while shards initialize), built from what the error still
|
|
51
|
+
* carries: the client's error name, the HTTP status and the response body.
|
|
52
|
+
*/
|
|
53
|
+
function describeUnreadableError(error) {
|
|
54
|
+
const statusCode = error.meta?.statusCode;
|
|
55
|
+
const body = error.meta?.body;
|
|
56
|
+
const hasBody = body !== undefined && body !== null && body !== "";
|
|
57
|
+
let description = statusCode === undefined
|
|
58
|
+
? "Elasticsearch request failed"
|
|
59
|
+
: `Elasticsearch responded ${statusCode}`;
|
|
60
|
+
description += hasBody
|
|
61
|
+
? `: ${(0, node_util_1.inspect)(body, { breakLength: Infinity, depth: 3 })}`
|
|
62
|
+
: " (no body)";
|
|
63
|
+
return error.name && error.name !== "Error"
|
|
64
|
+
? `${error.name}: ${description}`
|
|
65
|
+
: description;
|
|
66
|
+
}
|
|
38
67
|
const errorMessagesMapping = [
|
|
39
68
|
{
|
|
40
69
|
regex: /^\[es_rejected_execution_exception] rejected execution .*? on EsThreadPoolExecutor\[(.*?), .*$/,
|
|
@@ -258,8 +287,8 @@ class ESWrapper {
|
|
|
258
287
|
return kerror.get("unexpected_bad_request", errorMessage);
|
|
259
288
|
}
|
|
260
289
|
_handleUnknownError(error, message) {
|
|
261
|
-
debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "
|
|
262
|
-
return kerror.get("unexpected_error", message);
|
|
290
|
+
debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "meta.statusCode", "<no status code>"), error);
|
|
291
|
+
return kerror.get("unexpected_error", isUsableMessage(message) ? message : describeUnreadableError(error));
|
|
263
292
|
}
|
|
264
293
|
}
|
|
265
294
|
module.exports = ESWrapper;
|
|
@@ -443,6 +443,24 @@ export declare class ES8 {
|
|
|
443
443
|
mappings?: estypes.MappingTypeMapping;
|
|
444
444
|
settings?: estypes.IndicesIndexSettings;
|
|
445
445
|
}): Promise<null>;
|
|
446
|
+
/**
|
|
447
|
+
* Applies the mappings and settings of a `createCollection` call to a
|
|
448
|
+
* collection that already exists because another request created it,
|
|
449
|
+
* maybe so recently that its primary shards are still initializing.
|
|
450
|
+
*/
|
|
451
|
+
private _updateConcurrentlyCreatedCollection;
|
|
452
|
+
/**
|
|
453
|
+
* Waits until the primary shards of a collection are active, for at most
|
|
454
|
+
* `timeout`, and rejects if they are not by then.
|
|
455
|
+
*
|
|
456
|
+
* A collection another node has just created is in the cluster state, so
|
|
457
|
+
* it exists, before its primary shard is active: until then, Elasticsearch
|
|
458
|
+
* answers 503 to any document request on it. The node that creates a
|
|
459
|
+
* collection waits for its shards (`wait_for_active_shards`); a node that
|
|
460
|
+
* finds it already there calls this instead. It answers at once when the
|
|
461
|
+
* shards are already active.
|
|
462
|
+
*/
|
|
463
|
+
waitForCollectionShards(index: string, collection: string, timeout?: string): Promise<void>;
|
|
446
464
|
/**
|
|
447
465
|
* Retrieves settings definition for index/type
|
|
448
466
|
*
|
|
@@ -89,6 +89,9 @@ const HIDDEN_COLLECTION = "_kuzzle_keep";
|
|
|
89
89
|
// is refreshed while held, so a live holder is never overtaken; a crashed
|
|
90
90
|
// one leaves it for at most `ttl` (30 s).
|
|
91
91
|
const HIDDEN_COLLECTION_LOCK = { retryAttempts: Number.POSITIVE_INFINITY };
|
|
92
|
+
// How long `waitForCollectionShards` lets Elasticsearch wait for a
|
|
93
|
+
// collection's primary shards, which is also Elasticsearch's own default
|
|
94
|
+
const COLLECTION_SHARDS_TIMEOUT = "30s";
|
|
92
95
|
const ALIAS_PREFIX = "@"; // @todo next major release: Add ALIAS_PREFIX in FORBIDDEN_CHARS
|
|
93
96
|
const PRIVATE_PREFIX = "%";
|
|
94
97
|
const PUBLIC_PREFIX = "&";
|
|
@@ -1168,20 +1171,23 @@ class ES8 {
|
|
|
1168
1171
|
this._checkDynamicProperty(mappings);
|
|
1169
1172
|
const exists = await this.hasCollection(index, collection);
|
|
1170
1173
|
if (exists) {
|
|
1171
|
-
return this.
|
|
1174
|
+
return this._updateConcurrentlyCreatedCollection(index, collection, mappings, settings);
|
|
1172
1175
|
}
|
|
1173
1176
|
this._checkMappings(mappings);
|
|
1177
|
+
// Copies, not in-place changes: the race branch below must hand the
|
|
1178
|
+
// caller's own mappings and settings to `updateCollection`
|
|
1174
1179
|
esRequest.mappings = {
|
|
1175
1180
|
_meta: mappings._meta || this._config.commonMapping._meta,
|
|
1176
1181
|
dynamic: mappings.dynamic || this._config.commonMapping.dynamic,
|
|
1177
|
-
properties: lodash_1.default.merge(mappings.properties, this._config.commonMapping.properties),
|
|
1182
|
+
properties: lodash_1.default.merge({}, mappings.properties, this._config.commonMapping.properties),
|
|
1183
|
+
};
|
|
1184
|
+
esRequest.settings = {
|
|
1185
|
+
...settings,
|
|
1186
|
+
number_of_replicas: settings.number_of_replicas ||
|
|
1187
|
+
this._config.defaultSettings.number_of_replicas,
|
|
1188
|
+
number_of_shards: settings.number_of_shards ||
|
|
1189
|
+
this._config.defaultSettings.number_of_shards,
|
|
1178
1190
|
};
|
|
1179
|
-
settings.number_of_replicas =
|
|
1180
|
-
settings.number_of_replicas ||
|
|
1181
|
-
this._config.defaultSettings.number_of_replicas;
|
|
1182
|
-
settings.number_of_shards =
|
|
1183
|
-
settings.number_of_shards ||
|
|
1184
|
-
this._config.defaultSettings.number_of_shards;
|
|
1185
1191
|
try {
|
|
1186
1192
|
await this._client.indices.create(esRequest);
|
|
1187
1193
|
}
|
|
@@ -1189,8 +1195,10 @@ class ES8 {
|
|
|
1189
1195
|
if (lodash_1.default.get(error, "meta.body.error.type") ===
|
|
1190
1196
|
"resource_already_exists_exception") {
|
|
1191
1197
|
// race condition: the indice has been created between the "exists"
|
|
1192
|
-
// check above and this "create" attempt
|
|
1193
|
-
|
|
1198
|
+
// check above and this "create" attempt, by a request that may not
|
|
1199
|
+
// have waited for its shards yet. This request's mappings and
|
|
1200
|
+
// settings must still be applied, as if it had seen it exist
|
|
1201
|
+
return this._updateConcurrentlyCreatedCollection(index, collection, mappings, settings);
|
|
1194
1202
|
}
|
|
1195
1203
|
throw this._esWrapper.formatESError(error);
|
|
1196
1204
|
}
|
|
@@ -1209,6 +1217,38 @@ class ES8 {
|
|
|
1209
1217
|
}
|
|
1210
1218
|
return null;
|
|
1211
1219
|
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Applies the mappings and settings of a `createCollection` call to a
|
|
1222
|
+
* collection that already exists because another request created it,
|
|
1223
|
+
* maybe so recently that its primary shards are still initializing.
|
|
1224
|
+
*/
|
|
1225
|
+
async _updateConcurrentlyCreatedCollection(index, collection, mappings, settings) {
|
|
1226
|
+
await this.waitForCollectionShards(index, collection);
|
|
1227
|
+
return this.updateCollection(index, collection, { mappings, settings });
|
|
1228
|
+
}
|
|
1229
|
+
/**
|
|
1230
|
+
* Waits until the primary shards of a collection are active, for at most
|
|
1231
|
+
* `timeout`, and rejects if they are not by then.
|
|
1232
|
+
*
|
|
1233
|
+
* A collection another node has just created is in the cluster state, so
|
|
1234
|
+
* it exists, before its primary shard is active: until then, Elasticsearch
|
|
1235
|
+
* answers 503 to any document request on it. The node that creates a
|
|
1236
|
+
* collection waits for its shards (`wait_for_active_shards`); a node that
|
|
1237
|
+
* finds it already there calls this instead. It answers at once when the
|
|
1238
|
+
* shards are already active.
|
|
1239
|
+
*/
|
|
1240
|
+
async waitForCollectionShards(index, collection, timeout = COLLECTION_SHARDS_TIMEOUT) {
|
|
1241
|
+
try {
|
|
1242
|
+
await this._client.cluster.health({
|
|
1243
|
+
index: this._getAlias(index, collection),
|
|
1244
|
+
timeout,
|
|
1245
|
+
wait_for_status: "yellow",
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
catch (error) {
|
|
1249
|
+
throw this._esWrapper.formatESError(error);
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1212
1252
|
/**
|
|
1213
1253
|
* Retrieves settings definition for index/type
|
|
1214
1254
|
*
|
|
@@ -35,6 +35,35 @@ const kerror = (0, kerror_1.wrap)("services", "storage");
|
|
|
35
35
|
function isResponseError(error) {
|
|
36
36
|
return Boolean(error.meta);
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* What the client puts in `message` when the response carried nothing it could
|
|
40
|
+
* read: this placeholder, or an empty string for a bodyless response.
|
|
41
|
+
*/
|
|
42
|
+
const CLIENT_PLACEHOLDER_MESSAGE = "Response Error";
|
|
43
|
+
function isUsableMessage(message) {
|
|
44
|
+
return (typeof message === "string" &&
|
|
45
|
+
message.trim() !== "" &&
|
|
46
|
+
message !== CLIENT_PLACEHOLDER_MESSAGE);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* A message for an error that came with none worth keeping (typically a 503
|
|
50
|
+
* with no body while shards initialize), built from what the error still
|
|
51
|
+
* carries: the client's error name, the HTTP status and the response body.
|
|
52
|
+
*/
|
|
53
|
+
function describeUnreadableError(error) {
|
|
54
|
+
const statusCode = error.meta?.statusCode;
|
|
55
|
+
const body = error.meta?.body;
|
|
56
|
+
const hasBody = body !== undefined && body !== null && body !== "";
|
|
57
|
+
let description = statusCode === undefined
|
|
58
|
+
? "Elasticsearch request failed"
|
|
59
|
+
: `Elasticsearch responded ${statusCode}`;
|
|
60
|
+
description += hasBody
|
|
61
|
+
? `: ${(0, node_util_1.inspect)(body, { breakLength: Infinity, depth: 3 })}`
|
|
62
|
+
: " (no body)";
|
|
63
|
+
return error.name && error.name !== "Error"
|
|
64
|
+
? `${error.name}: ${description}`
|
|
65
|
+
: description;
|
|
66
|
+
}
|
|
38
67
|
const errorMessagesMapping = [
|
|
39
68
|
{
|
|
40
69
|
regex: /^\[es_rejected_execution_exception] rejected execution .*? on EsThreadPoolExecutor\[(.*?), .*$/,
|
|
@@ -258,8 +287,8 @@ class ESWrapper {
|
|
|
258
287
|
return kerror.get("unexpected_bad_request", errorMessage);
|
|
259
288
|
}
|
|
260
289
|
_handleUnknownError(error, message) {
|
|
261
|
-
debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "
|
|
262
|
-
return kerror.get("unexpected_error", message);
|
|
290
|
+
debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "meta.statusCode", "<no status code>"), error);
|
|
291
|
+
return kerror.get("unexpected_error", isUsableMessage(message) ? message : describeUnreadableError(error));
|
|
263
292
|
}
|
|
264
293
|
}
|
|
265
294
|
module.exports = ESWrapper;
|
|
@@ -163,6 +163,13 @@ export interface IKuzzleConfiguration {
|
|
|
163
163
|
* Optional here for the reason given on {@link IKuzzleConfiguration.version}.
|
|
164
164
|
*/
|
|
165
165
|
allowAllOrigins?: boolean;
|
|
166
|
+
/**
|
|
167
|
+
* Derived at startup: the deprecated configuration keys the user set
|
|
168
|
+
* (today, those of "security.jwt"), logged as a warning.
|
|
169
|
+
*
|
|
170
|
+
* Optional here for the reason given on {@link IKuzzleConfiguration.version}.
|
|
171
|
+
*/
|
|
172
|
+
deprecatedSettings?: string[];
|
|
166
173
|
};
|
|
167
174
|
/**
|
|
168
175
|
* Collection specifications to apply at startup, by index then collection.
|
|
@@ -21,9 +21,10 @@ export type SecurityConfiguration = {
|
|
|
21
21
|
/**
|
|
22
22
|
* @deprecated Use `security.authToken` instead.
|
|
23
23
|
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* v2.56.0 declared it:
|
|
24
|
+
* Each setting the configuration sets here applies unless `authToken` sets
|
|
25
|
+
* it too (`loadConfig()` resolves both sections to the values in effect,
|
|
26
|
+
* see `authTokenSetting`). Optional all the same, as v2.56.0 declared it:
|
|
27
|
+
* a required member breaks code that builds this type.
|
|
27
28
|
*/
|
|
28
29
|
jwt?: JSONObject;
|
|
29
30
|
/**
|
|
@@ -54,8 +54,8 @@ export type ControllerDefinition = {
|
|
|
54
54
|
handler: (request: KuzzleRequest) => Promise<any>;
|
|
55
55
|
/**
|
|
56
56
|
* Declare HTTP routes (optional).
|
|
57
|
-
*
|
|
58
|
-
*
|
|
57
|
+
* When omitted or empty, a default route is generated:
|
|
58
|
+
* `GET /_/<controller>/<action>`, both names in kebab-case.
|
|
59
59
|
*/
|
|
60
60
|
http?: HttpRoute[];
|
|
61
61
|
};
|
|
@@ -71,8 +71,8 @@ export type HttpRoute = {
|
|
|
71
71
|
verb: "get" | "head" | "post" | "put" | "delete" | "patch" | "options";
|
|
72
72
|
/**
|
|
73
73
|
* Route path.
|
|
74
|
-
* A route starting with `/`
|
|
75
|
-
*
|
|
74
|
+
* A route starting with `/` is used as is, any other route is prefixed
|
|
75
|
+
* with `/_/`.
|
|
76
76
|
*/
|
|
77
77
|
path: string;
|
|
78
78
|
/**
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { KDocument, KDocumentContent } from "kuzzle-types";
|
|
2
2
|
import type { JSONObject } from "../JSONObject";
|
|
3
|
-
import type { KuzzleRequest
|
|
3
|
+
import type { KuzzleRequest } from "../../api/request";
|
|
4
|
+
import type { PipeEventHandler } from "../EventHandler";
|
|
4
5
|
/**
|
|
5
6
|
* Events with documents only having the `_id`
|
|
6
7
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { JSONObject } from "../JSONObject";
|
|
2
|
-
import type { PipeEventHandler } from "
|
|
2
|
+
import type { PipeEventHandler } from "../EventHandler";
|
|
3
3
|
import type { ClientConnection } from "../ClientConnection";
|
|
4
4
|
import type { HttpMessage } from "../HttpMessage";
|
|
5
5
|
export type EventHTTPBeforeParsingPayload = {
|
package/dist/lib/util/mutex.d.ts
CHANGED
|
@@ -86,9 +86,9 @@ export declare class Mutex {
|
|
|
86
86
|
*
|
|
87
87
|
* @return {Promise.<boolean>} True if the ressource has been unlocked before the `timeout`
|
|
88
88
|
*/
|
|
89
|
-
wait({ attemptDelay, timeout, }
|
|
90
|
-
attemptDelay?: number
|
|
91
|
-
timeout?: number
|
|
89
|
+
wait({ attemptDelay, timeout, }?: {
|
|
90
|
+
attemptDelay?: number;
|
|
91
|
+
timeout?: number;
|
|
92
92
|
}): Promise<boolean>;
|
|
93
93
|
/**
|
|
94
94
|
* Frees every lock this process holds, and stops it from taking new ones.
|
|
@@ -101,7 +101,8 @@ export declare class Mutex {
|
|
|
101
101
|
*
|
|
102
102
|
* From then on, `lock()` never resolves: the process is about to exit, and
|
|
103
103
|
* answering `false` would send its caller down a path meant for a lock
|
|
104
|
-
* held by another node.
|
|
104
|
+
* held by another node. And `unlock()` on a lock it already freed resolves
|
|
105
|
+
* without doing anything, since its holder may still try to give it back.
|
|
105
106
|
*/
|
|
106
107
|
static releaseAllBeforeExit(): Promise<void>;
|
|
107
108
|
get locked(): boolean;
|