@resolveio/server-lib 22.2.18 → 22.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/managers/error-auto-fix.manager.d.ts +6 -0
  2. package/managers/error-auto-fix.manager.js +215 -85
  3. package/managers/error-auto-fix.manager.js.map +1 -1
  4. package/managers/method.manager.js +2 -0
  5. package/managers/method.manager.js.map +1 -1
  6. package/managers/mongo.manager.d.ts +3 -0
  7. package/managers/mongo.manager.js +184 -65
  8. package/managers/mongo.manager.js.map +1 -1
  9. package/managers/slow-query-verifier.manager.d.ts +30 -0
  10. package/managers/slow-query-verifier.manager.js +691 -192
  11. package/managers/slow-query-verifier.manager.js.map +1 -1
  12. package/managers/subscription.manager.d.ts +1 -0
  13. package/managers/subscription.manager.js +60 -21
  14. package/managers/subscription.manager.js.map +1 -1
  15. package/managers/worker-dispatcher.manager.js +60 -6
  16. package/managers/worker-dispatcher.manager.js.map +1 -1
  17. package/methods/ai-terminal.d.ts +4 -0
  18. package/methods/ai-terminal.js +64 -26
  19. package/methods/ai-terminal.js.map +1 -1
  20. package/methods/app-settings.js +2 -2
  21. package/methods/app-settings.js.map +1 -1
  22. package/methods/diagnostics.d.ts +2 -0
  23. package/methods/diagnostics.js +514 -0
  24. package/methods/diagnostics.js.map +1 -0
  25. package/methods.ts +15 -0
  26. package/package.json +1 -1
  27. package/resolveio-server-app.d.ts +17 -1
  28. package/resolveio-server-app.js +293 -25
  29. package/resolveio-server-app.js.map +1 -1
  30. package/server-app.js +22 -36
  31. package/server-app.js.map +1 -1
  32. package/util/error-reporter.js +26 -126
  33. package/util/error-reporter.js.map +1 -1
  34. package/util/slow-query-reporter.d.ts +0 -3
  35. package/util/slow-query-reporter.js +13 -120
  36. package/util/slow-query-reporter.js.map +1 -1
package/managers/mongo.manager.js CHANGED
@@ -252,18 +252,18 @@ var MongoManager = /** @class */ (function () {
252
252
  attempt = 0;
253
253
  maxRetries = 4;
254
254
  _loop_1 = function () {
255
- var session, store, result_1, err_1, jitter, _b, activeStore;
256
- return __generator(this, function (_c) {
257
- switch (_c.label) {
255
+ var session, store, result_1, err_1, _b, jitter, _c, activeStore;
256
+ return __generator(this, function (_d) {
257
+ switch (_d.label) {
258
258
  case 0:
259
259
  session = resolveio_server_app_1.ResolveIOServer.getMongoConnection().startSession();
260
260
  store = asyncLocalStorage.getStore();
261
261
  if (store) {
262
262
  store.session = session;
263
263
  }
264
- _c.label = 1;
264
+ _d.label = 1;
265
265
  case 1:
266
- _c.trys.push([1, 3, 5, 9]);
266
+ _d.trys.push([1, 3, 9, 13]);
267
267
  return [4 /*yield*/, session.withTransaction(function () { return __awaiter(_this, void 0, void 0, function () {
268
268
  return __generator(this, function (_a) {
269
269
  switch (_a.label) {
@@ -275,35 +275,47 @@ var MongoManager = /** @class */ (function () {
275
275
  });
276
276
  }); })];
277
277
  case 2:
278
- _c.sent();
278
+ _d.sent();
279
279
  return [2 /*return*/, { value: result_1 }];
280
280
  case 3:
281
- err_1 = _c.sent();
281
+ err_1 = _d.sent();
282
282
  if (!this_1.shouldRetryTransactionError(err_1) || attempt >= maxRetries) {
283
283
  throw err_1;
284
284
  }
285
- attempt += 1;
286
- jitter = Math.floor(Math.random() * 60);
287
- return [4 /*yield*/, this_1.delay((120 * attempt) + jitter)];
285
+ if (!this_1.shouldRetryConnectionError(err_1)) return [3 /*break*/, 7];
286
+ _d.label = 4;
288
287
  case 4:
289
- _c.sent();
290
- return [3 /*break*/, 9];
288
+ _d.trys.push([4, 6, , 7]);
289
+ return [4 /*yield*/, resolveio_server_app_1.ResolveIOServer.requestMongoReconnect('mongo-transaction-retry', err_1)];
291
290
  case 5:
292
- _c.trys.push([5, 7, , 8]);
293
- return [4 /*yield*/, session.endSession()];
291
+ _d.sent();
292
+ return [3 /*break*/, 7];
294
293
  case 6:
295
- _c.sent();
296
- return [3 /*break*/, 8];
294
+ _b = _d.sent();
295
+ return [3 /*break*/, 7];
297
296
  case 7:
298
- _b = _c.sent();
299
- return [3 /*break*/, 8];
297
+ attempt += 1;
298
+ jitter = Math.floor(Math.random() * 60);
299
+ return [4 /*yield*/, this_1.delay((120 * attempt) + jitter)];
300
300
  case 8:
301
+ _d.sent();
302
+ return [3 /*break*/, 13];
303
+ case 9:
304
+ _d.trys.push([9, 11, , 12]);
305
+ return [4 /*yield*/, session.endSession()];
306
+ case 10:
307
+ _d.sent();
308
+ return [3 /*break*/, 12];
309
+ case 11:
310
+ _c = _d.sent();
311
+ return [3 /*break*/, 12];
312
+ case 12:
301
313
  activeStore = asyncLocalStorage.getStore();
302
314
  if (activeStore) {
303
315
  delete activeStore.session;
304
316
  }
305
317
  return [7 /*endfinally*/];
306
- case 9: return [2 /*return*/];
318
+ case 13: return [2 /*return*/];
307
319
  }
308
320
  });
309
321
  };
@@ -428,7 +440,7 @@ var MongoManager = /** @class */ (function () {
428
440
  if (!err) {
429
441
  return false;
430
442
  }
431
- if (this.shouldRetryWriteConflict(err) || this.shouldRetryWaitQueueTimeout(err)) {
443
+ if (this.shouldRetryWriteConflict(err) || this.shouldRetryWaitQueueTimeout(err) || this.shouldRetryConnectionError(err)) {
432
444
  return true;
433
445
  }
434
446
  return this.hasMongoErrorLabel(err, 'TransientTransactionError')
@@ -443,37 +455,72 @@ var MongoManager = /** @class */ (function () {
443
455
  }
444
456
  return typeof err.message === 'string' && err.message.includes('Timed out while checking out a connection from connection pool');
445
457
  };
458
+ MongoManager.prototype.shouldRetryConnectionError = function (err) {
459
+ if (!err) {
460
+ return false;
461
+ }
462
+ var name = String(err.name || err.codeName || '');
463
+ if (name === 'MongoPoolClearedError'
464
+ || name === 'MongoNetworkError'
465
+ || name === 'MongoNetworkTimeoutError'
466
+ || name === 'MongoServerSelectionError'
467
+ || name === 'MongoTopologyClosedError'
468
+ || name === 'MongoNotConnectedError'
469
+ || name === 'MongoClientClosedError') {
470
+ return true;
471
+ }
472
+ var message = String(err.message || '');
473
+ return /connection pool .* was cleared/i.test(message)
474
+ || /topology is closed/i.test(message)
475
+ || /client was closed/i.test(message)
476
+ || /not connected/i.test(message)
477
+ || /server selection timed out/i.test(message)
478
+ || /econnrefused|econnreset|etimedout|ehostunreach/i.test(message);
479
+ };
446
480
  MongoManager.prototype.retryRead = function (operation, options) {
447
481
  return __awaiter(this, void 0, void 0, function () {
448
- var attempt, maxRetries, err_2;
449
- return __generator(this, function (_a) {
450
- switch (_a.label) {
482
+ var attempt, maxRetries, err_2, shouldRetry, _a;
483
+ return __generator(this, function (_b) {
484
+ switch (_b.label) {
451
485
  case 0:
452
486
  if (options && options.session && typeof options.session.inTransaction === 'function' && options.session.inTransaction()) {
453
487
  return [2 /*return*/, operation()];
454
488
  }
455
489
  attempt = 0;
456
490
  maxRetries = 2;
457
- _a.label = 1;
491
+ _b.label = 1;
458
492
  case 1:
459
- if (!true) return [3 /*break*/, 7];
460
- _a.label = 2;
493
+ if (!true) return [3 /*break*/, 11];
494
+ _b.label = 2;
461
495
  case 2:
462
- _a.trys.push([2, 4, , 6]);
496
+ _b.trys.push([2, 4, , 10]);
463
497
  return [4 /*yield*/, operation()];
464
- case 3: return [2 /*return*/, _a.sent()];
498
+ case 3: return [2 /*return*/, _b.sent()];
465
499
  case 4:
466
- err_2 = _a.sent();
467
- if (!this.shouldRetryWaitQueueTimeout(err_2) || attempt >= maxRetries) {
500
+ err_2 = _b.sent();
501
+ shouldRetry = this.shouldRetryWaitQueueTimeout(err_2) || this.shouldRetryConnectionError(err_2);
502
+ if (!shouldRetry || attempt >= maxRetries) {
468
503
  throw err_2;
469
504
  }
505
+ if (!this.shouldRetryConnectionError(err_2)) return [3 /*break*/, 8];
506
+ _b.label = 5;
507
+ case 5:
508
+ _b.trys.push([5, 7, , 8]);
509
+ return [4 /*yield*/, resolveio_server_app_1.ResolveIOServer.requestMongoReconnect('mongo-read-retry', err_2)];
510
+ case 6:
511
+ _b.sent();
512
+ return [3 /*break*/, 8];
513
+ case 7:
514
+ _a = _b.sent();
515
+ return [3 /*break*/, 8];
516
+ case 8:
470
517
  attempt += 1;
471
518
  return [4 /*yield*/, this.delay(100 * attempt)];
472
- case 5:
473
- _a.sent();
474
- return [3 /*break*/, 6];
475
- case 6: return [3 /*break*/, 1];
476
- case 7: return [2 /*return*/];
519
+ case 9:
520
+ _b.sent();
521
+ return [3 /*break*/, 10];
522
+ case 10: return [3 /*break*/, 1];
523
+ case 11: return [2 /*return*/];
477
524
  }
478
525
  });
479
526
  });
@@ -999,6 +1046,31 @@ var MongoManager = /** @class */ (function () {
999
1046
  finally { if (e_5) throw e_5.error; }
1000
1047
  }
1001
1048
  };
1049
+ MongoManager.prototype.handleMongoReconnect = function (reason) {
1050
+ return __awaiter(this, void 0, void 0, function () {
1051
+ var error_1;
1052
+ return __generator(this, function (_a) {
1053
+ switch (_a.label) {
1054
+ case 0:
1055
+ this.clearQueryCache("mongo-reconnect:".concat(reason));
1056
+ if (!(this._isWorkersEnabled && this._isWorkerInstance)) return [3 /*break*/, 4];
1057
+ _a.label = 1;
1058
+ case 1:
1059
+ _a.trys.push([1, 3, , 4]);
1060
+ return [4 /*yield*/, this.setupChangeStream()];
1061
+ case 2:
1062
+ _a.sent();
1063
+ return [3 /*break*/, 4];
1064
+ case 3:
1065
+ error_1 = _a.sent();
1066
+ console.log(new Date(), 'Mongo handle reconnect: change stream restart failed', reason, error_1);
1067
+ this.scheduleChangeStreamRestart();
1068
+ return [3 /*break*/, 4];
1069
+ case 4: return [2 /*return*/];
1070
+ }
1071
+ });
1072
+ });
1073
+ };
1002
1074
  MongoManager.prototype.setupChangeStream = function () {
1003
1075
  return __awaiter(this, void 0, void 0, function () {
1004
1076
  var client, watchDatabases, pipeline, changeStream;
@@ -1314,6 +1386,28 @@ var MongoManagerCollection = /** @class */ (function () {
1314
1386
  }
1315
1387
  return typeof err.message === 'string' && err.message.includes('Timed out while checking out a connection from connection pool');
1316
1388
  };
1389
+ MongoManagerCollection.prototype.shouldRetryConnectionError = function (err) {
1390
+ if (!err) {
1391
+ return false;
1392
+ }
1393
+ var name = String(err.name || err.codeName || '');
1394
+ if (name === 'MongoPoolClearedError'
1395
+ || name === 'MongoNetworkError'
1396
+ || name === 'MongoNetworkTimeoutError'
1397
+ || name === 'MongoServerSelectionError'
1398
+ || name === 'MongoTopologyClosedError'
1399
+ || name === 'MongoNotConnectedError'
1400
+ || name === 'MongoClientClosedError') {
1401
+ return true;
1402
+ }
1403
+ var message = String(err.message || '');
1404
+ return /connection pool .* was cleared/i.test(message)
1405
+ || /topology is closed/i.test(message)
1406
+ || /client was closed/i.test(message)
1407
+ || /not connected/i.test(message)
1408
+ || /server selection timed out/i.test(message)
1409
+ || /econnrefused|econnreset|etimedout|ehostunreach/i.test(message);
1410
+ };
1317
1411
  MongoManagerCollection.prototype.delay = function (ms) {
1318
1412
  return __awaiter(this, void 0, void 0, function () {
1319
1413
  return __generator(this, function (_a) {
@@ -1324,9 +1418,9 @@ var MongoManagerCollection = /** @class */ (function () {
1324
1418
  };
1325
1419
  MongoManagerCollection.prototype.retryWrite = function (operation, options) {
1326
1420
  return __awaiter(this, void 0, void 0, function () {
1327
- var attempt, configuredMaxRetries, maxRetries, configuredBaseDelay, baseDelayMs, err_9, shouldRetry, backoff, jitter;
1328
- return __generator(this, function (_a) {
1329
- switch (_a.label) {
1421
+ var attempt, configuredMaxRetries, maxRetries, configuredBaseDelay, baseDelayMs, err_9, shouldRetry, _a, backoff, jitter;
1422
+ return __generator(this, function (_b) {
1423
+ switch (_b.label) {
1330
1424
  case 0:
1331
1425
  if (options && options.session && typeof options.session.inTransaction === 'function' && options.session.inTransaction()) {
1332
1426
  return [2 /*return*/, operation()];
@@ -1340,64 +1434,89 @@ var MongoManagerCollection = /** @class */ (function () {
1340
1434
  baseDelayMs = Number.isFinite(configuredBaseDelay) && configuredBaseDelay > 0
1341
1435
  ? Math.min(Math.floor(configuredBaseDelay), 5000)
1342
1436
  : 120;
1343
- _a.label = 1;
1437
+ _b.label = 1;
1344
1438
  case 1:
1345
- if (!true) return [3 /*break*/, 7];
1346
- _a.label = 2;
1439
+ if (!true) return [3 /*break*/, 11];
1440
+ _b.label = 2;
1347
1441
  case 2:
1348
- _a.trys.push([2, 4, , 6]);
1442
+ _b.trys.push([2, 4, , 10]);
1349
1443
  return [4 /*yield*/, operation()];
1350
- case 3: return [2 /*return*/, _a.sent()];
1444
+ case 3: return [2 /*return*/, _b.sent()];
1351
1445
  case 4:
1352
- err_9 = _a.sent();
1353
- shouldRetry = this.shouldRetryWriteConflict(err_9) || this.shouldRetryWaitQueueTimeout(err_9);
1446
+ err_9 = _b.sent();
1447
+ shouldRetry = this.shouldRetryWriteConflict(err_9) || this.shouldRetryWaitQueueTimeout(err_9) || this.shouldRetryConnectionError(err_9);
1354
1448
  if (!shouldRetry || attempt >= maxRetries) {
1355
1449
  throw err_9;
1356
1450
  }
1451
+ if (!this.shouldRetryConnectionError(err_9)) return [3 /*break*/, 8];
1452
+ _b.label = 5;
1453
+ case 5:
1454
+ _b.trys.push([5, 7, , 8]);
1455
+ return [4 /*yield*/, resolveio_server_app_1.ResolveIOServer.requestMongoReconnect("mongo-collection-write-retry:".concat(this.collectionName), err_9)];
1456
+ case 6:
1457
+ _b.sent();
1458
+ return [3 /*break*/, 8];
1459
+ case 7:
1460
+ _a = _b.sent();
1461
+ return [3 /*break*/, 8];
1462
+ case 8:
1357
1463
  attempt += 1;
1358
1464
  backoff = Math.min(5000, baseDelayMs * (Math.pow(2, (attempt - 1))));
1359
1465
  jitter = Math.floor(Math.random() * 90);
1360
1466
  return [4 /*yield*/, this.delay(backoff + jitter)];
1361
- case 5:
1362
- _a.sent();
1363
- return [3 /*break*/, 6];
1364
- case 6: return [3 /*break*/, 1];
1365
- case 7: return [2 /*return*/];
1467
+ case 9:
1468
+ _b.sent();
1469
+ return [3 /*break*/, 10];
1470
+ case 10: return [3 /*break*/, 1];
1471
+ case 11: return [2 /*return*/];
1366
1472
  }
1367
1473
  });
1368
1474
  });
1369
1475
  };
1370
1476
  MongoManagerCollection.prototype.retryRead = function (operation, options) {
1371
1477
  return __awaiter(this, void 0, void 0, function () {
1372
- var attempt, maxRetries, err_10;
1373
- return __generator(this, function (_a) {
1374
- switch (_a.label) {
1478
+ var attempt, maxRetries, err_10, shouldRetry, _a;
1479
+ return __generator(this, function (_b) {
1480
+ switch (_b.label) {
1375
1481
  case 0:
1376
1482
  if (options && options.session && typeof options.session.inTransaction === 'function' && options.session.inTransaction()) {
1377
1483
  return [2 /*return*/, operation()];
1378
1484
  }
1379
1485
  attempt = 0;
1380
1486
  maxRetries = 2;
1381
- _a.label = 1;
1487
+ _b.label = 1;
1382
1488
  case 1:
1383
- if (!true) return [3 /*break*/, 7];
1384
- _a.label = 2;
1489
+ if (!true) return [3 /*break*/, 11];
1490
+ _b.label = 2;
1385
1491
  case 2:
1386
- _a.trys.push([2, 4, , 6]);
1492
+ _b.trys.push([2, 4, , 10]);
1387
1493
  return [4 /*yield*/, operation()];
1388
- case 3: return [2 /*return*/, _a.sent()];
1494
+ case 3: return [2 /*return*/, _b.sent()];
1389
1495
  case 4:
1390
- err_10 = _a.sent();
1391
- if (!this.shouldRetryWaitQueueTimeout(err_10) || attempt >= maxRetries) {
1496
+ err_10 = _b.sent();
1497
+ shouldRetry = this.shouldRetryWaitQueueTimeout(err_10) || this.shouldRetryConnectionError(err_10);
1498
+ if (!shouldRetry || attempt >= maxRetries) {
1392
1499
  throw err_10;
1393
1500
  }
1501
+ if (!this.shouldRetryConnectionError(err_10)) return [3 /*break*/, 8];
1502
+ _b.label = 5;
1503
+ case 5:
1504
+ _b.trys.push([5, 7, , 8]);
1505
+ return [4 /*yield*/, resolveio_server_app_1.ResolveIOServer.requestMongoReconnect("mongo-collection-read-retry:".concat(this.collectionName), err_10)];
1506
+ case 6:
1507
+ _b.sent();
1508
+ return [3 /*break*/, 8];
1509
+ case 7:
1510
+ _a = _b.sent();
1511
+ return [3 /*break*/, 8];
1512
+ case 8:
1394
1513
  attempt += 1;
1395
1514
  return [4 /*yield*/, this.delay(100 * attempt)];
1396
- case 5:
1397
- _a.sent();
1398
- return [3 /*break*/, 6];
1399
- case 6: return [3 /*break*/, 1];
1400
- case 7: return [2 /*return*/];
1515
+ case 9:
1516
+ _b.sent();
1517
+ return [3 /*break*/, 10];
1518
+ case 10: return [3 /*break*/, 1];
1519
+ case 11: return [2 /*return*/];
1401
1520
  }
1402
1521
  });
1403
1522
  });