@oneuptime/common 12.0.28 → 12.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/Models/DatabaseModels/DashboardDomain.ts +43 -0
  2. package/Models/DatabaseModels/MonitorTemplate.ts +15 -3
  3. package/Models/DatabaseModels/StatusPageDomain.ts +43 -0
  4. package/Server/API/DashboardDomainAPI.ts +74 -0
  5. package/Server/API/MicrosoftTeamsAPI.ts +81 -180
  6. package/Server/API/StatusPageDomainAPI.ts +74 -0
  7. package/Server/Infrastructure/Postgres/SchemaMigrations/1790200000000-AllowNullMonitorNameOnMonitorTemplate.ts +43 -0
  8. package/Server/Infrastructure/Postgres/SchemaMigrations/1790300000000-AddCertificateReissueRequestedAtToDomains.ts +40 -0
  9. package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +4 -0
  10. package/Server/Middleware/ProjectAuthorization.ts +6 -1
  11. package/Server/Services/DashboardDomainService.ts +131 -0
  12. package/Server/Services/LogPipelineProcessorService.ts +88 -0
  13. package/Server/Services/MonitorTemplateService.ts +70 -11
  14. package/Server/Services/StatusPageDomainService.ts +163 -3
  15. package/Server/Services/StatusPagePrivateUserSessionService.ts +197 -56
  16. package/Server/Types/Domain.ts +17 -0
  17. package/Server/Utils/APIKey/AccessPermission.ts +24 -0
  18. package/Server/Utils/Greenlock/Greenlock.ts +156 -58
  19. package/Server/Utils/LogPipelineProcessorValidation.ts +112 -0
  20. package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.ts +1094 -45
  21. package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.ts +153 -17
  22. package/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.ts +23 -11
  23. package/Server/Utils/Workspace/MicrosoftTeams/Actions/Auth.ts +0 -170
  24. package/Server/Utils/Workspace/MicrosoftTeams/Actions/Authorization.ts +158 -0
  25. package/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.ts +23 -11
  26. package/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.ts +35 -13
  27. package/Tests/App/Dashboard/MonitorTemplateColumnAndFacet.test.tsx +505 -0
  28. package/Tests/App/StatusPage/StatusPageLoginCodeBootstrap.test.ts +106 -0
  29. package/Tests/App/StatusPage/StatusPageLoginCodeUtil.test.ts +84 -0
  30. package/Tests/App/StatusPage/StatusPageRedirectSafety.test.ts +119 -0
  31. package/Tests/Models/MonitorTemplateMonitorNameColumn.test.ts +258 -0
  32. package/Tests/Server/API/CustomDomainReissueSslAPI.test.ts +469 -0
  33. package/Tests/Server/API/MicrosoftTeamsBotMessagesEndpoint.test.ts +729 -0
  34. package/Tests/Server/API/MicrosoftTeamsBotTestEndpoint.test.ts +26 -0
  35. package/Tests/Server/API/MicrosoftTeamsLegacyWebhookRemoval.test.ts +402 -0
  36. package/Tests/Server/Middleware/ProjectAuthorizationApiKeyMiddleware.test.ts +4 -0
  37. package/Tests/Server/Services/AddCertificateReissueRequestedAtToDomainsMigration.test.ts +287 -0
  38. package/Tests/Server/Services/CustomDomainCertificateReissue.test.ts +477 -0
  39. package/Tests/Server/Services/LogPipelineProcessorSaveValidation.test.ts +249 -0
  40. package/Tests/Server/Services/MonitorTemplateServiceBulkSync.test.ts +549 -0
  41. package/Tests/Server/Services/MonitorTemplateServiceNetworkDeviceSync.test.ts +16 -3
  42. package/Tests/Server/Services/NetworkDeviceAutoImportRuleEngineService.test.ts +57 -0
  43. package/Tests/Server/Services/SecurityEventLastErrorColumnWidth.test.ts +151 -15
  44. package/Tests/Server/Services/StatusPagePrivateUserSessionService.test.ts +587 -0
  45. package/Tests/Server/Types/Database/Permissions/ApiKeyFileAccess.test.ts +286 -0
  46. package/Tests/Server/Types/Database/Permissions/MonitorTemplateColumnAccess.test.ts +328 -0
  47. package/Tests/Server/Types/Database/QueryHelperFindWithSameTextAndIsNull.test.ts +83 -0
  48. package/Tests/Server/Utils/AI/SRE/ConfidenceSignal.test.ts +377 -0
  49. package/Tests/Server/Utils/APIKey/AccessPermission.test.ts +16 -1
  50. package/Tests/Server/Utils/Greenlock/CertificateRenewalSchedule.test.ts +250 -0
  51. package/Tests/Server/Utils/LogPipelineProcessorValidation.test.ts +198 -0
  52. package/Tests/Server/Utils/SecurityEvent/ConnectorErrorMessage.test.ts +48 -9
  53. package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsAuth.test.ts +726 -0
  54. package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsClient.test.ts +74 -11
  55. package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsErrorHandling.test.ts +1107 -0
  56. package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsPoller.test.ts +20 -3
  57. package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsPollerFailureTaxonomy.test.ts +51 -11
  58. package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsPollerHardening.test.ts +1246 -0
  59. package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsRequestContract.test.ts +961 -0
  60. package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsResponseParsing.test.ts +653 -0
  61. package/Tests/Server/Utils/Workspace/MicrosoftTeamsActionAuthorization.test.ts +880 -0
  62. package/Tests/Types/API/HTTPResponse.test.ts +175 -0
  63. package/Tests/Types/API/Route.test.ts +36 -4
  64. package/Tests/Types/Database/DatabaseProperty.test.ts +171 -0
  65. package/Tests/Types/DateLocalShortDateTimeString.test.ts +518 -0
  66. package/Tests/Types/DateLocalTimeString.test.ts +259 -0
  67. package/Tests/Types/DateUserPrefers12HourFormat.test.ts +316 -0
  68. package/Tests/UI/Components/LogsHistogram.test.tsx +41 -2
  69. package/Tests/UI/Components/LogsViewerEmptyStateTimeRange.test.tsx +212 -0
  70. package/Tests/UI/Components/TelemetryChartClockFormat.test.tsx +359 -0
  71. package/Tests/UI/Components/TimeRangePickerClockFormat.test.tsx +422 -0
  72. package/Tests/UI/Components/TimeRangePickerDropdown.test.tsx +146 -8
  73. package/Tests/UI/MicrosoftTeams/MicrosoftTeamsMessagingEndpointGuidance.test.tsx +520 -0
  74. package/Tests/UI/Utils/Navigation.test.ts +82 -0
  75. package/Tests/Utils/Array.test.ts +100 -0
  76. package/Tests/Utils/CertificateReissue.test.ts +225 -0
  77. package/Tests/Utils/Grok/Grok.test.ts +492 -0
  78. package/Tests/Utils/Monitor/NetworkDeviceMonitorTemplateUtil.test.ts +181 -13
  79. package/Types/API/Route.ts +20 -5
  80. package/Types/Date.ts +143 -13
  81. package/Types/Rum/SessionReplay.ts +62 -1
  82. package/UI/Components/Date/TimeRangePickerDropdown.tsx +9 -5
  83. package/UI/Components/LogsViewer/LogsViewer.tsx +12 -7
  84. package/UI/Components/LogsViewer/components/HistogramTooltip.tsx +18 -17
  85. package/UI/Components/LogsViewer/components/LogsAnalyticsView.tsx +21 -28
  86. package/UI/Components/LogsViewer/components/LogsHistogram.tsx +2 -4
  87. package/UI/Components/TelemetryViewer/components/TelemetryHistogram.tsx +2 -4
  88. package/UI/Components/TelemetryViewer/components/TelemetryHistogramTooltip.tsx +17 -16
  89. package/UI/Utils/Navigation.ts +42 -0
  90. package/Utils/Array.ts +37 -0
  91. package/Utils/CertificateReissue.ts +128 -0
  92. package/Utils/Grok/Grok.ts +461 -0
  93. package/Utils/Grok/GrokPatterns.ts +118 -0
  94. package/Utils/Monitor/NetworkDeviceMonitorTemplateUtil.ts +58 -6
  95. package/build/dist/Models/DatabaseModels/DashboardDomain.js +44 -0
  96. package/build/dist/Models/DatabaseModels/DashboardDomain.js.map +1 -1
  97. package/build/dist/Models/DatabaseModels/MonitorTemplate.js +17 -4
  98. package/build/dist/Models/DatabaseModels/MonitorTemplate.js.map +1 -1
  99. package/build/dist/Models/DatabaseModels/StatusPageDomain.js +44 -0
  100. package/build/dist/Models/DatabaseModels/StatusPageDomain.js.map +1 -1
  101. package/build/dist/Server/API/DashboardDomainAPI.js +43 -1
  102. package/build/dist/Server/API/DashboardDomainAPI.js.map +1 -1
  103. package/build/dist/Server/API/MicrosoftTeamsAPI.js +74 -118
  104. package/build/dist/Server/API/MicrosoftTeamsAPI.js.map +1 -1
  105. package/build/dist/Server/API/StatusPageDomainAPI.js +43 -1
  106. package/build/dist/Server/API/StatusPageDomainAPI.js.map +1 -1
  107. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1790200000000-AllowNullMonitorNameOnMonitorTemplate.js +34 -0
  108. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1790200000000-AllowNullMonitorNameOnMonitorTemplate.js.map +1 -0
  109. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1790300000000-AddCertificateReissueRequestedAtToDomains.js +28 -0
  110. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1790300000000-AddCertificateReissueRequestedAtToDomains.js.map +1 -0
  111. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +4 -0
  112. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
  113. package/build/dist/Server/Middleware/ProjectAuthorization.js +6 -1
  114. package/build/dist/Server/Middleware/ProjectAuthorization.js.map +1 -1
  115. package/build/dist/Server/Services/DashboardDomainService.js +104 -0
  116. package/build/dist/Server/Services/DashboardDomainService.js.map +1 -1
  117. package/build/dist/Server/Services/LogPipelineProcessorService.js +65 -0
  118. package/build/dist/Server/Services/LogPipelineProcessorService.js.map +1 -1
  119. package/build/dist/Server/Services/MonitorTemplateService.js +57 -11
  120. package/build/dist/Server/Services/MonitorTemplateService.js.map +1 -1
  121. package/build/dist/Server/Services/StatusPageDomainService.js +132 -3
  122. package/build/dist/Server/Services/StatusPageDomainService.js.map +1 -1
  123. package/build/dist/Server/Services/StatusPagePrivateUserSessionService.js +112 -32
  124. package/build/dist/Server/Services/StatusPagePrivateUserSessionService.js.map +1 -1
  125. package/build/dist/Server/Types/Domain.js +16 -0
  126. package/build/dist/Server/Types/Domain.js.map +1 -1
  127. package/build/dist/Server/Utils/APIKey/AccessPermission.js +24 -0
  128. package/build/dist/Server/Utils/APIKey/AccessPermission.js.map +1 -1
  129. package/build/dist/Server/Utils/Greenlock/Greenlock.js +110 -35
  130. package/build/dist/Server/Utils/Greenlock/Greenlock.js.map +1 -1
  131. package/build/dist/Server/Utils/LogPipelineProcessorValidation.js +77 -0
  132. package/build/dist/Server/Utils/LogPipelineProcessorValidation.js.map +1 -0
  133. package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.js +777 -36
  134. package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.js.map +1 -1
  135. package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.js +108 -17
  136. package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.js.map +1 -1
  137. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.js +17 -6
  138. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.js.map +1 -1
  139. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Auth.js +0 -159
  140. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Auth.js.map +1 -1
  141. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Authorization.js +128 -0
  142. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Authorization.js.map +1 -0
  143. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.js +17 -6
  144. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.js.map +1 -1
  145. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.js +25 -13
  146. package/build/dist/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.js.map +1 -1
  147. package/build/dist/Types/API/Route.js +16 -5
  148. package/build/dist/Types/API/Route.js.map +1 -1
  149. package/build/dist/Types/Date.js +115 -13
  150. package/build/dist/Types/Date.js.map +1 -1
  151. package/build/dist/Types/Rum/SessionReplay.js +38 -0
  152. package/build/dist/Types/Rum/SessionReplay.js.map +1 -1
  153. package/build/dist/UI/Components/Date/TimeRangePickerDropdown.js +9 -5
  154. package/build/dist/UI/Components/Date/TimeRangePickerDropdown.js.map +1 -1
  155. package/build/dist/UI/Components/LogsViewer/LogsViewer.js +12 -7
  156. package/build/dist/UI/Components/LogsViewer/LogsViewer.js.map +1 -1
  157. package/build/dist/UI/Components/LogsViewer/components/HistogramTooltip.js +15 -14
  158. package/build/dist/UI/Components/LogsViewer/components/HistogramTooltip.js.map +1 -1
  159. package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js +17 -22
  160. package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js.map +1 -1
  161. package/build/dist/UI/Components/LogsViewer/components/LogsHistogram.js +2 -4
  162. package/build/dist/UI/Components/LogsViewer/components/LogsHistogram.js.map +1 -1
  163. package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogram.js +2 -4
  164. package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogram.js.map +1 -1
  165. package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogramTooltip.js +15 -14
  166. package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogramTooltip.js.map +1 -1
  167. package/build/dist/UI/Utils/Navigation.js +30 -0
  168. package/build/dist/UI/Utils/Navigation.js.map +1 -1
  169. package/build/dist/Utils/Array.js +25 -0
  170. package/build/dist/Utils/Array.js.map +1 -1
  171. package/build/dist/Utils/CertificateReissue.js +97 -0
  172. package/build/dist/Utils/CertificateReissue.js.map +1 -0
  173. package/build/dist/Utils/Grok/Grok.js +324 -0
  174. package/build/dist/Utils/Grok/Grok.js.map +1 -0
  175. package/build/dist/Utils/Grok/GrokPatterns.js +112 -0
  176. package/build/dist/Utils/Grok/GrokPatterns.js.map +1 -0
  177. package/build/dist/Utils/Monitor/NetworkDeviceMonitorTemplateUtil.js +63 -3
  178. package/build/dist/Utils/Monitor/NetworkDeviceMonitorTemplateUtil.js.map +1 -1
  179. package/package.json +1 -1
  180. package/UI/Utils/JsonWebToken.ts +0 -14
  181. package/build/dist/UI/Utils/JsonWebToken.js +0 -10
  182. package/build/dist/UI/Utils/JsonWebToken.js.map +0 -1
@@ -21,21 +21,39 @@ import ServerException from "../../../Types/Exception/ServerException";
21
21
  import Text from "../../../Types/Text";
22
22
  import AcmeCertificate from "../../../Models/DatabaseModels/AcmeCertificate";
23
23
  import AcmeChallenge from "../../../Models/DatabaseModels/AcmeChallenge";
24
+ import ArrayUtil from "../../../Utils/Array";
24
25
  import acme from "acme-client";
25
26
  import CaptureSpan from "../Telemetry/CaptureSpan";
26
- export default class GreenlockUtil {
27
+ class GreenlockUtil {
28
+ /*
29
+ * Stable per-domain lead time, somewhere in the range above. Same domain,
30
+ * same answer, on every run and every replica.
31
+ */
32
+ static getRenewalLeadTimeInDays(domain) {
33
+ const spanInDays = GreenlockUtil.RENEW_LEAD_TIME_MAX_IN_DAYS -
34
+ GreenlockUtil.RENEW_LEAD_TIME_MIN_IN_DAYS;
35
+ let hash = 0;
36
+ for (let i = 0; i < domain.length; i++) {
37
+ hash = (hash * 31 + domain.charCodeAt(i)) % 1000003;
38
+ }
39
+ return (GreenlockUtil.RENEW_LEAD_TIME_MIN_IN_DAYS + (hash % (spanInDays + 1)));
40
+ }
27
41
  static async renewAllCertsWhichAreExpiringSoon(data) {
28
42
  try {
29
43
  logger.debug("Renewing all certificates");
30
- // get all certificates which are expiring soon
44
+ /*
45
+ * Read the widest window any domain's lead time can make it due in, then
46
+ * keep only the domains whose own lead time has actually been reached.
47
+ */
31
48
  const certificates = await AcmeCertificateService.findBy({
32
49
  query: {
33
- expiresAt: QueryHelper.lessThanEqualTo(OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), 40)),
50
+ expiresAt: QueryHelper.lessThanEqualTo(OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), GreenlockUtil.RENEW_LEAD_TIME_MAX_IN_DAYS)),
34
51
  },
35
52
  limit: LIMIT_MAX,
36
53
  skip: 0,
37
54
  select: {
38
55
  domain: true,
56
+ expiresAt: true,
39
57
  },
40
58
  sort: {
41
59
  expiresAt: SortOrder.Ascending,
@@ -44,40 +62,31 @@ export default class GreenlockUtil {
44
62
  isRoot: true,
45
63
  },
46
64
  });
47
- logger.debug(`Found ${certificates.length} certificates which are expiring soon`, { certificateCount: certificates.length });
48
- // order certificate for each domain
49
- for (const certificate of certificates) {
50
- if (!certificate.domain) {
51
- continue;
65
+ const now = OneUptimeDate.getCurrentDate();
66
+ const dueCertificates = certificates.filter((certificate) => {
67
+ if (!certificate.domain || !certificate.expiresAt) {
68
+ return false;
52
69
  }
53
- const certLogAttributes = {
70
+ const renewAt = OneUptimeDate.addRemoveDays(certificate.expiresAt, -GreenlockUtil.getRenewalLeadTimeInDays(certificate.domain));
71
+ return !OneUptimeDate.isAfter(renewAt, now);
72
+ });
73
+ /*
74
+ * Still sorted by expiry, so a run that cannot take the whole backlog
75
+ * spends itself on the domains closest to expiring and leaves the rest -
76
+ * which by construction still have weeks of lead time - to the next run.
77
+ */
78
+ const batch = dueCertificates.slice(0, GreenlockUtil.RENEW_MAX_PER_RUN);
79
+ logger.debug(`Found ${dueCertificates.length} certificates due for renewal, renewing ${batch.length} in this run`, {
80
+ dueCount: dueCertificates.length,
81
+ batchCount: batch.length,
82
+ });
83
+ await ArrayUtil.forEachWithConcurrency(batch, GreenlockUtil.RENEW_CONCURRENCY, async (certificate) => {
84
+ await GreenlockUtil.renewCertForDomain({
54
85
  domain: certificate.domain,
55
- };
56
- logger.debug(`Renewing certificate for domain: ${certificate.domain}`, certLogAttributes);
57
- try {
58
- //validate cname
59
- const isValidCname = await data.validateCname(certificate.domain);
60
- if (!isValidCname) {
61
- logger.debug(`CNAME is not valid for domain: ${certificate.domain}`, certLogAttributes);
62
- // if cname is not valid then remove the domain
63
- await GreenlockUtil.removeDomain(certificate.domain);
64
- await data.notifyDomainRemoved(certificate.domain);
65
- logger.error(`Cname is not valid for domain: ${certificate.domain}`, certLogAttributes);
66
- }
67
- else {
68
- logger.debug(`CNAME is valid for domain: ${certificate.domain}`, certLogAttributes);
69
- await GreenlockUtil.orderCert({
70
- domain: certificate.domain,
71
- validateCname: data.validateCname,
72
- });
73
- logger.debug(`Certificate renewed for domain: ${certificate.domain}`, certLogAttributes);
74
- }
75
- }
76
- catch (e) {
77
- logger.error(`Error renewing certificate for domain: ${certificate.domain}`, certLogAttributes);
78
- logger.error(e, certLogAttributes);
79
- }
80
- }
86
+ validateCname: data.validateCname,
87
+ notifyDomainRemoved: data.notifyDomainRemoved,
88
+ });
89
+ });
81
90
  }
82
91
  catch (e) {
83
92
  logger.error("Error renewing all certificates");
@@ -85,6 +94,41 @@ export default class GreenlockUtil {
85
94
  throw e;
86
95
  }
87
96
  }
97
+ /*
98
+ * Renew one domain. Never throws: one domain that cannot be renewed - a CNAME
99
+ * that no longer points here, a challenge the CA would not accept - must not
100
+ * take down the rest of the run with it.
101
+ */
102
+ static async renewCertForDomain(data) {
103
+ const { domain } = data;
104
+ const certLogAttributes = {
105
+ domain: domain,
106
+ };
107
+ logger.debug(`Renewing certificate for domain: ${domain}`, certLogAttributes);
108
+ try {
109
+ //validate cname
110
+ const isValidCname = await data.validateCname(domain);
111
+ if (!isValidCname) {
112
+ logger.debug(`CNAME is not valid for domain: ${domain}`, certLogAttributes);
113
+ // if cname is not valid then remove the domain
114
+ await GreenlockUtil.removeDomain(domain);
115
+ await data.notifyDomainRemoved(domain);
116
+ logger.error(`Cname is not valid for domain: ${domain}`, certLogAttributes);
117
+ }
118
+ else {
119
+ logger.debug(`CNAME is valid for domain: ${domain}`, certLogAttributes);
120
+ await GreenlockUtil.orderCert({
121
+ domain: domain,
122
+ validateCname: data.validateCname,
123
+ });
124
+ logger.debug(`Certificate renewed for domain: ${domain}`, certLogAttributes);
125
+ }
126
+ }
127
+ catch (e) {
128
+ logger.error(`Error renewing certificate for domain: ${domain}`, certLogAttributes);
129
+ logger.error(e, certLogAttributes);
130
+ }
131
+ }
88
132
  static async removeDomain(domain) {
89
133
  try {
90
134
  // remove certificate for this domain.
@@ -252,6 +296,37 @@ export default class GreenlockUtil {
252
296
  }
253
297
  }
254
298
  }
299
+ /*
300
+ * How early a certificate is renewed, as a range rather than a single value.
301
+ *
302
+ * Let's Encrypt issues for 90 days, so every domain ordered on the same day
303
+ * expires on the same day. Renewing them all at one fixed lead time keeps
304
+ * that batch together for every cycle that follows, and a single bad day -
305
+ * an upstream outage, a rate limit, a run that does not finish - then expires
306
+ * the whole batch at once instead of one domain.
307
+ *
308
+ * Spreading the lead time across a range pulls the batch apart: domains that
309
+ * expire on the same day come due days apart, and having been renewed days
310
+ * apart they expire days apart next cycle too. The offset is derived from the
311
+ * domain name so it is stable - a domain must not drift in and out of
312
+ * eligibility between two runs, which would leave it renewed by neither.
313
+ */
314
+ GreenlockUtil.RENEW_LEAD_TIME_MAX_IN_DAYS = 40;
315
+ GreenlockUtil.RENEW_LEAD_TIME_MIN_IN_DAYS = 25;
316
+ /*
317
+ * A run renews at most this many domains, at most this many at a time.
318
+ *
319
+ * Both bound how much of a backlog is dispatched at once. Let's Encrypt
320
+ * allows 300 new orders per account per three hours, and the reactive
321
+ * provisioning sweep spends from the same allowance, so a run that tried to
322
+ * clear a large backlog in one pass would spend the account's budget and get
323
+ * the renewals behind it refused. At the schedule this job runs on the cap
324
+ * still clears far more per day than one sequential daily pass ever did,
325
+ * while leaving room under the limit.
326
+ */
327
+ GreenlockUtil.RENEW_MAX_PER_RUN = 10;
328
+ GreenlockUtil.RENEW_CONCURRENCY = 5;
329
+ export default GreenlockUtil;
255
330
  __decorate([
256
331
  CaptureSpan(),
257
332
  __metadata("design:type", Function),
@@ -1 +1 @@
1
- {"version":3,"file":"Greenlock.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/Greenlock.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EACL,gBAAgB,EAChB,qBAAqB,EACrB,4BAA4B,GAC7B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,sBAAsB,MAAM,uCAAuC,CAAC;AAC3E,OAAO,oBAAoB,MAAM,qCAAqC,CAAC;AACvE,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,MAAyB,MAAM,WAAW,CAAC;AAClD,OAAO,SAAS,MAAM,uCAAuC,CAAC;AAC9D,OAAO,SAAS,MAAM,kCAAkC,CAAC;AACzD,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAChD,OAAO,gBAAgB,MAAM,2CAA2C,CAAC;AACzE,OAAO,SAAS,MAAM,oCAAoC,CAAC;AAC3D,OAAO,eAAe,MAAM,0CAA0C,CAAC;AACvE,OAAO,IAAI,MAAM,qBAAqB,CAAC;AACvC,OAAO,eAAe,MAAM,gDAAgD,CAAC;AAC7E,OAAO,aAAa,MAAM,8CAA8C,CAAC;AACzE,OAAO,IAAI,MAAM,aAAa,CAAC;AAE/B,OAAO,WAAW,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,OAAO,OAAO,aAAa;IAEZ,AAAb,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,IAGrD;QACC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAE1C,+CAA+C;YAE/C,MAAM,YAAY,GAChB,MAAM,sBAAsB,CAAC,MAAM,CAAC;gBAClC,KAAK,EAAE;oBACL,SAAS,EAAE,WAAW,CAAC,eAAe,CACpC,aAAa,CAAC,aAAa,CACzB,aAAa,CAAC,cAAc,EAAE,EAC9B,EAAE,CACH,CACF;iBACF;gBACD,KAAK,EAAE,SAAS;gBAChB,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;gBACD,IAAI,EAAE;oBACJ,SAAS,EAAE,SAAS,CAAC,SAAS;iBAC/B;gBACD,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEL,MAAM,CAAC,KAAK,CACV,SAAS,YAAY,CAAC,MAAM,uCAAuC,EACnE,EAAE,gBAAgB,EAAE,YAAY,CAAC,MAAM,EAAE,CAC1C,CAAC;YAEF,oCAAoC;YAEpC,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;gBACvC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;oBACxB,SAAS;gBACX,CAAC;gBAED,MAAM,iBAAiB,GAAkB;oBACvC,MAAM,EAAE,WAAW,CAAC,MAAM;iBAC3B,CAAC;gBAEF,MAAM,CAAC,KAAK,CACV,oCAAoC,WAAW,CAAC,MAAM,EAAE,EACxD,iBAAiB,CAClB,CAAC;gBAEF,IAAI,CAAC;oBACH,gBAAgB;oBAChB,MAAM,YAAY,GAAY,MAAM,IAAI,CAAC,aAAa,CACpD,WAAW,CAAC,MAAM,CACnB,CAAC;oBAEF,IAAI,CAAC,YAAY,EAAE,CAAC;wBAClB,MAAM,CAAC,KAAK,CACV,kCAAkC,WAAW,CAAC,MAAM,EAAE,EACtD,iBAAiB,CAClB,CAAC;wBAEF,+CAA+C;wBAC/C,MAAM,aAAa,CAAC,YAAY,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;wBACrD,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;wBAEnD,MAAM,CAAC,KAAK,CACV,kCAAkC,WAAW,CAAC,MAAM,EAAE,EACtD,iBAAiB,CAClB,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,KAAK,CACV,8BAA8B,WAAW,CAAC,MAAM,EAAE,EAClD,iBAAiB,CAClB,CAAC;wBAEF,MAAM,aAAa,CAAC,SAAS,CAAC;4BAC5B,MAAM,EAAE,WAAW,CAAC,MAAM;4BAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;yBAClC,CAAC,CAAC;wBAEH,MAAM,CAAC,KAAK,CACV,mCAAmC,WAAW,CAAC,MAAM,EAAE,EACvD,iBAAiB,CAClB,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,CAAC,KAAK,CACV,0CAA0C,WAAW,CAAC,MAAM,EAAE,EAC9D,iBAAiB,CAClB,CAAC;oBACF,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YAChD,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEhB,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAGmB,AAAb,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,MAAc;QAC7C,IAAI,CAAC;YACH,sCAAsC;YACtC,MAAM,sBAAsB,CAAC,QAAQ,CAAC;gBACpC,KAAK,EAAE;oBACL,MAAM,EAAE,MAAM;iBACf;gBACD,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,0BAA0B,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7D,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAGmB,AAAb,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAG7B;QACC,MAAM,kBAAkB,GAAkB;YACxC,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CACV,oDAAoD,IAAI,CAAC,MAAM,EAAE,EACjE,kBAAkB,CACnB,CAAC;YAEF,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;YAEtB,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,kBAAkB,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;YAEtC,MAAM,sBAAsB,GAAW,qBAAqB,CAAC;YAE7D,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC5B,MAAM,IAAI,eAAe,CACvB,6EAA6E,CAC9E,CAAC;YACJ,CAAC;YAED,IAAI,cAAc,GAAW,MAAM,CAAC,IAAI,CACtC,sBAAsB,EACtB,QAAQ,CACT,CAAC,QAAQ,EAAE,CAAC;YAEb,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YAE9D,gBAAgB;YAEhB,MAAM,CAAC,KAAK,CACV,gCAAgC,MAAM,EAAE,EACxC,kBAAkB,CACnB,CAAC;YAEF,MAAM,YAAY,GAAY,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE/D,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,CAAC,KAAK,CACV,kCAAkC,MAAM,EAAE,EAC1C,kBAAkB,CACnB,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,oBAAoB,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC;gBAE/D,MAAM,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBACzC,MAAM,CAAC,KAAK,CACV,kCAAkC,MAAM,EAAE,EAC1C,kBAAkB,CACnB,CAAC;gBACF,MAAM,IAAI,gBAAgB,CAAC,gCAAgC,GAAG,MAAM,CAAC,CAAC;YACxE,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,8BAA8B,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC;YAEzE,MAAM,MAAM,GAAgB,IAAI,IAAI,CAAC,MAAM,CAAC;gBAC1C,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,UAAU;gBACnD,UAAU,EAAE,cAAc;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,cAAc,EAAE,kBAAkB,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;gBACvE,UAAU,EAAE,MAAM;aACnB,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CACV,oCAAoC,MAAM,EAAE,EAC5C,kBAAkB,CACnB,CAAC;YAEF,MAAM,WAAW,GAAW,MAAM,MAAM,CAAC,IAAI,CAAC;gBAC5C,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,4BAA4B,CAAC,QAAQ,EAAE;gBAC9C,oBAAoB,EAAE,IAAI;gBAC1B,iBAAiB,EAAE,CAAC,SAAS,CAAC,EAAE,mDAAmD;gBACnF,iBAAiB,EAAE,KAAK,EACtB,KAAyB,EACzB,SAAoB,EACpB,gBAAwB,EACxB,EAAE;oBACF,yBAAyB;oBACzB,aAAa;oBACb,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;wBACjC,MAAM,CAAC,KAAK,CACV,kCAAkC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAC1D,kBAAkB,CACnB,CAAC;wBAEF,MAAM,aAAa,GAAkB,IAAI,aAAa,EAAE,CAAC;wBACzD,aAAa,CAAC,SAAS,GAAG,gBAAgB,CAAC;wBAC3C,aAAa,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;wBACtC,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;wBAE9C,MAAM,oBAAoB,CAAC,MAAM,CAAC;4BAChC,IAAI,EAAE,aAAa;4BACnB,KAAK,EAAE;gCACL,MAAM,EAAE,IAAI;6BACb;yBACF,CAAC,CAAC;wBAEH,MAAM,CAAC,KAAK,CACV,iCAAiC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EACzD,kBAAkB,CACnB,CAAC;oBACJ,CAAC;gBACH,CAAC;gBACD,iBAAiB,EAAE,KAAK,EACtB,KAAyB,EACzB,SAAoB,EACpB,EAAE;oBACF,0BAA0B;oBAE1B,MAAM,CAAC,KAAK,CACV,kCAAkC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAC1D,kBAAkB,CACnB,CAAC;oBAEF,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;wBACjC,MAAM,oBAAoB,CAAC,QAAQ,CAAC;4BAClC,KAAK,EAAE;gCACL,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK;6BAC/B;4BACD,KAAK,EAAE,CAAC;4BACR,IAAI,EAAE,CAAC;4BACP,KAAK,EAAE;gCACL,MAAM,EAAE,IAAI;6BACb;yBACF,CAAC,CAAC;oBACL,CAAC;oBAED,MAAM,CAAC,KAAK,CACV,iCAAiC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EACzD,kBAAkB,CACnB,CAAC;gBACJ,CAAC;aACF,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CACV,mCAAmC,MAAM,EAAE,EAC3C,kBAAkB,CACnB,CAAC;YAEF,uCAAuC;YACvC,MAAM,IAAI,GACR,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAS,IAAI,CAAC,SAAS,CAAC;YACtC,MAAM,SAAS,GAAS,IAAI,CAAC,QAAQ,CAAC;YAEtC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,EAAE,EAAE,kBAAkB,CAAC,CAAC;YACzE,MAAM,CAAC,KAAK,CAAC,0BAA0B,QAAQ,EAAE,EAAE,kBAAkB,CAAC,CAAC;YAEvE,uDAAuD;YACvD,MAAM,mBAAmB,GACvB,MAAM,sBAAsB,CAAC,SAAS,CAAC;gBACrC,KAAK,EAAE;oBACL,MAAM,EAAE,MAAM;iBACf;gBACD,MAAM,EAAE;oBACN,GAAG,EAAE,IAAI;iBACV;gBACD,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEL,IAAI,mBAAmB,EAAE,CAAC;gBACxB,MAAM,CAAC,KAAK,CACV,oCAAoC,MAAM,EAAE,EAC5C,kBAAkB,CACnB,CAAC;gBAEF,yBAAyB;gBACzB,MAAM,sBAAsB,CAAC,QAAQ,CAAC;oBACpC,KAAK,EAAE;wBACL,MAAM,EAAE,MAAM;qBACf;oBACD,KAAK,EAAE,CAAC;oBACR,IAAI,EAAE,CAAC;oBACP,IAAI,EAAE;wBACJ,WAAW,EAAE,WAAW,CAAC,QAAQ,EAAE;wBACnC,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;wBACzC,QAAQ,EAAE,QAAQ;wBAClB,SAAS,EAAE,SAAS;qBACrB;oBACD,KAAK,EAAE;wBACL,MAAM,EAAE,IAAI;qBACb;iBACF,CAAC,CAAC;gBAEH,MAAM,CAAC,KAAK,CACV,mCAAmC,MAAM,EAAE,EAC3C,kBAAkB,CACnB,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CACV,oCAAoC,MAAM,EAAE,EAC5C,kBAAkB,CACnB,CAAC;gBACF,yBAAyB;gBACzB,MAAM,eAAe,GAAoB,IAAI,eAAe,EAAE,CAAC;gBAE/D,eAAe,CAAC,MAAM,GAAG,MAAM,CAAC;gBAChC,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC,QAAQ,EAAE,CAAC;gBACrD,eAAe,CAAC,cAAc,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC;gBAC3D,eAAe,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBACpC,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;gBAEtC,MAAM,sBAAsB,CAAC,MAAM,CAAC;oBAClC,IAAI,EAAE,eAAe;oBACrB,KAAK,EAAE;wBACL,MAAM,EAAE,IAAI;qBACb;iBACF,CAAC,CAAC;gBAEH,MAAM,CAAC,KAAK,CACV,mCAAmC,MAAM,EAAE,EAC3C,kBAAkB,CACnB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CACV,0CAA0C,IAAI,CAAC,MAAM,EAAE,EACvD,kBAAkB,CACnB,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAEpC,IAAI,CAAC,YAAY,SAAS,EAAE,CAAC;gBAC3B,MAAM,CAAC,CAAC;YACV,CAAC;YAED,IAAI,gBAAgB,EAAE,CAAC;gBACrB,MAAM,IAAI,eAAe,CACvB,mCAAmC,IAAI,CAAC,MAAM,yEAAyE,CACxH,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,eAAe,CACvB,mCAAmC,IAAI,CAAC,MAAM,wPAAwP,CACvS,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAnXqB;IADnB,WAAW,EAAE;;;;4DAwGb;AAGmB;IADnB,WAAW,EAAE;;;;uCAkBb;AAGmB;IADnB,WAAW,EAAE;;;;oCAqPb"}
1
+ {"version":3,"file":"Greenlock.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/Greenlock.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EACL,gBAAgB,EAChB,qBAAqB,EACrB,4BAA4B,GAC7B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,sBAAsB,MAAM,uCAAuC,CAAC;AAC3E,OAAO,oBAAoB,MAAM,qCAAqC,CAAC;AACvE,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,MAAyB,MAAM,WAAW,CAAC;AAClD,OAAO,SAAS,MAAM,uCAAuC,CAAC;AAC9D,OAAO,SAAS,MAAM,kCAAkC,CAAC;AACzD,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAChD,OAAO,gBAAgB,MAAM,2CAA2C,CAAC;AACzE,OAAO,SAAS,MAAM,oCAAoC,CAAC;AAC3D,OAAO,eAAe,MAAM,0CAA0C,CAAC;AACvE,OAAO,IAAI,MAAM,qBAAqB,CAAC;AACvC,OAAO,eAAe,MAAM,gDAAgD,CAAC;AAC7E,OAAO,aAAa,MAAM,8CAA8C,CAAC;AACzE,OAAO,SAAS,MAAM,sBAAsB,CAAC;AAC7C,OAAO,IAAI,MAAM,aAAa,CAAC;AAE/B,OAAO,WAAW,MAAM,0BAA0B,CAAC;AAEnD,MAAqB,aAAa;IAiChC;;;OAGG;IACI,MAAM,CAAC,wBAAwB,CAAC,MAAc;QACnD,MAAM,UAAU,GACd,aAAa,CAAC,2BAA2B;YACzC,aAAa,CAAC,2BAA2B,CAAC;QAE5C,IAAI,IAAI,GAAW,CAAC,CAAC;QAErB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,IAAI,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QACtD,CAAC;QAED,OAAO,CACL,aAAa,CAAC,2BAA2B,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CACtE,CAAC;IACJ,CAAC;IAGmB,AAAb,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,IAGrD;QACC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAE1C;;;eAGG;YACH,MAAM,YAAY,GAChB,MAAM,sBAAsB,CAAC,MAAM,CAAC;gBAClC,KAAK,EAAE;oBACL,SAAS,EAAE,WAAW,CAAC,eAAe,CACpC,aAAa,CAAC,aAAa,CACzB,aAAa,CAAC,cAAc,EAAE,EAC9B,aAAa,CAAC,2BAA2B,CAC1C,CACF;iBACF;gBACD,KAAK,EAAE,SAAS;gBAChB,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,SAAS,EAAE,IAAI;iBAChB;gBACD,IAAI,EAAE;oBACJ,SAAS,EAAE,SAAS,CAAC,SAAS;iBAC/B;gBACD,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEL,MAAM,GAAG,GAAS,aAAa,CAAC,cAAc,EAAE,CAAC;YAEjD,MAAM,eAAe,GAAsB,YAAY,CAAC,MAAM,CAC5D,CAAC,WAA4B,EAAE,EAAE;gBAC/B,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;oBAClD,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,MAAM,OAAO,GAAS,aAAa,CAAC,aAAa,CAC/C,WAAW,CAAC,SAAS,EACrB,CAAC,aAAa,CAAC,wBAAwB,CAAC,WAAW,CAAC,MAAM,CAAC,CAC5D,CAAC;gBAEF,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC9C,CAAC,CACF,CAAC;YAEF;;;;eAIG;YACH,MAAM,KAAK,GAAsB,eAAe,CAAC,KAAK,CACpD,CAAC,EACD,aAAa,CAAC,iBAAiB,CAChC,CAAC;YAEF,MAAM,CAAC,KAAK,CACV,SAAS,eAAe,CAAC,MAAM,2CAA2C,KAAK,CAAC,MAAM,cAAc,EACpG;gBACE,QAAQ,EAAE,eAAe,CAAC,MAAM;gBAChC,UAAU,EAAE,KAAK,CAAC,MAAM;aACzB,CACF,CAAC;YAEF,MAAM,SAAS,CAAC,sBAAsB,CACpC,KAAK,EACL,aAAa,CAAC,iBAAiB,EAC/B,KAAK,EAAE,WAA4B,EAAiB,EAAE;gBACpD,MAAM,aAAa,CAAC,kBAAkB,CAAC;oBACrC,MAAM,EAAE,WAAW,CAAC,MAAgB;oBACpC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;iBAC9C,CAAC,CAAC;YACL,CAAC,CACF,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YAChD,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEhB,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAIvC;QACC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAExB,MAAM,iBAAiB,GAAkB;YACvC,MAAM,EAAE,MAAM;SACf,CAAC;QAEF,MAAM,CAAC,KAAK,CACV,oCAAoC,MAAM,EAAE,EAC5C,iBAAiB,CAClB,CAAC;QAEF,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,YAAY,GAAY,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE/D,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,CAAC,KAAK,CACV,kCAAkC,MAAM,EAAE,EAC1C,iBAAiB,CAClB,CAAC;gBAEF,+CAA+C;gBAC/C,MAAM,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBACzC,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAEvC,MAAM,CAAC,KAAK,CACV,kCAAkC,MAAM,EAAE,EAC1C,iBAAiB,CAClB,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,8BAA8B,MAAM,EAAE,EAAE,iBAAiB,CAAC,CAAC;gBAExE,MAAM,aAAa,CAAC,SAAS,CAAC;oBAC5B,MAAM,EAAE,MAAM;oBACd,aAAa,EAAE,IAAI,CAAC,aAAa;iBAClC,CAAC,CAAC;gBAEH,MAAM,CAAC,KAAK,CACV,mCAAmC,MAAM,EAAE,EAC3C,iBAAiB,CAClB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CACV,0CAA0C,MAAM,EAAE,EAClD,iBAAiB,CAClB,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAGmB,AAAb,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,MAAc;QAC7C,IAAI,CAAC;YACH,sCAAsC;YACtC,MAAM,sBAAsB,CAAC,QAAQ,CAAC;gBACpC,KAAK,EAAE;oBACL,MAAM,EAAE,MAAM;iBACf;gBACD,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,0BAA0B,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7D,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAGmB,AAAb,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAG7B;QACC,MAAM,kBAAkB,GAAkB;YACxC,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CACV,oDAAoD,IAAI,CAAC,MAAM,EAAE,EACjE,kBAAkB,CACnB,CAAC;YAEF,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;YAEtB,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,kBAAkB,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;YAEtC,MAAM,sBAAsB,GAAW,qBAAqB,CAAC;YAE7D,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC5B,MAAM,IAAI,eAAe,CACvB,6EAA6E,CAC9E,CAAC;YACJ,CAAC;YAED,IAAI,cAAc,GAAW,MAAM,CAAC,IAAI,CACtC,sBAAsB,EACtB,QAAQ,CACT,CAAC,QAAQ,EAAE,CAAC;YAEb,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YAE9D,gBAAgB;YAEhB,MAAM,CAAC,KAAK,CACV,gCAAgC,MAAM,EAAE,EACxC,kBAAkB,CACnB,CAAC;YAEF,MAAM,YAAY,GAAY,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE/D,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,CAAC,KAAK,CACV,kCAAkC,MAAM,EAAE,EAC1C,kBAAkB,CACnB,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,oBAAoB,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC;gBAE/D,MAAM,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBACzC,MAAM,CAAC,KAAK,CACV,kCAAkC,MAAM,EAAE,EAC1C,kBAAkB,CACnB,CAAC;gBACF,MAAM,IAAI,gBAAgB,CAAC,gCAAgC,GAAG,MAAM,CAAC,CAAC;YACxE,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,8BAA8B,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC;YAEzE,MAAM,MAAM,GAAgB,IAAI,IAAI,CAAC,MAAM,CAAC;gBAC1C,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,UAAU;gBACnD,UAAU,EAAE,cAAc;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,cAAc,EAAE,kBAAkB,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;gBACvE,UAAU,EAAE,MAAM;aACnB,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CACV,oCAAoC,MAAM,EAAE,EAC5C,kBAAkB,CACnB,CAAC;YAEF,MAAM,WAAW,GAAW,MAAM,MAAM,CAAC,IAAI,CAAC;gBAC5C,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,4BAA4B,CAAC,QAAQ,EAAE;gBAC9C,oBAAoB,EAAE,IAAI;gBAC1B,iBAAiB,EAAE,CAAC,SAAS,CAAC,EAAE,mDAAmD;gBACnF,iBAAiB,EAAE,KAAK,EACtB,KAAyB,EACzB,SAAoB,EACpB,gBAAwB,EACxB,EAAE;oBACF,yBAAyB;oBACzB,aAAa;oBACb,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;wBACjC,MAAM,CAAC,KAAK,CACV,kCAAkC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAC1D,kBAAkB,CACnB,CAAC;wBAEF,MAAM,aAAa,GAAkB,IAAI,aAAa,EAAE,CAAC;wBACzD,aAAa,CAAC,SAAS,GAAG,gBAAgB,CAAC;wBAC3C,aAAa,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;wBACtC,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;wBAE9C,MAAM,oBAAoB,CAAC,MAAM,CAAC;4BAChC,IAAI,EAAE,aAAa;4BACnB,KAAK,EAAE;gCACL,MAAM,EAAE,IAAI;6BACb;yBACF,CAAC,CAAC;wBAEH,MAAM,CAAC,KAAK,CACV,iCAAiC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EACzD,kBAAkB,CACnB,CAAC;oBACJ,CAAC;gBACH,CAAC;gBACD,iBAAiB,EAAE,KAAK,EACtB,KAAyB,EACzB,SAAoB,EACpB,EAAE;oBACF,0BAA0B;oBAE1B,MAAM,CAAC,KAAK,CACV,kCAAkC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAC1D,kBAAkB,CACnB,CAAC;oBAEF,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;wBACjC,MAAM,oBAAoB,CAAC,QAAQ,CAAC;4BAClC,KAAK,EAAE;gCACL,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK;6BAC/B;4BACD,KAAK,EAAE,CAAC;4BACR,IAAI,EAAE,CAAC;4BACP,KAAK,EAAE;gCACL,MAAM,EAAE,IAAI;6BACb;yBACF,CAAC,CAAC;oBACL,CAAC;oBAED,MAAM,CAAC,KAAK,CACV,iCAAiC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EACzD,kBAAkB,CACnB,CAAC;gBACJ,CAAC;aACF,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CACV,mCAAmC,MAAM,EAAE,EAC3C,kBAAkB,CACnB,CAAC;YAEF,uCAAuC;YACvC,MAAM,IAAI,GACR,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAS,IAAI,CAAC,SAAS,CAAC;YACtC,MAAM,SAAS,GAAS,IAAI,CAAC,QAAQ,CAAC;YAEtC,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,EAAE,EAAE,kBAAkB,CAAC,CAAC;YACzE,MAAM,CAAC,KAAK,CAAC,0BAA0B,QAAQ,EAAE,EAAE,kBAAkB,CAAC,CAAC;YAEvE,uDAAuD;YACvD,MAAM,mBAAmB,GACvB,MAAM,sBAAsB,CAAC,SAAS,CAAC;gBACrC,KAAK,EAAE;oBACL,MAAM,EAAE,MAAM;iBACf;gBACD,MAAM,EAAE;oBACN,GAAG,EAAE,IAAI;iBACV;gBACD,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEL,IAAI,mBAAmB,EAAE,CAAC;gBACxB,MAAM,CAAC,KAAK,CACV,oCAAoC,MAAM,EAAE,EAC5C,kBAAkB,CACnB,CAAC;gBAEF,yBAAyB;gBACzB,MAAM,sBAAsB,CAAC,QAAQ,CAAC;oBACpC,KAAK,EAAE;wBACL,MAAM,EAAE,MAAM;qBACf;oBACD,KAAK,EAAE,CAAC;oBACR,IAAI,EAAE,CAAC;oBACP,IAAI,EAAE;wBACJ,WAAW,EAAE,WAAW,CAAC,QAAQ,EAAE;wBACnC,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;wBACzC,QAAQ,EAAE,QAAQ;wBAClB,SAAS,EAAE,SAAS;qBACrB;oBACD,KAAK,EAAE;wBACL,MAAM,EAAE,IAAI;qBACb;iBACF,CAAC,CAAC;gBAEH,MAAM,CAAC,KAAK,CACV,mCAAmC,MAAM,EAAE,EAC3C,kBAAkB,CACnB,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CACV,oCAAoC,MAAM,EAAE,EAC5C,kBAAkB,CACnB,CAAC;gBACF,yBAAyB;gBACzB,MAAM,eAAe,GAAoB,IAAI,eAAe,EAAE,CAAC;gBAE/D,eAAe,CAAC,MAAM,GAAG,MAAM,CAAC;gBAChC,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC,QAAQ,EAAE,CAAC;gBACrD,eAAe,CAAC,cAAc,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC;gBAC3D,eAAe,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBACpC,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;gBAEtC,MAAM,sBAAsB,CAAC,MAAM,CAAC;oBAClC,IAAI,EAAE,eAAe;oBACrB,KAAK,EAAE;wBACL,MAAM,EAAE,IAAI;qBACb;iBACF,CAAC,CAAC;gBAEH,MAAM,CAAC,KAAK,CACV,mCAAmC,MAAM,EAAE,EAC3C,kBAAkB,CACnB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CACV,0CAA0C,IAAI,CAAC,MAAM,EAAE,EACvD,kBAAkB,CACnB,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAEpC,IAAI,CAAC,YAAY,SAAS,EAAE,CAAC;gBAC3B,MAAM,CAAC,CAAC;YACV,CAAC;YAED,IAAI,gBAAgB,EAAE,CAAC;gBACrB,MAAM,IAAI,eAAe,CACvB,mCAAmC,IAAI,CAAC,MAAM,yEAAyE,CACxH,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,eAAe,CACvB,mCAAmC,IAAI,CAAC,MAAM,wPAAwP,CACvS,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;;AApdD;;;;;;;;;;;;;;GAcG;AACoB,yCAA2B,GAAW,EAAE,CAAC;AACzC,yCAA2B,GAAW,EAAE,CAAC;AAEhE;;;;;;;;;;GAUG;AACoB,+BAAiB,GAAW,EAAE,CAAC;AAC/B,+BAAiB,GAAW,CAAC,CAAC;eA/BlC,aAAa;AAsDZ;IADnB,WAAW,EAAE;;;;4DAwFb;AAgEmB;IADnB,WAAW,EAAE;;;;uCAkBb;AAGmB;IADnB,WAAW,EAAE;;;;oCAqPb"}
@@ -0,0 +1,77 @@
1
+ import BadDataException from "../../Types/Exception/BadDataException";
2
+ import LogPipelineProcessorType from "../../Types/Log/LogPipelineProcessorType";
3
+ import { compileGrokPattern } from "../../Utils/Grok/Grok";
4
+ /*
5
+ * Save-time validation for log pipeline processors.
6
+ *
7
+ * A processor that cannot do anything is worse than a rejected one: it
8
+ * sits in the pipeline list looking configured while every log flows
9
+ * past it untouched, and nothing surfaces the fact. That is exactly how
10
+ * GrokParser behaved before it was implemented
11
+ * (OneUptime/oneuptime#2515).
12
+ *
13
+ * The grok pattern is the one field where a typo is both easy and
14
+ * invisible - `%{IPV4:client ip}` or a stray bracket compiles to
15
+ * nothing. Compiling it here is cheap, happens once per human edit, and
16
+ * puts the error in front of the only person who can fix it. The ingest
17
+ * path stays defensive anyway: rows saved before this validation
18
+ * existed still fail closed to "leave the log alone".
19
+ */
20
+ /*
21
+ * Prefixes become the leading part of an attribute key, so hold them to
22
+ * what an attribute key can sensibly be.
23
+ */
24
+ const TARGET_PREFIX_REGEX = /^[A-Za-z_][A-Za-z0-9_.@:-]*$/;
25
+ /*
26
+ * `configuration` is a jsonb column, but the dashboard's JSON form field
27
+ * has historically persisted it as a JSON string literal - so both
28
+ * shapes reach here (see LogPipelineService.normalizeProcessorConfig).
29
+ */
30
+ function readConfiguration(configuration) {
31
+ if (configuration && typeof configuration === "object") {
32
+ return configuration;
33
+ }
34
+ if (typeof configuration === "string") {
35
+ try {
36
+ const parsed = JSON.parse(configuration);
37
+ if (parsed && typeof parsed === "object") {
38
+ return parsed;
39
+ }
40
+ }
41
+ catch (_a) {
42
+ throw new BadDataException("Processor configuration is not valid JSON.");
43
+ }
44
+ }
45
+ return null;
46
+ }
47
+ export function validateLogPipelineProcessor(candidate) {
48
+ if (candidate.processorType !== LogPipelineProcessorType.GrokParser) {
49
+ return;
50
+ }
51
+ const configuration = readConfiguration(candidate.configuration);
52
+ if (!configuration) {
53
+ throw new BadDataException("A Grok Parser processor needs a configuration with a grok pattern.");
54
+ }
55
+ const config = configuration;
56
+ const pattern = typeof config.pattern === "string" ? config.pattern.trim() : "";
57
+ if (!pattern) {
58
+ throw new BadDataException("A Grok Parser processor needs a grok pattern, for example: %{IPV4:client_ip} %{WORD:verb}");
59
+ }
60
+ // Throws BadDataException with a message written for the person editing.
61
+ compileGrokPattern(pattern);
62
+ if (config.targetPrefix !== undefined && config.targetPrefix !== null) {
63
+ if (typeof config.targetPrefix !== "string") {
64
+ throw new BadDataException("Target prefix must be text.");
65
+ }
66
+ const targetPrefix = config.targetPrefix.trim();
67
+ if (targetPrefix && !TARGET_PREFIX_REGEX.test(targetPrefix)) {
68
+ throw new BadDataException(`"${config.targetPrefix}" is not a valid target prefix. Use letters, digits, and . _ - : @ (starting with a letter or underscore).`);
69
+ }
70
+ }
71
+ if (config.source !== undefined && config.source !== null) {
72
+ if (typeof config.source !== "string") {
73
+ throw new BadDataException("Source field must be text.");
74
+ }
75
+ }
76
+ }
77
+ //# sourceMappingURL=LogPipelineProcessorValidation.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LogPipelineProcessorValidation.js","sourceRoot":"","sources":["../../../../Server/Utils/LogPipelineProcessorValidation.ts"],"names":[],"mappings":"AAAA,OAAO,gBAAgB,MAAM,wCAAwC,CAAC;AAEtE,OAAO,wBAEN,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D;;;;;;;;;;;;;;;GAeG;AAEH;;;GAGG;AACH,MAAM,mBAAmB,GAAW,8BAA8B,CAAC;AAOnE;;;;GAIG;AACH,SAAS,iBAAiB,CACxB,aAAqD;IAErD,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACvD,OAAO,aAA2B,CAAC;IACrC,CAAC;IAED,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAElD,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACzC,OAAO,MAAoB,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,WAAM,CAAC;YACP,MAAM,IAAI,gBAAgB,CAAC,4CAA4C,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,SAAwC;IAExC,IAAI,SAAS,CAAC,aAAa,KAAK,wBAAwB,CAAC,UAAU,EAAE,CAAC;QACpE,OAAO;IACT,CAAC;IAED,MAAM,aAAa,GAAsB,iBAAiB,CACxD,SAAS,CAAC,aAAa,CACxB,CAAC;IAEF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,gBAAgB,CACxB,oEAAoE,CACrE,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAqB,aAA4C,CAAC;IAC9E,MAAM,OAAO,GACX,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAElE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,gBAAgB,CACxB,2FAA2F,CAC5F,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAE5B,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QACtE,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAC5C,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,YAAY,GAAW,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAExD,IAAI,YAAY,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,gBAAgB,CACxB,IAAI,MAAM,CAAC,YAAY,4GAA4G,CACpI,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QAC1D,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,MAAM,IAAI,gBAAgB,CAAC,4BAA4B,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;AACH,CAAC"}