@oneuptime/common 12.0.27 → 12.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/RumSessionChunk.ts +52 -0
- package/Models/DatabaseModels/CloudResource.ts +17 -1
- package/Models/DatabaseModels/DashboardDomain.ts +43 -0
- package/Models/DatabaseModels/MonitorTemplate.ts +15 -3
- package/Models/DatabaseModels/RumApplication.ts +35 -2
- package/Models/DatabaseModels/StatusPageDomain.ts +43 -0
- package/Server/API/DashboardDomainAPI.ts +74 -0
- package/Server/API/MicrosoftTeamsAPI.ts +81 -180
- package/Server/API/StatusPageDomainAPI.ts +74 -0
- package/Server/API/TelemetryAPI.ts +104 -15
- package/Server/Infrastructure/Postgres/SchemaMigrations/1790200000000-AllowNullMonitorNameOnMonitorTemplate.ts +43 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1790300000000-AddCertificateReissueRequestedAtToDomains.ts +40 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +4 -0
- package/Server/Middleware/ProjectAuthorization.ts +6 -1
- package/Server/Services/DashboardDomainService.ts +131 -0
- package/Server/Services/LogPipelineProcessorService.ts +88 -0
- package/Server/Services/MonitorTemplateService.ts +70 -11
- package/Server/Services/StatusPageDomainService.ts +163 -3
- package/Server/Services/StatusPagePrivateUserSessionService.ts +197 -56
- package/Server/Types/Domain.ts +17 -0
- package/Server/Utils/APIKey/AccessPermission.ts +24 -0
- package/Server/Utils/Greenlock/Greenlock.ts +156 -58
- package/Server/Utils/LogPipelineProcessorValidation.ts +112 -0
- package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.ts +1094 -45
- package/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.ts +153 -17
- package/Server/Utils/SessionReplay/SessionReplayGateCache.ts +12 -5
- package/Server/Utils/SessionReplay/SessionReplayIdentity.ts +102 -0
- package/Server/Utils/SessionReplay/SessionReplayReadService.ts +11 -2
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.ts +23 -11
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/Auth.ts +0 -170
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/Authorization.ts +158 -0
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.ts +23 -11
- package/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.ts +35 -13
- package/Tests/App/Dashboard/MonitorTemplateColumnAndFacet.test.tsx +505 -0
- package/Tests/App/StatusPage/StatusPageLoginCodeBootstrap.test.ts +106 -0
- package/Tests/App/StatusPage/StatusPageLoginCodeUtil.test.ts +84 -0
- package/Tests/App/StatusPage/StatusPageRedirectSafety.test.ts +119 -0
- package/Tests/Models/DatabaseModels/SessionReplayModels.test.ts +81 -0
- package/Tests/Models/MonitorTemplateMonitorNameColumn.test.ts +258 -0
- package/Tests/Server/API/CustomDomainReissueSslAPI.test.ts +469 -0
- package/Tests/Server/API/MicrosoftTeamsBotMessagesEndpoint.test.ts +729 -0
- package/Tests/Server/API/MicrosoftTeamsBotTestEndpoint.test.ts +26 -0
- package/Tests/Server/API/MicrosoftTeamsLegacyWebhookRemoval.test.ts +402 -0
- package/Tests/Server/API/SessionReplayAPI.test.ts +232 -0
- package/Tests/Server/Middleware/ProjectAuthorizationApiKeyMiddleware.test.ts +4 -0
- package/Tests/Server/Services/AddCertificateReissueRequestedAtToDomainsMigration.test.ts +287 -0
- package/Tests/Server/Services/CustomDomainCertificateReissue.test.ts +477 -0
- package/Tests/Server/Services/LogPipelineProcessorSaveValidation.test.ts +249 -0
- package/Tests/Server/Services/MonitorTemplateServiceBulkSync.test.ts +549 -0
- package/Tests/Server/Services/MonitorTemplateServiceNetworkDeviceSync.test.ts +16 -3
- package/Tests/Server/Services/NetworkDeviceAutoImportRuleEngineService.test.ts +57 -0
- package/Tests/Server/Services/SecurityEventLastErrorColumnWidth.test.ts +151 -15
- package/Tests/Server/Services/StatusPagePrivateUserSessionService.test.ts +587 -0
- package/Tests/Server/Types/Database/Permissions/ApiKeyFileAccess.test.ts +286 -0
- package/Tests/Server/Types/Database/Permissions/MonitorTemplateColumnAccess.test.ts +328 -0
- package/Tests/Server/Types/Database/QueryHelperFindWithSameTextAndIsNull.test.ts +83 -0
- package/Tests/Server/Utils/AI/SRE/ConfidenceSignal.test.ts +377 -0
- package/Tests/Server/Utils/APIKey/AccessPermission.test.ts +16 -1
- package/Tests/Server/Utils/Greenlock/CertificateRenewalSchedule.test.ts +250 -0
- package/Tests/Server/Utils/LogPipelineProcessorValidation.test.ts +198 -0
- package/Tests/Server/Utils/SecurityEvent/ConnectorErrorMessage.test.ts +48 -9
- package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsAuth.test.ts +726 -0
- package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsClient.test.ts +74 -11
- package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsErrorHandling.test.ts +1107 -0
- package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsPoller.test.ts +20 -3
- package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsPollerFailureTaxonomy.test.ts +51 -11
- package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsPollerHardening.test.ts +1246 -0
- package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsRequestContract.test.ts +961 -0
- package/Tests/Server/Utils/SecurityEvent/GoogleSecOpsResponseParsing.test.ts +653 -0
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsActionAuthorization.test.ts +880 -0
- package/Tests/Types/API/HTTPResponse.test.ts +175 -0
- package/Tests/Types/API/Route.test.ts +36 -4
- package/Tests/Types/Database/DatabaseProperty.test.ts +171 -0
- package/Tests/Types/DateLocalShortDateTimeString.test.ts +518 -0
- package/Tests/Types/DateLocalTimeString.test.ts +259 -0
- package/Tests/Types/DateUserPrefers12HourFormat.test.ts +316 -0
- package/Tests/UI/Components/LogsHistogram.test.tsx +41 -2
- package/Tests/UI/Components/LogsViewerEmptyStateTimeRange.test.tsx +212 -0
- package/Tests/UI/Components/TelemetryChartClockFormat.test.tsx +359 -0
- package/Tests/UI/Components/TimeRangePickerClockFormat.test.tsx +422 -0
- package/Tests/UI/Components/TimeRangePickerDropdown.test.tsx +146 -8
- package/Tests/UI/MicrosoftTeams/MicrosoftTeamsMessagingEndpointGuidance.test.tsx +520 -0
- package/Tests/UI/Utils/Navigation.test.ts +82 -0
- package/Tests/Utils/Array.test.ts +100 -0
- package/Tests/Utils/CertificateReissue.test.ts +225 -0
- package/Tests/Utils/Grok/Grok.test.ts +492 -0
- package/Tests/Utils/Monitor/NetworkDeviceMonitorTemplateUtil.test.ts +181 -13
- package/Types/API/Route.ts +20 -5
- package/Types/Date.ts +143 -13
- package/Types/Rum/SessionReplay.ts +76 -1
- package/UI/Components/Date/TimeRangePickerDropdown.tsx +9 -5
- package/UI/Components/LogsViewer/LogsViewer.tsx +12 -7
- package/UI/Components/LogsViewer/components/HistogramTooltip.tsx +18 -17
- package/UI/Components/LogsViewer/components/LogsAnalyticsView.tsx +21 -28
- package/UI/Components/LogsViewer/components/LogsHistogram.tsx +2 -4
- package/UI/Components/TelemetryViewer/components/TelemetryHistogram.tsx +2 -4
- package/UI/Components/TelemetryViewer/components/TelemetryHistogramTooltip.tsx +17 -16
- package/UI/Utils/Navigation.ts +42 -0
- package/Utils/Array.ts +37 -0
- package/Utils/CertificateReissue.ts +128 -0
- package/Utils/Grok/Grok.ts +461 -0
- package/Utils/Grok/GrokPatterns.ts +118 -0
- package/Utils/Monitor/NetworkDeviceMonitorTemplateUtil.ts +58 -6
- package/build/dist/Models/AnalyticsModels/RumSessionChunk.js +48 -0
- package/build/dist/Models/AnalyticsModels/RumSessionChunk.js.map +1 -1
- package/build/dist/Models/DatabaseModels/CloudResource.js +19 -2
- package/build/dist/Models/DatabaseModels/CloudResource.js.map +1 -1
- package/build/dist/Models/DatabaseModels/DashboardDomain.js +44 -0
- package/build/dist/Models/DatabaseModels/DashboardDomain.js.map +1 -1
- package/build/dist/Models/DatabaseModels/MonitorTemplate.js +17 -4
- package/build/dist/Models/DatabaseModels/MonitorTemplate.js.map +1 -1
- package/build/dist/Models/DatabaseModels/RumApplication.js +37 -3
- package/build/dist/Models/DatabaseModels/RumApplication.js.map +1 -1
- package/build/dist/Models/DatabaseModels/StatusPageDomain.js +44 -0
- package/build/dist/Models/DatabaseModels/StatusPageDomain.js.map +1 -1
- package/build/dist/Server/API/DashboardDomainAPI.js +43 -1
- package/build/dist/Server/API/DashboardDomainAPI.js.map +1 -1
- package/build/dist/Server/API/MicrosoftTeamsAPI.js +74 -118
- package/build/dist/Server/API/MicrosoftTeamsAPI.js.map +1 -1
- package/build/dist/Server/API/StatusPageDomainAPI.js +43 -1
- package/build/dist/Server/API/StatusPageDomainAPI.js.map +1 -1
- package/build/dist/Server/API/TelemetryAPI.js +67 -14
- package/build/dist/Server/API/TelemetryAPI.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1790200000000-AllowNullMonitorNameOnMonitorTemplate.js +34 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1790200000000-AllowNullMonitorNameOnMonitorTemplate.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1790300000000-AddCertificateReissueRequestedAtToDomains.js +28 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1790300000000-AddCertificateReissueRequestedAtToDomains.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +4 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Middleware/ProjectAuthorization.js +6 -1
- package/build/dist/Server/Middleware/ProjectAuthorization.js.map +1 -1
- package/build/dist/Server/Services/DashboardDomainService.js +104 -0
- package/build/dist/Server/Services/DashboardDomainService.js.map +1 -1
- package/build/dist/Server/Services/LogPipelineProcessorService.js +65 -0
- package/build/dist/Server/Services/LogPipelineProcessorService.js.map +1 -1
- package/build/dist/Server/Services/MonitorTemplateService.js +57 -11
- package/build/dist/Server/Services/MonitorTemplateService.js.map +1 -1
- package/build/dist/Server/Services/StatusPageDomainService.js +132 -3
- package/build/dist/Server/Services/StatusPageDomainService.js.map +1 -1
- package/build/dist/Server/Services/StatusPagePrivateUserSessionService.js +112 -32
- package/build/dist/Server/Services/StatusPagePrivateUserSessionService.js.map +1 -1
- package/build/dist/Server/Types/Domain.js +16 -0
- package/build/dist/Server/Types/Domain.js.map +1 -1
- package/build/dist/Server/Utils/APIKey/AccessPermission.js +24 -0
- package/build/dist/Server/Utils/APIKey/AccessPermission.js.map +1 -1
- package/build/dist/Server/Utils/Greenlock/Greenlock.js +110 -35
- package/build/dist/Server/Utils/Greenlock/Greenlock.js.map +1 -1
- package/build/dist/Server/Utils/LogPipelineProcessorValidation.js +77 -0
- package/build/dist/Server/Utils/LogPipelineProcessorValidation.js.map +1 -0
- package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.js +777 -36
- package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsClient.js.map +1 -1
- package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.js +108 -17
- package/build/dist/Server/Utils/SecurityEvent/GoogleSecOps/GoogleSecOpsPoller.js.map +1 -1
- package/build/dist/Server/Utils/SessionReplay/SessionReplayGateCache.js.map +1 -1
- package/build/dist/Server/Utils/SessionReplay/SessionReplayIdentity.js +59 -0
- package/build/dist/Server/Utils/SessionReplay/SessionReplayIdentity.js.map +1 -0
- package/build/dist/Server/Utils/SessionReplay/SessionReplayReadService.js +9 -2
- package/build/dist/Server/Utils/SessionReplay/SessionReplayReadService.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.js +17 -6
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Auth.js +0 -159
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Auth.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Authorization.js +128 -0
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Authorization.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.js +17 -6
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.js +25 -13
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.js.map +1 -1
- package/build/dist/Types/API/Route.js +16 -5
- package/build/dist/Types/API/Route.js.map +1 -1
- package/build/dist/Types/Date.js +115 -13
- package/build/dist/Types/Date.js.map +1 -1
- package/build/dist/Types/Rum/SessionReplay.js +38 -0
- package/build/dist/Types/Rum/SessionReplay.js.map +1 -1
- package/build/dist/UI/Components/Date/TimeRangePickerDropdown.js +9 -5
- package/build/dist/UI/Components/Date/TimeRangePickerDropdown.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js +12 -7
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/HistogramTooltip.js +15 -14
- package/build/dist/UI/Components/LogsViewer/components/HistogramTooltip.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js +17 -22
- package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsHistogram.js +2 -4
- package/build/dist/UI/Components/LogsViewer/components/LogsHistogram.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogram.js +2 -4
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogram.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogramTooltip.js +15 -14
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogramTooltip.js.map +1 -1
- package/build/dist/UI/Utils/Navigation.js +30 -0
- package/build/dist/UI/Utils/Navigation.js.map +1 -1
- package/build/dist/Utils/Array.js +25 -0
- package/build/dist/Utils/Array.js.map +1 -1
- package/build/dist/Utils/CertificateReissue.js +97 -0
- package/build/dist/Utils/CertificateReissue.js.map +1 -0
- package/build/dist/Utils/Grok/Grok.js +324 -0
- package/build/dist/Utils/Grok/Grok.js.map +1 -0
- package/build/dist/Utils/Grok/GrokPatterns.js +112 -0
- package/build/dist/Utils/Grok/GrokPatterns.js.map +1 -0
- package/build/dist/Utils/Monitor/NetworkDeviceMonitorTemplateUtil.js +63 -3
- package/build/dist/Utils/Monitor/NetworkDeviceMonitorTemplateUtil.js.map +1 -1
- package/package.json +1 -1
- package/UI/Utils/JsonWebToken.ts +0 -14
- package/build/dist/UI/Utils/JsonWebToken.js +0 -10
- package/build/dist/UI/Utils/JsonWebToken.js.map +0 -1
|
@@ -1,4 +1,29 @@
|
|
|
1
1
|
export default class ArrayUtil {
|
|
2
|
+
/*
|
|
3
|
+
* Run handler over every item, at most `concurrency` of them in flight.
|
|
4
|
+
*
|
|
5
|
+
* For work that is spent waiting on the network rather than on the CPU -
|
|
6
|
+
* walking a list of customer domains and making a request against each -
|
|
7
|
+
* awaiting one at a time leaves the caller idle for almost all of its wall
|
|
8
|
+
* clock, and a single slow item delays every item behind it.
|
|
9
|
+
*
|
|
10
|
+
* Items are handed out in order, so a caller that has sorted by urgency still
|
|
11
|
+
* gets the urgent ones started first. handler is expected to deal with its
|
|
12
|
+
* own failures: as with Promise.all, the first rejection is what the caller
|
|
13
|
+
* sees, and it does not stop work already in flight.
|
|
14
|
+
*/
|
|
15
|
+
static async forEachWithConcurrency(array, concurrency, handler) {
|
|
16
|
+
let nextIndex = 0;
|
|
17
|
+
const worker = async () => {
|
|
18
|
+
while (nextIndex < array.length) {
|
|
19
|
+
const index = nextIndex++;
|
|
20
|
+
await handler(array[index]);
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
await Promise.all(Array.from({ length: Math.max(1, Math.min(concurrency, array.length)) }, () => {
|
|
24
|
+
return worker();
|
|
25
|
+
}));
|
|
26
|
+
}
|
|
2
27
|
static mergeStringArrays(array1, array2) {
|
|
3
28
|
return ArrayUtil.removeDuplicates([...array1, ...array2]);
|
|
4
29
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Array.js","sourceRoot":"","sources":["../../../Utils/Array.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAO,SAAS;
|
|
1
|
+
{"version":3,"file":"Array.js","sourceRoot":"","sources":["../../../Utils/Array.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAO,SAAS;IAC5B;;;;;;;;;;;;OAYG;IACI,MAAM,CAAC,KAAK,CAAC,sBAAsB,CACxC,KAAe,EACf,WAAmB,EACnB,OAAmC;QAEnC,IAAI,SAAS,GAAW,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAwB,KAAK,IAAmB,EAAE;YAC5D,OAAO,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAW,SAAS,EAAE,CAAC;gBAClC,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,CAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CACR,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAC5D,GAAkB,EAAE;YAClB,OAAO,MAAM,EAAE,CAAC;QAClB,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAC7B,MAAqB,EACrB,MAAqB;QAErB,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;IAC5D,CAAC;IAEM,MAAM,CAAC,kBAAkB,CAC9B,MAAqB,EACrB,MAAqB;QAErB,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAEM,MAAM,CAAC,gBAAgB,CAAC,KAAiB;QAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,KAAU,EAAE,KAAa,EAAE,IAAgB,EAAE,EAAE;YAClE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC;QACvC,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,MAAM,CAAC,OAAO,CAAI,KAAe;QACtC,MAAM,aAAa,GAAa,CAAC,GAAG,KAAK,CAAC,CAAC;QAC3C,KAAK,IAAI,CAAC,GAAW,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1D,MAAM,CAAC,GAAW,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAEtD,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,CAAC,aAAa,CAAC,CAAC,CAAQ,EAAE,aAAa,CAAC,CAAC,CAAQ,CAAC,GAAG;gBACnD,aAAa,CAAC,CAAC,CAAC;gBAChB,aAAa,CAAC,CAAC,CAAC;aACjB,CAAC;QACJ,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAEM,MAAM,CAAC,iCAAiC,CAC7C,KAAsB;QAEtB,MAAM,WAAW,GAAoB,EAAE,CAAC;QAExC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,IACE,WAAW,CAAC,MAAM,CAAC,CAAC,IAAc,EAAE,EAAE;gBACpC,OAAO,IAAI,CAAC,QAAQ,EAAE,KAAK,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACjD,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EACb,CAAC;gBACD,SAAS;YACX,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,CAAa,EAAE,CAAa;QAChD,2CAA2C;QAC3C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;YAC1B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,gDAAgD;QAChD,MAAM,UAAU,GAAW,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,UAAU,GAAW,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAEzD,4BAA4B;QAC5B,OAAO,UAAU,KAAK,UAAU,CAAC;IACnC,CAAC;IAEM,MAAM,CAAC,eAAe,CAAC,SAAiB;QAC7C,OAAO,CAAC,CAAM,EAAE,CAAM,EAAU,EAAE;YAChC,IAAI,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChC,OAAO,CAAC,CAAC,CAAC;YACZ,CAAC;YACD,IAAI,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChC,OAAO,CAAC,CAAC;YACX,CAAC;YACD,OAAO,CAAC,CAAC;QACX,CAAC,CAAC;IACJ,CAAC;IAEM,MAAM,CAAC,kBAAkB,CAAI,KAAe;QACjD,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAE,CAAC;IAC1D,CAAC;IAEM,MAAM,CAAC,mBAAmB,CAC/B,KAAiB,EACjB,SAAiB;QAEjB,qDAAqD;QACrD,MAAM,cAAc,GAAe,KAAK;aACrC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE;YACjB,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,KAAU,EAAE,KAAa,EAAE,IAAgB,EAAE,EAAE;YACtD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC;QACvC,CAAC,CAAC,CAAC;QAEL,8CAA8C;QAC9C,MAAM,aAAa,GAAe,EAAE,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,IAAI,GAAQ,KAAK,CAAC,IAAI,CAAC,CAAC,IAAS,EAAE,EAAE;gBACzC,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,KAAK,CAAC;YACnC,CAAC,CAAC,CAAC;YACH,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import OneUptimeDate from "../Types/Date";
|
|
2
|
+
/*
|
|
3
|
+
* The cooldown that sits behind the "Reissue SSL Certificate" button on a
|
|
4
|
+
* custom domain.
|
|
5
|
+
*
|
|
6
|
+
* OneUptime orders certificates from Let's Encrypt against a single ACME
|
|
7
|
+
* account shared by every customer on the installation, so a button that any
|
|
8
|
+
* customer can press is a button that spends everybody's allowance. Two of
|
|
9
|
+
* Let's Encrypt's limits are the ones this guards:
|
|
10
|
+
*
|
|
11
|
+
* - 5 failed validations per account, per hostname, per hour. A domain whose
|
|
12
|
+
* CNAME is half-broken fails validation every single time, so an
|
|
13
|
+
* unthrottled button is a way to burn that hostname's whole hourly budget
|
|
14
|
+
* in seconds - including the budget the automated renewal cron needs.
|
|
15
|
+
* - 5 duplicate certificates per week for the same exact set of hostnames.
|
|
16
|
+
* A reissue is by definition a duplicate certificate: same hostname, no
|
|
17
|
+
* change. This is the limit a determined presser reaches second.
|
|
18
|
+
*
|
|
19
|
+
* Deliberately pure and synchronous, and deliberately in Common/Utils rather
|
|
20
|
+
* than Common/Server: the dashboard renders the very same countdown the API
|
|
21
|
+
* will enforce, so the button explains itself before it is pressed instead of
|
|
22
|
+
* only after the request comes back rejected.
|
|
23
|
+
*
|
|
24
|
+
* The cooldown is keyed off when a reissue was REQUESTED, not when one
|
|
25
|
+
* succeeded. A request that fails at the CA still cost a validation attempt,
|
|
26
|
+
* and a failing domain is exactly the domain a frustrated customer presses
|
|
27
|
+
* again - so a failed attempt must start the clock just as a successful one
|
|
28
|
+
* does.
|
|
29
|
+
*/
|
|
30
|
+
class CertificateReissueUtil {
|
|
31
|
+
/*
|
|
32
|
+
* The newest "requested at" stamp that is already out of cooldown.
|
|
33
|
+
*
|
|
34
|
+
* A row is eligible for another reissue when its stamp is null or is at or
|
|
35
|
+
* before this instant. The server turns this into the WHERE clause it
|
|
36
|
+
* claims the row with, so the comparison that decides eligibility is the
|
|
37
|
+
* same one on both sides.
|
|
38
|
+
*/
|
|
39
|
+
static getCooldownCutoff(now) {
|
|
40
|
+
return OneUptimeDate.addRemoveHours(OneUptimeDate.fromString(now), -CertificateReissueUtil.COOLDOWN_IN_HOURS);
|
|
41
|
+
}
|
|
42
|
+
// When the domain may be reissued again, given its last request.
|
|
43
|
+
static getNextReissueAllowedAt(lastRequestedAt) {
|
|
44
|
+
return OneUptimeDate.addRemoveHours(OneUptimeDate.fromString(lastRequestedAt), CertificateReissueUtil.COOLDOWN_IN_HOURS);
|
|
45
|
+
}
|
|
46
|
+
/*
|
|
47
|
+
* A domain that has never been reissued is never cooling down, which is why
|
|
48
|
+
* null is accepted here rather than being the caller's problem.
|
|
49
|
+
*/
|
|
50
|
+
static isInCooldown(lastRequestedAt, now) {
|
|
51
|
+
if (!lastRequestedAt) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
return OneUptimeDate.isAfter(CertificateReissueUtil.getNextReissueAllowedAt(lastRequestedAt), OneUptimeDate.fromString(now));
|
|
55
|
+
}
|
|
56
|
+
/*
|
|
57
|
+
* How much longer the customer has to wait, in words. Whole minutes, since
|
|
58
|
+
* the countdown is only ever read by a person deciding whether to come back
|
|
59
|
+
* later.
|
|
60
|
+
*/
|
|
61
|
+
static getTimeRemainingText(lastRequestedAt, now) {
|
|
62
|
+
const nextAllowedAt = CertificateReissueUtil.getNextReissueAllowedAt(lastRequestedAt);
|
|
63
|
+
const totalMinutes = Math.max(0, OneUptimeDate.getMinutesBetweenTwoDates(OneUptimeDate.fromString(now), nextAllowedAt));
|
|
64
|
+
if (totalMinutes < 1) {
|
|
65
|
+
return "less than a minute";
|
|
66
|
+
}
|
|
67
|
+
const hours = Math.floor(totalMinutes / 60);
|
|
68
|
+
const minutes = totalMinutes % 60;
|
|
69
|
+
const parts = [];
|
|
70
|
+
if (hours > 0) {
|
|
71
|
+
parts.push(`${hours} ${hours === 1 ? "hour" : "hours"}`);
|
|
72
|
+
}
|
|
73
|
+
if (minutes > 0) {
|
|
74
|
+
parts.push(`${minutes} ${minutes === 1 ? "minute" : "minutes"}`);
|
|
75
|
+
}
|
|
76
|
+
return parts.join(" ");
|
|
77
|
+
}
|
|
78
|
+
/*
|
|
79
|
+
* The one sentence the API returns and the dashboard shows. Kept here so a
|
|
80
|
+
* customer reads the same explanation wherever they hit the limit.
|
|
81
|
+
*/
|
|
82
|
+
static getCooldownMessage(lastRequestedAt, now) {
|
|
83
|
+
return `A certificate reissue was already requested for this domain in the last ${CertificateReissueUtil.COOLDOWN_IN_HOURS} hours. Certificates are issued by Let's Encrypt, which rate limits how often the same domain can be issued, so a reissue can only be requested once every ${CertificateReissueUtil.COOLDOWN_IN_HOURS} hours. Please try again in ${CertificateReissueUtil.getTimeRemainingText(lastRequestedAt, now)}.`;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/*
|
|
87
|
+
* Note on the value: 24 hours permits 7 reissues a week, which is above the
|
|
88
|
+
* 5-duplicate-certificates-per-week limit described above. That is a
|
|
89
|
+
* deliberate choice, not an oversight - reaching it needs a customer to
|
|
90
|
+
* press the button on six consecutive days, and the failure that follows is
|
|
91
|
+
* a clear, temporary error from the CA rather than the sustained burst the
|
|
92
|
+
* per-hour validation limit punishes. The value is a single constant so the
|
|
93
|
+
* trade can be revisited in one place.
|
|
94
|
+
*/
|
|
95
|
+
CertificateReissueUtil.COOLDOWN_IN_HOURS = 24;
|
|
96
|
+
export default CertificateReissueUtil;
|
|
97
|
+
//# sourceMappingURL=CertificateReissue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CertificateReissue.js","sourceRoot":"","sources":["../../../Utils/CertificateReissue.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,MAAM,eAAe,CAAC;AAE1C;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAqB,sBAAsB;IAYzC;;;;;;;OAOG;IACI,MAAM,CAAC,iBAAiB,CAAC,GAAS;QACvC,OAAO,aAAa,CAAC,cAAc,CACjC,aAAa,CAAC,UAAU,CAAC,GAAG,CAAC,EAC7B,CAAC,sBAAsB,CAAC,iBAAiB,CAC1C,CAAC;IACJ,CAAC;IAED,iEAAiE;IAC1D,MAAM,CAAC,uBAAuB,CAAC,eAAqB;QACzD,OAAO,aAAa,CAAC,cAAc,CACjC,aAAa,CAAC,UAAU,CAAC,eAAe,CAAC,EACzC,sBAAsB,CAAC,iBAAiB,CACzC,CAAC;IACJ,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,YAAY,CACxB,eAAwC,EACxC,GAAS;QAET,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,aAAa,CAAC,OAAO,CAC1B,sBAAsB,CAAC,uBAAuB,CAAC,eAAe,CAAC,EAC/D,aAAa,CAAC,UAAU,CAAC,GAAG,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,oBAAoB,CAAC,eAAqB,EAAE,GAAS;QACjE,MAAM,aAAa,GACjB,sBAAsB,CAAC,uBAAuB,CAAC,eAAe,CAAC,CAAC;QAElE,MAAM,YAAY,GAAW,IAAI,CAAC,GAAG,CACnC,CAAC,EACD,aAAa,CAAC,yBAAyB,CACrC,aAAa,CAAC,UAAU,CAAC,GAAG,CAAC,EAC7B,aAAa,CACd,CACF,CAAC;QAEF,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,oBAAoB,CAAC;QAC9B,CAAC;QAED,MAAM,KAAK,GAAW,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;QACpD,MAAM,OAAO,GAAW,YAAY,GAAG,EAAE,CAAC;QAE1C,MAAM,KAAK,GAAkB,EAAE,CAAC;QAEhC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,kBAAkB,CAAC,eAAqB,EAAE,GAAS;QAC/D,OAAO,2EAA2E,sBAAsB,CAAC,iBAAiB,8JAA8J,sBAAsB,CAAC,iBAAiB,+BAA+B,sBAAsB,CAAC,oBAAoB,CAAC,eAAe,EAAE,GAAG,CAAC,GAAG,CAAC;IACta,CAAC;;AA/FD;;;;;;;;GAQG;AACoB,wCAAiB,GAAW,EAAE,CAAC;eAVnC,sBAAsB"}
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
import BadDataException from "../../Types/Exception/BadDataException";
|
|
2
|
+
import GrokPatterns from "./GrokPatterns";
|
|
3
|
+
/*
|
|
4
|
+
* Grok engine.
|
|
5
|
+
*
|
|
6
|
+
* A grok pattern is a regex with named references into a pattern
|
|
7
|
+
* library: `%{IPV4:client_ip} - %{WORD:verb}` means "the IPv4 regex,
|
|
8
|
+
* captured as client_ip, then a literal ' - ', then a word captured as
|
|
9
|
+
* verb". Anything that is not a `%{...}` reference is passed through to
|
|
10
|
+
* the regex verbatim, so raw regex and grok references can be mixed.
|
|
11
|
+
*
|
|
12
|
+
* Supported reference forms:
|
|
13
|
+
*
|
|
14
|
+
* %{NAME} match, capture nothing
|
|
15
|
+
* %{NAME:field} capture into `field` as a string
|
|
16
|
+
* %{NAME:field:type} capture into `field`, coerced (int/float/boolean)
|
|
17
|
+
*
|
|
18
|
+
* Compilation expands references recursively into one JavaScript RegExp
|
|
19
|
+
* and records, for each captured field, the internal group name that
|
|
20
|
+
* holds it. Only the groups this engine injects capture — every library
|
|
21
|
+
* definition uses `(?:...)` — so extraction is a `match.groups` lookup
|
|
22
|
+
* with no index arithmetic.
|
|
23
|
+
*
|
|
24
|
+
* Matching is deliberately UNANCHORED, like Logstash: a pattern
|
|
25
|
+
* describes a fragment of the line unless the author anchors it with
|
|
26
|
+
* `^`/`$` themselves.
|
|
27
|
+
*
|
|
28
|
+
* This runs once per ingested log record on the telemetry hot path
|
|
29
|
+
* against text the customer's users control, so the compiler enforces
|
|
30
|
+
* hard ceilings on expansion (depth, count, final source length) and
|
|
31
|
+
* the matcher refuses inputs above MAX_GROK_INPUT_LENGTH rather than
|
|
32
|
+
* handing an unbounded string to a backtracking engine.
|
|
33
|
+
*/
|
|
34
|
+
export var GrokFieldType;
|
|
35
|
+
(function (GrokFieldType) {
|
|
36
|
+
GrokFieldType["String"] = "string";
|
|
37
|
+
GrokFieldType["Integer"] = "integer";
|
|
38
|
+
GrokFieldType["Float"] = "float";
|
|
39
|
+
GrokFieldType["Boolean"] = "boolean";
|
|
40
|
+
})(GrokFieldType || (GrokFieldType = {}));
|
|
41
|
+
/** Longest raw pattern a user may save. */
|
|
42
|
+
export const MAX_GROK_PATTERN_LENGTH = 4000;
|
|
43
|
+
/** Longest expanded regex source a pattern may compile to. */
|
|
44
|
+
export const MAX_GROK_SOURCE_LENGTH = 20000;
|
|
45
|
+
/** How deep `%{A}` -> `%{B}` -> `%{C}` nesting may go. */
|
|
46
|
+
export const MAX_GROK_EXPANSION_DEPTH = 20;
|
|
47
|
+
/** Total number of references expanded while compiling one pattern. */
|
|
48
|
+
export const MAX_GROK_EXPANSIONS = 500;
|
|
49
|
+
/** Fields one pattern may capture. */
|
|
50
|
+
export const MAX_GROK_CAPTURES = 100;
|
|
51
|
+
/*
|
|
52
|
+
* Inputs longer than this are not parsed. A backtracking regex cannot be
|
|
53
|
+
* interrupted once started, so the only bound available on the ingest
|
|
54
|
+
* hot path is the length of what we feed it.
|
|
55
|
+
*/
|
|
56
|
+
export const MAX_GROK_INPUT_LENGTH = 32768;
|
|
57
|
+
/*
|
|
58
|
+
* `%{NAME}`, `%{NAME:field}` or `%{NAME:field:type}`. The field and type
|
|
59
|
+
* segments exclude `{`, `}` and `:` so a malformed reference fails to
|
|
60
|
+
* match here and is reported as unparsable rather than being silently
|
|
61
|
+
* split at the wrong colon.
|
|
62
|
+
*/
|
|
63
|
+
const GROK_REFERENCE_SOURCE = "%\\{([A-Za-z0-9_]+)(?::([^:{}]*))?(?::([^:{}]*))?\\}";
|
|
64
|
+
/*
|
|
65
|
+
* Field names become log attribute keys. Dots are allowed because OTel
|
|
66
|
+
* semantic-convention keys use them (`http.request.method`).
|
|
67
|
+
*/
|
|
68
|
+
const FIELD_NAME_REGEX = /^[A-Za-z_][A-Za-z0-9_.@-]*$/;
|
|
69
|
+
function parseFieldType(rawType) {
|
|
70
|
+
if (!rawType) {
|
|
71
|
+
return GrokFieldType.String;
|
|
72
|
+
}
|
|
73
|
+
switch (rawType.trim().toLowerCase()) {
|
|
74
|
+
case "int":
|
|
75
|
+
case "integer":
|
|
76
|
+
case "long":
|
|
77
|
+
return GrokFieldType.Integer;
|
|
78
|
+
case "float":
|
|
79
|
+
case "double":
|
|
80
|
+
case "number":
|
|
81
|
+
return GrokFieldType.Float;
|
|
82
|
+
case "bool":
|
|
83
|
+
case "boolean":
|
|
84
|
+
return GrokFieldType.Boolean;
|
|
85
|
+
case "string":
|
|
86
|
+
case "text":
|
|
87
|
+
return GrokFieldType.String;
|
|
88
|
+
default:
|
|
89
|
+
throw new BadDataException(`Unknown grok field type "${rawType}". Supported types are: int, long, float, double, boolean, string.`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function resolveReference(match, context) {
|
|
93
|
+
const patternName = match[1];
|
|
94
|
+
const rawFieldName = match[2];
|
|
95
|
+
const rawFieldType = match[3];
|
|
96
|
+
context.expansions++;
|
|
97
|
+
if (context.expansions > MAX_GROK_EXPANSIONS) {
|
|
98
|
+
throw new BadDataException(`This grok pattern expands to more than ${MAX_GROK_EXPANSIONS} sub-patterns. Please simplify it.`);
|
|
99
|
+
}
|
|
100
|
+
const definition = context.patterns[patternName];
|
|
101
|
+
if (definition === undefined) {
|
|
102
|
+
throw new BadDataException(`Unknown grok pattern "%{${patternName}}". Check the list of supported patterns.`);
|
|
103
|
+
}
|
|
104
|
+
if (context.stack.includes(patternName)) {
|
|
105
|
+
throw new BadDataException(`Grok pattern "%{${patternName}}" refers to itself. Circular pattern references are not supported.`);
|
|
106
|
+
}
|
|
107
|
+
if (context.stack.length >= MAX_GROK_EXPANSION_DEPTH) {
|
|
108
|
+
throw new BadDataException(`Grok pattern "%{${patternName}}" nests more than ${MAX_GROK_EXPANSION_DEPTH} levels deep. Please simplify it.`);
|
|
109
|
+
}
|
|
110
|
+
context.stack.push(patternName);
|
|
111
|
+
const expanded = expandPattern(definition, context);
|
|
112
|
+
context.stack.pop();
|
|
113
|
+
if (rawFieldName === undefined) {
|
|
114
|
+
return `(?:${expanded})`;
|
|
115
|
+
}
|
|
116
|
+
const fieldName = rawFieldName.trim();
|
|
117
|
+
if (!FIELD_NAME_REGEX.test(fieldName)) {
|
|
118
|
+
throw new BadDataException(`"${rawFieldName}" is not a valid grok field name. Use letters, digits, and . _ - @ (starting with a letter or underscore).`);
|
|
119
|
+
}
|
|
120
|
+
if (context.captures.length >= MAX_GROK_CAPTURES) {
|
|
121
|
+
throw new BadDataException(`This grok pattern captures more than ${MAX_GROK_CAPTURES} fields. Please capture fewer fields.`);
|
|
122
|
+
}
|
|
123
|
+
const groupName = `oug${context.captures.length}`;
|
|
124
|
+
context.captures.push({
|
|
125
|
+
groupName: groupName,
|
|
126
|
+
fieldName: fieldName,
|
|
127
|
+
fieldType: parseFieldType(rawFieldType),
|
|
128
|
+
});
|
|
129
|
+
return `(?<${groupName}>${expanded})`;
|
|
130
|
+
}
|
|
131
|
+
function expandPattern(source, context) {
|
|
132
|
+
/*
|
|
133
|
+
* A fresh RegExp per call: `lastIndex` is per-object state and
|
|
134
|
+
* expansion recurses, so a shared instance would have nested calls
|
|
135
|
+
* stepping on each other's scan position.
|
|
136
|
+
*/
|
|
137
|
+
const referenceRegex = new RegExp(GROK_REFERENCE_SOURCE, "g");
|
|
138
|
+
let output = "";
|
|
139
|
+
let cursor = 0;
|
|
140
|
+
let match = referenceRegex.exec(source);
|
|
141
|
+
while (match !== null) {
|
|
142
|
+
output += source.slice(cursor, match.index);
|
|
143
|
+
output += resolveReference(match, context);
|
|
144
|
+
cursor = match.index + match[0].length;
|
|
145
|
+
if (output.length > MAX_GROK_SOURCE_LENGTH) {
|
|
146
|
+
throw new BadDataException(`This grok pattern expands to more than ${MAX_GROK_SOURCE_LENGTH} characters of regex. Please simplify it.`);
|
|
147
|
+
}
|
|
148
|
+
match = referenceRegex.exec(source);
|
|
149
|
+
}
|
|
150
|
+
output += source.slice(cursor);
|
|
151
|
+
return output;
|
|
152
|
+
}
|
|
153
|
+
/*
|
|
154
|
+
* Compile a grok pattern. Throws BadDataException with a message meant
|
|
155
|
+
* for the person who typed the pattern — this is what save-time
|
|
156
|
+
* validation and the pattern tester in the dashboard show.
|
|
157
|
+
*/
|
|
158
|
+
export function compileGrokPattern(pattern, extraPatterns) {
|
|
159
|
+
if (typeof pattern !== "string" || pattern.trim().length === 0) {
|
|
160
|
+
throw new BadDataException("Grok pattern cannot be empty.");
|
|
161
|
+
}
|
|
162
|
+
if (pattern.length > MAX_GROK_PATTERN_LENGTH) {
|
|
163
|
+
throw new BadDataException(`Grok pattern cannot be longer than ${MAX_GROK_PATTERN_LENGTH} characters.`);
|
|
164
|
+
}
|
|
165
|
+
const context = {
|
|
166
|
+
patterns: extraPatterns
|
|
167
|
+
? Object.assign(Object.assign({}, GrokPatterns), extraPatterns) : GrokPatterns,
|
|
168
|
+
captures: [],
|
|
169
|
+
stack: [],
|
|
170
|
+
expansions: 0,
|
|
171
|
+
};
|
|
172
|
+
const source = expandPattern(pattern, context);
|
|
173
|
+
if (source.length > MAX_GROK_SOURCE_LENGTH) {
|
|
174
|
+
throw new BadDataException(`This grok pattern expands to more than ${MAX_GROK_SOURCE_LENGTH} characters of regex. Please simplify it.`);
|
|
175
|
+
}
|
|
176
|
+
let regex;
|
|
177
|
+
try {
|
|
178
|
+
/*
|
|
179
|
+
* No `g`/`y` flag on purpose: those carry `lastIndex` between calls
|
|
180
|
+
* and a compiled pattern is reused for every record.
|
|
181
|
+
*/
|
|
182
|
+
regex = new RegExp(source);
|
|
183
|
+
}
|
|
184
|
+
catch (err) {
|
|
185
|
+
throw new BadDataException(`This grok pattern is not valid: ${err instanceof Error ? err.message : String(err)}`);
|
|
186
|
+
}
|
|
187
|
+
return {
|
|
188
|
+
regex: regex,
|
|
189
|
+
captures: context.captures,
|
|
190
|
+
source: source,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
function coerceValue(raw, fieldType) {
|
|
194
|
+
if (fieldType === GrokFieldType.Integer) {
|
|
195
|
+
const parsed = parseInt(raw, 10);
|
|
196
|
+
return Number.isFinite(parsed) ? parsed : raw;
|
|
197
|
+
}
|
|
198
|
+
if (fieldType === GrokFieldType.Float) {
|
|
199
|
+
const parsed = parseFloat(raw);
|
|
200
|
+
return Number.isFinite(parsed) ? parsed : raw;
|
|
201
|
+
}
|
|
202
|
+
if (fieldType === GrokFieldType.Boolean) {
|
|
203
|
+
const normalized = raw.trim().toLowerCase();
|
|
204
|
+
if (normalized === "true" ||
|
|
205
|
+
normalized === "yes" ||
|
|
206
|
+
normalized === "y" ||
|
|
207
|
+
normalized === "1") {
|
|
208
|
+
return true;
|
|
209
|
+
}
|
|
210
|
+
if (normalized === "false" ||
|
|
211
|
+
normalized === "no" ||
|
|
212
|
+
normalized === "n" ||
|
|
213
|
+
normalized === "0") {
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
/*
|
|
217
|
+
* Not recognisably a boolean — keep the raw text rather than
|
|
218
|
+
* inventing a `false` the log never said.
|
|
219
|
+
*/
|
|
220
|
+
return raw;
|
|
221
|
+
}
|
|
222
|
+
return raw;
|
|
223
|
+
}
|
|
224
|
+
/*
|
|
225
|
+
* Run a compiled pattern against one string. Returns null when the
|
|
226
|
+
* pattern does not match; returns the captured fields otherwise (which
|
|
227
|
+
* is an empty object for a pattern that matches but captures nothing).
|
|
228
|
+
*
|
|
229
|
+
* Empty captures are dropped, matching Logstash's default
|
|
230
|
+
* `keep_empty_captures => false`: an optional group that did not
|
|
231
|
+
* participate should not write a blank attribute onto the log.
|
|
232
|
+
*/
|
|
233
|
+
export function matchGrokPattern(compiled, input) {
|
|
234
|
+
if (typeof input !== "string" || input.length === 0) {
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
if (input.length > MAX_GROK_INPUT_LENGTH) {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
const match = compiled.regex.exec(input);
|
|
241
|
+
if (!match) {
|
|
242
|
+
return null;
|
|
243
|
+
}
|
|
244
|
+
const groups = match.groups || {};
|
|
245
|
+
const extracted = {};
|
|
246
|
+
for (const capture of compiled.captures) {
|
|
247
|
+
const raw = groups[capture.groupName];
|
|
248
|
+
if (raw === undefined || raw === "") {
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
/*
|
|
252
|
+
* The same field name can legitimately appear twice (alternation
|
|
253
|
+
* branches, or a library pattern that already names it). Whichever
|
|
254
|
+
* group actually matched first wins, so a later branch that did not
|
|
255
|
+
* participate cannot blank out a real value.
|
|
256
|
+
*/
|
|
257
|
+
if (extracted[capture.fieldName] !== undefined) {
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
extracted[capture.fieldName] = coerceValue(raw, capture.fieldType);
|
|
261
|
+
}
|
|
262
|
+
return extracted;
|
|
263
|
+
}
|
|
264
|
+
/*
|
|
265
|
+
* Process-wide compile cache for the ingest path.
|
|
266
|
+
*
|
|
267
|
+
* Compiling walks the pattern library and builds a RegExp; doing that
|
|
268
|
+
* per log record would dominate the cost of the processor. Compiled
|
|
269
|
+
* patterns are immutable and carry no `lastIndex` state (no `g` flag),
|
|
270
|
+
* so one instance is safely shared by every project using the same
|
|
271
|
+
* pattern text.
|
|
272
|
+
*
|
|
273
|
+
* Failures are cached too. An invalid pattern that reaches ingest would
|
|
274
|
+
* otherwise be recompiled — and rejected — once per record.
|
|
275
|
+
*/
|
|
276
|
+
const compileCache = new Map();
|
|
277
|
+
export const MAX_CACHED_GROK_PATTERNS = 500;
|
|
278
|
+
export function compileGrokPatternCached(pattern) {
|
|
279
|
+
const cached = compileCache.get(pattern);
|
|
280
|
+
if (cached !== undefined) {
|
|
281
|
+
if (cached instanceof BadDataException) {
|
|
282
|
+
throw cached;
|
|
283
|
+
}
|
|
284
|
+
return cached;
|
|
285
|
+
}
|
|
286
|
+
let result;
|
|
287
|
+
try {
|
|
288
|
+
result = compileGrokPattern(pattern);
|
|
289
|
+
}
|
|
290
|
+
catch (err) {
|
|
291
|
+
result =
|
|
292
|
+
err instanceof BadDataException ? err : new BadDataException(String(err));
|
|
293
|
+
}
|
|
294
|
+
if (compileCache.size >= MAX_CACHED_GROK_PATTERNS) {
|
|
295
|
+
// Insertion-ordered: drop the oldest entry to bound the cache.
|
|
296
|
+
const oldestKey = compileCache.keys().next().value;
|
|
297
|
+
if (oldestKey !== undefined) {
|
|
298
|
+
compileCache.delete(oldestKey);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
compileCache.set(pattern, result);
|
|
302
|
+
if (result instanceof BadDataException) {
|
|
303
|
+
throw result;
|
|
304
|
+
}
|
|
305
|
+
return result;
|
|
306
|
+
}
|
|
307
|
+
/** Test seam: forget every cached compilation. */
|
|
308
|
+
export function clearGrokCompileCache() {
|
|
309
|
+
compileCache.clear();
|
|
310
|
+
}
|
|
311
|
+
/*
|
|
312
|
+
* Compile + match in one call. Convenience for the dashboard's pattern
|
|
313
|
+
* tester and for tests; ingest compiles once and reuses the result.
|
|
314
|
+
*/
|
|
315
|
+
export function parseWithGrokPattern(pattern, input) {
|
|
316
|
+
return matchGrokPattern(compileGrokPattern(pattern), input);
|
|
317
|
+
}
|
|
318
|
+
export default {
|
|
319
|
+
compileGrokPattern,
|
|
320
|
+
compileGrokPatternCached,
|
|
321
|
+
matchGrokPattern,
|
|
322
|
+
parseWithGrokPattern,
|
|
323
|
+
};
|
|
324
|
+
//# sourceMappingURL=Grok.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Grok.js","sourceRoot":"","sources":["../../../../Utils/Grok/Grok.ts"],"names":[],"mappings":"AAAA,OAAO,gBAAgB,MAAM,wCAAwC,CAAC;AACtE,OAAO,YAAY,MAAM,gBAAgB,CAAC;AAE1C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,MAAM,CAAN,IAAY,aAKX;AALD,WAAY,aAAa;IACvB,kCAAiB,CAAA;IACjB,oCAAmB,CAAA;IACnB,gCAAe,CAAA;IACf,oCAAmB,CAAA;AACrB,CAAC,EALW,aAAa,KAAb,aAAa,QAKxB;AAmBD,2CAA2C;AAC3C,MAAM,CAAC,MAAM,uBAAuB,GAAW,IAAI,CAAC;AAEpD,8DAA8D;AAC9D,MAAM,CAAC,MAAM,sBAAsB,GAAW,KAAK,CAAC;AAEpD,0DAA0D;AAC1D,MAAM,CAAC,MAAM,wBAAwB,GAAW,EAAE,CAAC;AAEnD,uEAAuE;AACvE,MAAM,CAAC,MAAM,mBAAmB,GAAW,GAAG,CAAC;AAE/C,sCAAsC;AACtC,MAAM,CAAC,MAAM,iBAAiB,GAAW,GAAG,CAAC;AAE7C;;;;GAIG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAW,KAAK,CAAC;AAEnD;;;;;GAKG;AACH,MAAM,qBAAqB,GACzB,sDAAsD,CAAC;AAEzD;;;GAGG;AACH,MAAM,gBAAgB,GAAW,6BAA6B,CAAC;AAU/D,SAAS,cAAc,CAAC,OAA2B;IACjD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,aAAa,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QACrC,KAAK,KAAK,CAAC;QACX,KAAK,SAAS,CAAC;QACf,KAAK,MAAM;YACT,OAAO,aAAa,CAAC,OAAO,CAAC;QAC/B,KAAK,OAAO,CAAC;QACb,KAAK,QAAQ,CAAC;QACd,KAAK,QAAQ;YACX,OAAO,aAAa,CAAC,KAAK,CAAC;QAC7B,KAAK,MAAM,CAAC;QACZ,KAAK,SAAS;YACZ,OAAO,aAAa,CAAC,OAAO,CAAC;QAC/B,KAAK,QAAQ,CAAC;QACd,KAAK,MAAM;YACT,OAAO,aAAa,CAAC,MAAM,CAAC;QAC9B;YACE,MAAM,IAAI,gBAAgB,CACxB,4BAA4B,OAAO,oEAAoE,CACxG,CAAC;IACN,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CACvB,KAAsB,EACtB,OAAuB;IAEvB,MAAM,WAAW,GAAW,KAAK,CAAC,CAAC,CAAW,CAAC;IAC/C,MAAM,YAAY,GAAuB,KAAK,CAAC,CAAC,CAAC,CAAC;IAClD,MAAM,YAAY,GAAuB,KAAK,CAAC,CAAC,CAAC,CAAC;IAElD,OAAO,CAAC,UAAU,EAAE,CAAC;IAErB,IAAI,OAAO,CAAC,UAAU,GAAG,mBAAmB,EAAE,CAAC;QAC7C,MAAM,IAAI,gBAAgB,CACxB,0CAA0C,mBAAmB,oCAAoC,CAClG,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAuB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAErE,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,gBAAgB,CACxB,2BAA2B,WAAW,2CAA2C,CAClF,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,gBAAgB,CACxB,mBAAmB,WAAW,qEAAqE,CACpG,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,IAAI,wBAAwB,EAAE,CAAC;QACrD,MAAM,IAAI,gBAAgB,CACxB,mBAAmB,WAAW,sBAAsB,wBAAwB,mCAAmC,CAChH,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAW,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAC5D,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAEpB,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,MAAM,QAAQ,GAAG,CAAC;IAC3B,CAAC;IAED,MAAM,SAAS,GAAW,YAAY,CAAC,IAAI,EAAE,CAAC;IAE9C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,gBAAgB,CACxB,IAAI,YAAY,4GAA4G,CAC7H,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,gBAAgB,CACxB,wCAAwC,iBAAiB,uCAAuC,CACjG,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAW,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAE1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;QACpB,SAAS,EAAE,SAAS;QACpB,SAAS,EAAE,SAAS;QACpB,SAAS,EAAE,cAAc,CAAC,YAAY,CAAC;KACxC,CAAC,CAAC;IAEH,OAAO,MAAM,SAAS,IAAI,QAAQ,GAAG,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,MAAc,EAAE,OAAuB;IAC5D;;;;OAIG;IACH,MAAM,cAAc,GAAW,IAAI,MAAM,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;IAEtE,IAAI,MAAM,GAAW,EAAE,CAAC;IACxB,IAAI,MAAM,GAAW,CAAC,CAAC;IACvB,IAAI,KAAK,GAA2B,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhE,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC3C,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAEvC,IAAI,MAAM,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;YAC3C,MAAM,IAAI,gBAAgB,CACxB,0CAA0C,sBAAsB,2CAA2C,CAC5G,CAAC;QACJ,CAAC;QAED,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAE/B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAAe,EACf,aAAkD;IAElD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,gBAAgB,CAAC,+BAA+B,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QAC7C,MAAM,IAAI,gBAAgB,CACxB,sCAAsC,uBAAuB,cAAc,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAmB;QAC9B,QAAQ,EAAE,aAAa;YACrB,CAAC,iCAAM,YAAY,GAAK,aAAa,EACrC,CAAC,CAAC,YAAY;QAChB,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,UAAU,EAAE,CAAC;KACd,CAAC;IAEF,MAAM,MAAM,GAAW,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAEvD,IAAI,MAAM,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC3C,MAAM,IAAI,gBAAgB,CACxB,0CAA0C,sBAAsB,2CAA2C,CAC5G,CAAC;IACJ,CAAC;IAED,IAAI,KAAa,CAAC;IAElB,IAAI,CAAC;QACH;;;WAGG;QACH,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,gBAAgB,CACxB,mCAAmC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK,EAAE,KAAK;QACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,MAAM;KACf,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,SAAwB;IACxD,IAAI,SAAS,KAAK,aAAa,CAAC,OAAO,EAAE,CAAC;QACxC,MAAM,MAAM,GAAW,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;IAChD,CAAC;IAED,IAAI,SAAS,KAAK,aAAa,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,MAAM,GAAW,UAAU,CAAC,GAAG,CAAC,CAAC;QACvC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;IAChD,CAAC;IAED,IAAI,SAAS,KAAK,aAAa,CAAC,OAAO,EAAE,CAAC;QACxC,MAAM,UAAU,GAAW,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAEpD,IACE,UAAU,KAAK,MAAM;YACrB,UAAU,KAAK,KAAK;YACpB,UAAU,KAAK,GAAG;YAClB,UAAU,KAAK,GAAG,EAClB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IACE,UAAU,KAAK,OAAO;YACtB,UAAU,KAAK,IAAI;YACnB,UAAU,KAAK,GAAG;YAClB,UAAU,KAAK,GAAG,EAClB,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED;;;WAGG;QACH,OAAO,GAAG,CAAC;IACb,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAC9B,QAA6B,EAC7B,KAAa;IAEb,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAA2B,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEjE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAuC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;IACtE,MAAM,SAAS,GAA8B,EAAE,CAAC;IAEhD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,GAAG,GAAuB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAE1D,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;YACpC,SAAS;QACX,CAAC;QAED;;;;;WAKG;QACH,IAAI,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE,CAAC;YAC/C,SAAS;QACX,CAAC;QAED,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,YAAY,GAChB,IAAI,GAAG,EAAkD,CAAC;AAE5D,MAAM,CAAC,MAAM,wBAAwB,GAAW,GAAG,CAAC;AAEpD,MAAM,UAAU,wBAAwB,CAAC,OAAe;IACtD,MAAM,MAAM,GACV,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAE5B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,IAAI,MAAM,YAAY,gBAAgB,EAAE,CAAC;YACvC,MAAM,MAAM,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,MAA8C,CAAC;IAEnD,IAAI,CAAC;QACH,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM;YACJ,GAAG,YAAY,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,YAAY,CAAC,IAAI,IAAI,wBAAwB,EAAE,CAAC;QAClD,+DAA+D;QAC/D,MAAM,SAAS,GAAuB,YAAY,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAEvE,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAElC,IAAI,MAAM,YAAY,gBAAgB,EAAE,CAAC;QACvC,MAAM,MAAM,CAAC;IACf,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,qBAAqB;IACnC,YAAY,CAAC,KAAK,EAAE,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAe,EACf,KAAa;IAEb,OAAO,gBAAgB,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED,eAAe;IACb,kBAAkB;IAClB,wBAAwB;IACxB,gBAAgB;IAChB,oBAAoB;CACrB,CAAC"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Built-in grok pattern library.
|
|
3
|
+
*
|
|
4
|
+
* Grok is regex with names: `%{IPV4:client_ip}` is just the IPv4 regex
|
|
5
|
+
* plus "put what it matched in client_ip". This file holds the named
|
|
6
|
+
* regex halves; Grok.ts does the naming.
|
|
7
|
+
*
|
|
8
|
+
* Two rules every entry here must follow:
|
|
9
|
+
*
|
|
10
|
+
* 1. NO capturing groups. Use `(?:...)`. Grok.ts injects its own
|
|
11
|
+
* named groups around the pattern it is asked to capture and reads
|
|
12
|
+
* results back through `match.groups`, so a stray `(...)` in a
|
|
13
|
+
* definition would silently shift nothing but still cost a capture
|
|
14
|
+
* slot on every record.
|
|
15
|
+
*
|
|
16
|
+
* 2. JavaScript-compatible syntax only. The upstream Logstash
|
|
17
|
+
* definitions are Oniguruma and use possessive quantifiers (`++`)
|
|
18
|
+
* and atomic groups (`(?>...)`), neither of which V8 accepts —
|
|
19
|
+
* those have been rewritten here rather than copied.
|
|
20
|
+
*
|
|
21
|
+
* Every repetition of a group consumes at least one character, which is
|
|
22
|
+
* what keeps `(?:/[\w]*)+`-shaped patterns linear instead of
|
|
23
|
+
* exponential. Keep it that way when adding patterns: these run once
|
|
24
|
+
* per ingested log record against attacker-supplied text.
|
|
25
|
+
*/
|
|
26
|
+
const GrokPatterns = {
|
|
27
|
+
/* --- Primitives --- */
|
|
28
|
+
USERNAME: String.raw `[a-zA-Z0-9._-]+`,
|
|
29
|
+
USER: String.raw `%{USERNAME}`,
|
|
30
|
+
INT: String.raw `(?:[+-]?(?:[0-9]+))`,
|
|
31
|
+
BASE10NUM: String.raw `(?:[+-]?(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+))`,
|
|
32
|
+
NUMBER: String.raw `(?:%{BASE10NUM})`,
|
|
33
|
+
BASE16NUM: String.raw `(?:0[xX])?(?:[0-9A-Fa-f]+)`,
|
|
34
|
+
POSINT: String.raw `\b(?:[1-9][0-9]*)\b`,
|
|
35
|
+
NONNEGINT: String.raw `\b(?:[0-9]+)\b`,
|
|
36
|
+
WORD: String.raw `\b\w+\b`,
|
|
37
|
+
NOTSPACE: String.raw `\S+`,
|
|
38
|
+
SPACE: String.raw `\s*`,
|
|
39
|
+
DATA: String.raw `.*?`,
|
|
40
|
+
GREEDYDATA: String.raw `.*`,
|
|
41
|
+
QUOTEDSTRING: String.raw `(?:"(?:\\.|[^\\"])*"|'(?:\\.|[^\\'])*')`,
|
|
42
|
+
QS: String.raw `%{QUOTEDSTRING}`,
|
|
43
|
+
UUID: String.raw `[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}`,
|
|
44
|
+
/* --- Network --- */
|
|
45
|
+
IPV4: String.raw `(?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?![0-9])`,
|
|
46
|
+
/*
|
|
47
|
+
* Alternation is first-match, not longest-match, so order carries
|
|
48
|
+
* meaning here: the IPv4-embedded forms come first (a generic hex form
|
|
49
|
+
* would otherwise stop before the dotted quad), and the "::"-elision
|
|
50
|
+
* forms are ordered so a shape that can only match a prefix never sits
|
|
51
|
+
* ahead of one that matches the whole address.
|
|
52
|
+
*/
|
|
53
|
+
IPV6: String.raw `(?:::(?:[Ff]{4}(?::0{1,4})?:)?%{IPV4}|(?:[0-9A-Fa-f]{1,4}:){1,4}:%{IPV4}|(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|[0-9A-Fa-f]{1,4}:(?::[0-9A-Fa-f]{1,4}){1,6}|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|:(?:(?::[0-9A-Fa-f]{1,4}){1,7}|:))`,
|
|
54
|
+
IP: String.raw `(?:%{IPV6}|%{IPV4})`,
|
|
55
|
+
HOSTNAME: String.raw `\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*\.?`,
|
|
56
|
+
IPORHOST: String.raw `(?:%{IP}|%{HOSTNAME})`,
|
|
57
|
+
HOSTPORT: String.raw `%{IPORHOST}:%{POSINT}`,
|
|
58
|
+
COMMONMAC: String.raw `(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})`,
|
|
59
|
+
CISCOMAC: String.raw `(?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})`,
|
|
60
|
+
WINDOWSMAC: String.raw `(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})`,
|
|
61
|
+
MAC: String.raw `(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})`,
|
|
62
|
+
EMAILLOCALPART: String.raw `[a-zA-Z0-9!#$%&'*+\-/=?^_{|}~]{1,64}(?:\.[a-zA-Z0-9!#$%&'*+\-/=?^_{|}~]{1,62}){0,63}`,
|
|
63
|
+
EMAILADDRESS: String.raw `%{EMAILLOCALPART}@%{HOSTNAME}`,
|
|
64
|
+
/* --- Paths and URIs --- */
|
|
65
|
+
UNIXPATH: String.raw `(?:/[\w_%!$@:.,+~-]*)+`,
|
|
66
|
+
WINPATH: String.raw `(?:[A-Za-z]+:|\\)(?:\\[^\\?*]*)+`,
|
|
67
|
+
PATH: String.raw `(?:%{UNIXPATH}|%{WINPATH})`,
|
|
68
|
+
URIPROTO: String.raw `[A-Za-z][A-Za-z0-9+.-]+`,
|
|
69
|
+
URIHOST: String.raw `%{IPORHOST}(?::%{POSINT})?`,
|
|
70
|
+
URIPATH: String.raw `(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_-]*)+`,
|
|
71
|
+
URIPARAM: String.raw `\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?<>\[\]-]*`,
|
|
72
|
+
URIPATHPARAM: String.raw `%{URIPATH}(?:%{URIPARAM})?`,
|
|
73
|
+
URI: String.raw `%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?`,
|
|
74
|
+
/* --- Dates and times --- */
|
|
75
|
+
MONTH: String.raw `\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b`,
|
|
76
|
+
MONTHNUM: String.raw `(?:0?[1-9]|1[0-2])`,
|
|
77
|
+
MONTHNUM2: String.raw `(?:0[1-9]|1[0-2])`,
|
|
78
|
+
MONTHDAY: String.raw `(?:0[1-9]|[12][0-9]|3[01]|[1-9])`,
|
|
79
|
+
DAY: String.raw `(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)`,
|
|
80
|
+
YEAR: String.raw `(?:\d\d){1,2}`,
|
|
81
|
+
HOUR: String.raw `(?:2[0123]|[01]?[0-9])`,
|
|
82
|
+
MINUTE: String.raw `(?:[0-5][0-9])`,
|
|
83
|
+
SECOND: String.raw `(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)`,
|
|
84
|
+
TIME: String.raw `(?:%{HOUR}:%{MINUTE}(?::%{SECOND})?)`,
|
|
85
|
+
DATE_US: String.raw `%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}`,
|
|
86
|
+
DATE_EU: String.raw `%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}`,
|
|
87
|
+
DATE: String.raw `(?:%{DATE_US}|%{DATE_EU})`,
|
|
88
|
+
ISO8601_TIMEZONE: String.raw `(?:Z|[+-]%{HOUR}(?::?%{MINUTE})?)`,
|
|
89
|
+
TIMESTAMP_ISO8601: String.raw `%{YEAR}-%{MONTHNUM2}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?(?:%{ISO8601_TIMEZONE})?`,
|
|
90
|
+
DATESTAMP: String.raw `%{DATE}[- ]%{TIME}`,
|
|
91
|
+
TZ: String.raw `(?:[APMCE][SD]T|UTC|GMT)`,
|
|
92
|
+
HTTPDATE: String.raw `%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}`,
|
|
93
|
+
SYSLOGTIMESTAMP: String.raw `%{MONTH} +%{MONTHDAY} %{TIME}`,
|
|
94
|
+
/* --- Log shapes --- */
|
|
95
|
+
LOGLEVEL: String.raw `(?:[Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo(?:rmation)?|INFO(?:RMATION)?|[Ww]arn(?:ing)?|WARN(?:ING)?|[Ee]rr(?:or)?|ERR(?:OR)?|[Cc]rit(?:ical)?|CRIT(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|[Ee]merg(?:ency)?|EMERG(?:ENCY)?)`,
|
|
96
|
+
PROG: String.raw `[\x21-\x5a\x5c\x5e-\x7e]+`,
|
|
97
|
+
SYSLOGPROG: String.raw `%{PROG:program}(?:\[%{POSINT:pid}\])?`,
|
|
98
|
+
SYSLOGFACILITY: String.raw `<%{NONNEGINT:facility}\.%{NONNEGINT:priority}>`,
|
|
99
|
+
SYSLOGBASE: String.raw `%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{IPORHOST:logsource} %{SYSLOGPROG}:`,
|
|
100
|
+
JAVACLASS: String.raw `(?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*`,
|
|
101
|
+
COMMONAPACHELOG: String.raw `%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`,
|
|
102
|
+
COMBINEDAPACHELOG: String.raw `%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}`,
|
|
103
|
+
};
|
|
104
|
+
/*
|
|
105
|
+
* Names only, sorted — the pattern picker in the processor form lists
|
|
106
|
+
* these so a user can see what is available without leaving the page.
|
|
107
|
+
*/
|
|
108
|
+
export function getGrokPatternNames() {
|
|
109
|
+
return Object.keys(GrokPatterns).sort();
|
|
110
|
+
}
|
|
111
|
+
export default GrokPatterns;
|
|
112
|
+
//# sourceMappingURL=GrokPatterns.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GrokPatterns.js","sourceRoot":"","sources":["../../../../Utils/Grok/GrokPatterns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,MAAM,YAAY,GAA2B;IAC3C,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,iBAAiB;IACrC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,aAAa;IAC7B,GAAG,EAAE,MAAM,CAAC,GAAG,CAAA,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAA,2CAA2C;IAChE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAA,kBAAkB;IACpC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAA,4BAA4B;IACjD,MAAM,EAAE,MAAM,CAAC,GAAG,CAAA,qBAAqB;IACvC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAA,gBAAgB;IACrC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,SAAS;IACzB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,KAAK;IACzB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAA,KAAK;IACtB,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,KAAK;IACrB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAA,IAAI;IAC1B,YAAY,EAAE,MAAM,CAAC,GAAG,CAAA,yCAAyC;IACjE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAA,iBAAiB;IAC/B,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,sDAAsD;IAEtE,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,8GAA8G;IAC9H;;;;;;OAMG;IACH,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,8dAA8d;IAC9e,EAAE,EAAE,MAAM,CAAC,GAAG,CAAA,qBAAqB;IACnC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,gFAAgF;IACpG,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,uBAAuB;IAC3C,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,uBAAuB;IAC3C,SAAS,EAAE,MAAM,CAAC,GAAG,CAAA,0CAA0C;IAC/D,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,2CAA2C;IAC/D,UAAU,EAAE,MAAM,CAAC,GAAG,CAAA,0CAA0C;IAChE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAA,4CAA4C;IAC3D,cAAc,EAAE,MAAM,CAAC,GAAG,CAAA,sFAAsF;IAChH,YAAY,EAAE,MAAM,CAAC,GAAG,CAAA,+BAA+B;IAEvD,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,wBAAwB;IAC5C,OAAO,EAAE,MAAM,CAAC,GAAG,CAAA,kCAAkC;IACrD,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,4BAA4B;IAC5C,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,yBAAyB;IAC7C,OAAO,EAAE,MAAM,CAAC,GAAG,CAAA,4BAA4B;IAC/C,OAAO,EAAE,MAAM,CAAC,GAAG,CAAA,yCAAyC;IAC5D,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,8CAA8C;IAClE,YAAY,EAAE,MAAM,CAAC,GAAG,CAAA,4BAA4B;IACpD,GAAG,EAAE,MAAM,CAAC,GAAG,CAAA,2EAA2E;IAE1F,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC,GAAG,CAAA,sJAAsJ;IACvK,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,oBAAoB;IACxC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAA,mBAAmB;IACxC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,kCAAkC;IACtD,GAAG,EAAE,MAAM,CAAC,GAAG,CAAA,iGAAiG;IAChH,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,wBAAwB;IACxC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAA,gBAAgB;IAClC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAA,wCAAwC;IAC1D,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,sCAAsC;IACtD,OAAO,EAAE,MAAM,CAAC,GAAG,CAAA,uCAAuC;IAC1D,OAAO,EAAE,MAAM,CAAC,GAAG,CAAA,yCAAyC;IAC5D,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,2BAA2B;IAC3C,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAA,mCAAmC;IAC/D,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAA,gGAAgG;IAC7H,SAAS,EAAE,MAAM,CAAC,GAAG,CAAA,oBAAoB;IACzC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAA,0BAA0B;IACxC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,6CAA6C;IACjE,eAAe,EAAE,MAAM,CAAC,GAAG,CAAA,+BAA+B;IAE1D,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAA,8PAA8P;IAClR,IAAI,EAAE,MAAM,CAAC,GAAG,CAAA,2BAA2B;IAC3C,UAAU,EAAE,MAAM,CAAC,GAAG,CAAA,uCAAuC;IAC7D,cAAc,EAAE,MAAM,CAAC,GAAG,CAAA,gDAAgD;IAC1E,UAAU,EAAE,MAAM,CAAC,GAAG,CAAA,0FAA0F;IAChH,SAAS,EAAE,MAAM,CAAC,GAAG,CAAA,yDAAyD;IAC9E,eAAe,EAAE,MAAM,CAAC,GAAG,CAAA,8MAA8M;IACzO,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAA,+CAA+C;CAC7E,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,eAAe,YAAY,CAAC"}
|