@oneuptime/common 12.0.2 → 12.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel.ts +14 -0
- package/Models/DatabaseModels/Index.ts +2 -0
- package/Models/DatabaseModels/LogDropFilter.ts +9 -20
- package/Models/DatabaseModels/StatusPageMonitorRule.ts +911 -0
- package/Models/DatabaseModels/StatusPageResource.ts +73 -0
- package/Models/DatabaseModels/TraceDropFilter.ts +9 -20
- package/Server/API/BaseAPI.ts +13 -2
- package/Server/API/DashboardAPI.ts +39 -0
- package/Server/EnvironmentConfig.ts +31 -1
- package/Server/Infrastructure/GlobalCache.ts +115 -0
- package/Server/Infrastructure/Postgres/DataSourceOptions.ts +16 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1786005052769-AddStatusPageMonitorRule.ts +123 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +2 -0
- package/Server/Services/AlertStateTimelineService.ts +20 -0
- package/Server/Services/CephClusterService.ts +26 -86
- package/Server/Services/CloudResourceService.ts +23 -83
- package/Server/Services/DatabaseService.ts +111 -0
- package/Server/Services/DockerHostService.ts +19 -79
- package/Server/Services/DockerSwarmClusterService.ts +27 -88
- package/Server/Services/HostService.ts +46 -99
- package/Server/Services/IncidentStateTimelineService.ts +20 -0
- package/Server/Services/Index.ts +2 -0
- package/Server/Services/IoTFleetService.ts +19 -79
- package/Server/Services/KubernetesClusterService.ts +17 -78
- package/Server/Services/LabelService.ts +57 -10
- package/Server/Services/MetricTypeService.ts +84 -0
- package/Server/Services/MonitorService.ts +89 -14
- package/Server/Services/MonitorStatusTimelineService.ts +29 -0
- package/Server/Services/MonitorTemplateService.ts +65 -0
- package/Server/Services/PodmanHostService.ts +19 -79
- package/Server/Services/ProxmoxClusterService.ts +23 -83
- package/Server/Services/RumApplicationService.ts +19 -80
- package/Server/Services/ScheduledMaintenanceStateTimelineService.ts +21 -0
- package/Server/Services/ServerlessFunctionService.ts +24 -85
- package/Server/Services/ServiceService.ts +299 -73
- package/Server/Services/StatusPageMonitorRuleEngineService.ts +752 -0
- package/Server/Services/StatusPageMonitorRuleService.ts +471 -0
- package/Server/Services/TelemetryEntityRelationshipService.ts +1 -0
- package/Server/Services/TelemetryEntityService.ts +1 -0
- package/Server/Utils/Database/ProjectScopedReferenceValidator.ts +170 -39
- package/Server/Utils/Monitor/MonitorResource.ts +63 -36
- package/Server/Utils/Monitor/MonitorStatusTimeline.ts +36 -0
- package/Server/Utils/Monitor/MonitorStepsProjectValidator.ts +148 -127
- package/Server/Utils/Monitor/MonitorStepsReferenceExtractor.ts +434 -0
- package/Server/Utils/SingleFlight.ts +83 -0
- package/Server/Utils/StatusPage/MonitorRulePatternValidator.ts +46 -0
- package/Server/Utils/Telemetry/EntityRegistry.ts +107 -26
- package/Server/Utils/Telemetry/ResourceHeartbeat.ts +314 -0
- package/Server/Utils/Telemetry/Telemetry.ts +351 -122
- package/Tests/App/Dashboard/WidgetCatalog.test.ts +2 -0
- package/Tests/Models/DatabaseModels/ColumnTransformerInvariants.test.ts +306 -0
- package/Tests/Models/DatabaseModels/NumberColumnDeserialization.test.ts +333 -0
- package/Tests/Models/DatabaseModels/StatusPageMonitorRuleModel.test.ts +190 -0
- package/Tests/Server/API/DashboardPublicResourceListAPI.test.ts +26 -0
- package/Tests/Server/Infrastructure/GlobalCache.test.ts +226 -0
- package/Tests/Server/Infrastructure/PostgresLockTimeoutOptions.test.ts +181 -0
- package/Tests/Server/Services/DatabaseServiceUpdateColumnsIfUnlocked.test.ts +342 -0
- package/Tests/Server/Services/DropFilterSaveValidation.test.ts +245 -0
- package/Tests/Server/Services/HostServiceUpdateLastSeen.test.ts +163 -15
- package/Tests/Server/Services/MetricTypeAttachServices.test.ts +222 -0
- package/Tests/Server/Services/MissingReferenceWriteGuard.test.ts +614 -0
- package/Tests/Server/Services/MonitorStatusTimelineFastPath.test.ts +23 -0
- package/Tests/Server/Services/ResourceUpdateLastSeenLivenessFallback.test.ts +305 -56
- package/Tests/Server/Services/ServiceUpdateLastSeenSplitGates.test.ts +705 -0
- package/Tests/Server/Services/StatusPageMonitorRuleEngineService.test.ts +1719 -0
- package/Tests/Server/Services/StatusPageMonitorRuleHooks.test.ts +1212 -0
- package/Tests/Server/Services/StatusPageMonitorRuleMonitorCreateHook.test.ts +228 -0
- package/Tests/Server/Utils/APIKey/AccessPermission.test.ts +267 -0
- package/Tests/Server/Utils/Database/ProjectScopedReferenceValidator.test.ts +260 -13
- package/Tests/Server/Utils/EntityRegistryRowFence.test.ts +302 -0
- package/Tests/Server/Utils/MetricTypeIndexConvergence.test.ts +598 -0
- package/Tests/Server/Utils/Monitor/MonitorStatusTimelineUnusableStatus.test.ts +154 -0
- package/Tests/Server/Utils/Monitor/MonitorStepsProjectValidator.test.ts +638 -92
- package/Tests/Server/Utils/Monitor/MonitorStepsReferenceExtractor.test.ts +724 -0
- package/Tests/Server/Utils/SingleFlight.test.ts +221 -0
- package/Tests/Types/Dashboard/NetworkDashboardTemplate.test.ts +793 -0
- package/Tests/Types/Database/BigIntColumnTransformer.test.ts +147 -0
- package/Tests/Types/Database/DatabasePropertyTransformer.test.ts +140 -0
- package/Tests/Types/Database/NumericColumnValue.test.ts +127 -0
- package/Tests/Types/Decimal.test.ts +37 -0
- package/Tests/Types/Monitor/SnmpVersionParsing.test.ts +152 -0
- package/Tests/Types/Port.test.ts +38 -0
- package/Tests/UI/Components/LogsHistogram.test.tsx +276 -0
- package/Tests/UI/Components/UseLiveLogsRefresh.test.tsx +489 -0
- package/Tests/UI/Components/UseLogsHistogram.test.tsx +500 -0
- package/Tests/Utils/Dashboard/Components/DashboardComponentsUtil.test.ts +120 -0
- package/Tests/Utils/Dashboard/DashboardNetworkMapComponent.test.ts +327 -0
- package/Types/Dashboard/DashboardComponentType.ts +1 -0
- package/Types/Dashboard/DashboardComponents/ComponentArgument.ts +1 -0
- package/Types/Dashboard/DashboardComponents/DashboardNetworkMapComponent.ts +40 -0
- package/Types/Dashboard/DashboardTemplates.ts +397 -0
- package/Types/Database/BigIntColumnTransformer.ts +74 -0
- package/Types/Database/DatabaseProperty.ts +26 -0
- package/Types/Database/NumericColumnValue.ts +128 -0
- package/Types/Decimal.ts +16 -4
- package/Types/Permission.ts +48 -0
- package/Types/Port.ts +16 -4
- package/UI/Components/LogsViewer/useLiveLogsRefresh.ts +86 -0
- package/UI/Components/LogsViewer/useLogsHistogram.ts +113 -0
- package/Utils/Dashboard/Components/DashboardNetworkMapComponent.ts +136 -0
- package/Utils/Dashboard/Components/Index.ts +7 -0
- package/build/dist/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel.js +12 -0
- package/build/dist/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Index.js +2 -0
- package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
- package/build/dist/Models/DatabaseModels/LogDropFilter.js +9 -20
- package/build/dist/Models/DatabaseModels/LogDropFilter.js.map +1 -1
- package/build/dist/Models/DatabaseModels/StatusPageMonitorRule.js +925 -0
- package/build/dist/Models/DatabaseModels/StatusPageMonitorRule.js.map +1 -0
- package/build/dist/Models/DatabaseModels/StatusPageResource.js +72 -0
- package/build/dist/Models/DatabaseModels/StatusPageResource.js.map +1 -1
- package/build/dist/Models/DatabaseModels/TraceDropFilter.js +9 -20
- package/build/dist/Models/DatabaseModels/TraceDropFilter.js.map +1 -1
- package/build/dist/Server/API/BaseAPI.js +11 -1
- package/build/dist/Server/API/BaseAPI.js.map +1 -1
- package/build/dist/Server/API/DashboardAPI.js +39 -0
- package/build/dist/Server/API/DashboardAPI.js.map +1 -1
- package/build/dist/Server/EnvironmentConfig.js +27 -1
- package/build/dist/Server/EnvironmentConfig.js.map +1 -1
- package/build/dist/Server/Infrastructure/GlobalCache.js +91 -0
- package/build/dist/Server/Infrastructure/GlobalCache.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/DataSourceOptions.js +5 -3
- package/build/dist/Server/Infrastructure/Postgres/DataSourceOptions.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1786005052769-AddStatusPageMonitorRule.js +48 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1786005052769-AddStatusPageMonitorRule.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +2 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/AlertStateTimelineService.js +19 -0
- package/build/dist/Server/Services/AlertStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/CephClusterService.js +26 -76
- package/build/dist/Server/Services/CephClusterService.js.map +1 -1
- package/build/dist/Server/Services/CloudResourceService.js +23 -73
- package/build/dist/Server/Services/CloudResourceService.js.map +1 -1
- package/build/dist/Server/Services/DatabaseService.js +86 -0
- package/build/dist/Server/Services/DatabaseService.js.map +1 -1
- package/build/dist/Server/Services/DockerHostService.js +19 -69
- package/build/dist/Server/Services/DockerHostService.js.map +1 -1
- package/build/dist/Server/Services/DockerSwarmClusterService.js +27 -78
- package/build/dist/Server/Services/DockerSwarmClusterService.js.map +1 -1
- package/build/dist/Server/Services/HostService.js +58 -102
- package/build/dist/Server/Services/HostService.js.map +1 -1
- package/build/dist/Server/Services/IncidentStateTimelineService.js +19 -0
- package/build/dist/Server/Services/IncidentStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/Index.js +2 -0
- package/build/dist/Server/Services/Index.js.map +1 -1
- package/build/dist/Server/Services/IoTFleetService.js +19 -69
- package/build/dist/Server/Services/IoTFleetService.js.map +1 -1
- package/build/dist/Server/Services/KubernetesClusterService.js +17 -68
- package/build/dist/Server/Services/KubernetesClusterService.js.map +1 -1
- package/build/dist/Server/Services/LabelService.js +42 -7
- package/build/dist/Server/Services/LabelService.js.map +1 -1
- package/build/dist/Server/Services/MetricTypeService.js +74 -0
- package/build/dist/Server/Services/MetricTypeService.js.map +1 -1
- package/build/dist/Server/Services/MonitorService.js +81 -9
- package/build/dist/Server/Services/MonitorService.js.map +1 -1
- package/build/dist/Server/Services/MonitorStatusTimelineService.js +28 -0
- package/build/dist/Server/Services/MonitorStatusTimelineService.js.map +1 -1
- package/build/dist/Server/Services/MonitorTemplateService.js +60 -0
- package/build/dist/Server/Services/MonitorTemplateService.js.map +1 -1
- package/build/dist/Server/Services/PodmanHostService.js +19 -69
- package/build/dist/Server/Services/PodmanHostService.js.map +1 -1
- package/build/dist/Server/Services/ProxmoxClusterService.js +23 -73
- package/build/dist/Server/Services/ProxmoxClusterService.js.map +1 -1
- package/build/dist/Server/Services/RumApplicationService.js +19 -70
- package/build/dist/Server/Services/RumApplicationService.js.map +1 -1
- package/build/dist/Server/Services/ScheduledMaintenanceStateTimelineService.js +20 -0
- package/build/dist/Server/Services/ScheduledMaintenanceStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/ServerlessFunctionService.js +24 -75
- package/build/dist/Server/Services/ServerlessFunctionService.js.map +1 -1
- package/build/dist/Server/Services/ServiceService.js +236 -61
- package/build/dist/Server/Services/ServiceService.js.map +1 -1
- package/build/dist/Server/Services/StatusPageMonitorRuleEngineService.js +603 -0
- package/build/dist/Server/Services/StatusPageMonitorRuleEngineService.js.map +1 -0
- package/build/dist/Server/Services/StatusPageMonitorRuleService.js +401 -0
- package/build/dist/Server/Services/StatusPageMonitorRuleService.js.map +1 -0
- package/build/dist/Server/Services/TelemetryEntityRelationshipService.js +1 -0
- package/build/dist/Server/Services/TelemetryEntityRelationshipService.js.map +1 -1
- package/build/dist/Server/Services/TelemetryEntityService.js +1 -0
- package/build/dist/Server/Services/TelemetryEntityService.js.map +1 -1
- package/build/dist/Server/Utils/Database/ProjectScopedReferenceValidator.js +105 -29
- package/build/dist/Server/Utils/Database/ProjectScopedReferenceValidator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorResource.js +56 -32
- package/build/dist/Server/Utils/Monitor/MonitorResource.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStatusTimeline.js +30 -3
- package/build/dist/Server/Utils/Monitor/MonitorStatusTimeline.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStepsProjectValidator.js +110 -98
- package/build/dist/Server/Utils/Monitor/MonitorStepsProjectValidator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStepsReferenceExtractor.js +312 -0
- package/build/dist/Server/Utils/Monitor/MonitorStepsReferenceExtractor.js.map +1 -0
- package/build/dist/Server/Utils/SingleFlight.js +76 -0
- package/build/dist/Server/Utils/SingleFlight.js.map +1 -0
- package/build/dist/Server/Utils/StatusPage/MonitorRulePatternValidator.js +29 -0
- package/build/dist/Server/Utils/StatusPage/MonitorRulePatternValidator.js.map +1 -0
- package/build/dist/Server/Utils/Telemetry/EntityRegistry.js +78 -19
- package/build/dist/Server/Utils/Telemetry/EntityRegistry.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/ResourceHeartbeat.js +160 -0
- package/build/dist/Server/Utils/Telemetry/ResourceHeartbeat.js.map +1 -0
- package/build/dist/Server/Utils/Telemetry/Telemetry.js +279 -104
- package/build/dist/Server/Utils/Telemetry/Telemetry.js.map +1 -1
- package/build/dist/Types/Dashboard/DashboardComponentType.js +1 -0
- package/build/dist/Types/Dashboard/DashboardComponentType.js.map +1 -1
- package/build/dist/Types/Dashboard/DashboardComponents/ComponentArgument.js +1 -0
- package/build/dist/Types/Dashboard/DashboardComponents/ComponentArgument.js.map +1 -1
- package/build/dist/Types/Dashboard/DashboardComponents/DashboardNetworkMapComponent.js +2 -0
- package/build/dist/Types/Dashboard/DashboardComponents/DashboardNetworkMapComponent.js.map +1 -0
- package/build/dist/Types/Dashboard/DashboardTemplates.js +368 -0
- package/build/dist/Types/Dashboard/DashboardTemplates.js.map +1 -1
- package/build/dist/Types/Database/BigIntColumnTransformer.js +66 -0
- package/build/dist/Types/Database/BigIntColumnTransformer.js.map +1 -0
- package/build/dist/Types/Database/DatabaseProperty.js +25 -0
- package/build/dist/Types/Database/DatabaseProperty.js.map +1 -1
- package/build/dist/Types/Database/NumericColumnValue.js +97 -0
- package/build/dist/Types/Database/NumericColumnValue.js.map +1 -0
- package/build/dist/Types/Decimal.js +14 -3
- package/build/dist/Types/Decimal.js.map +1 -1
- package/build/dist/Types/Permission.js +42 -0
- package/build/dist/Types/Permission.js.map +1 -1
- package/build/dist/Types/Port.js +15 -5
- package/build/dist/Types/Port.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/useLiveLogsRefresh.js +44 -0
- package/build/dist/UI/Components/LogsViewer/useLiveLogsRefresh.js.map +1 -0
- package/build/dist/UI/Components/LogsViewer/useLogsHistogram.js +75 -0
- package/build/dist/UI/Components/LogsViewer/useLogsHistogram.js.map +1 -0
- package/build/dist/Utils/Dashboard/Components/DashboardNetworkMapComponent.js +114 -0
- package/build/dist/Utils/Dashboard/Components/DashboardNetworkMapComponent.js.map +1 -0
- package/build/dist/Utils/Dashboard/Components/Index.js +4 -0
- package/build/dist/Utils/Dashboard/Components/Index.js.map +1 -1
- package/package.json +1 -1
|
@@ -18,12 +18,23 @@ import ObjectID from "../../Types/ObjectID";
|
|
|
18
18
|
import OneUptimeDate from "../../Types/Date";
|
|
19
19
|
import Model from "../../Models/DatabaseModels/Service";
|
|
20
20
|
import GlobalCache from "../Infrastructure/GlobalCache";
|
|
21
|
+
import SingleFlight from "../Utils/SingleFlight";
|
|
21
22
|
import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
|
|
22
23
|
import logger from "../Utils/Logger";
|
|
23
24
|
import crypto from "crypto";
|
|
24
25
|
const DEFAULT_TELEMETRY_RETENTION_IN_DAYS = 15;
|
|
25
26
|
const LAST_SEEN_CACHE_NAMESPACE = "service-last-seen";
|
|
26
27
|
const LAST_SEEN_THROTTLE_SECONDS = 60;
|
|
28
|
+
/*
|
|
29
|
+
* Enrichment (the descriptive, collector-supplied columns) is gated
|
|
30
|
+
* separately from liveness — see the long note on updateLastSeen. The
|
|
31
|
+
* fingerprint key detects a genuine change; the write-window key bounds how
|
|
32
|
+
* often a service whose producers permanently disagree can act on one.
|
|
33
|
+
*/
|
|
34
|
+
const METADATA_FINGERPRINT_CACHE_NAMESPACE = "service-metadata-fingerprint";
|
|
35
|
+
const METADATA_FINGERPRINT_TTL_SECONDS = 60 * 60;
|
|
36
|
+
const METADATA_WRITE_WINDOW_CACHE_NAMESPACE = "service-metadata-write-window";
|
|
37
|
+
const METADATA_WRITE_THROTTLE_SECONDS = 5 * 60;
|
|
27
38
|
const LABELS_APPLIED_CACHE_NAMESPACE = "service-labels-applied";
|
|
28
39
|
const LABELS_APPLIED_CACHE_TTL_SECONDS = 60;
|
|
29
40
|
export class Service extends DatabaseService {
|
|
@@ -98,83 +109,247 @@ export class Service extends DatabaseService {
|
|
|
98
109
|
return DEFAULT_TELEMETRY_RETENTION_IN_DAYS;
|
|
99
110
|
}
|
|
100
111
|
/*
|
|
101
|
-
* Refresh `lastSeenAt`
|
|
102
|
-
*
|
|
103
|
-
* re-resolves the same serviceId
|
|
104
|
-
*
|
|
112
|
+
* Refresh `lastSeenAt` (and, occasionally, the descriptive columns) for a
|
|
113
|
+
* service. This is the single hottest UPDATE in the system: every
|
|
114
|
+
* metric/log/trace/profile batch re-resolves the same serviceId, and every
|
|
115
|
+
* OTLP resource inside a batch does it again.
|
|
116
|
+
*
|
|
117
|
+
* It is also the write that took production down, so the throttling here is
|
|
118
|
+
* deliberate in a way worth reading before changing.
|
|
119
|
+
*
|
|
120
|
+
* WHAT WENT WRONG. The throttle used to key on the serviceId but STORE a
|
|
121
|
+
* fingerprint of the metadata, and skip only when the stored fingerprint
|
|
122
|
+
* matched. That works when one producer owns a row. It collapses when two
|
|
123
|
+
* producers of the SAME row disagree, because `Service` is unique on
|
|
124
|
+
* (projectId, name) alone: the same service.name running in dev and prod
|
|
125
|
+
* inside one project is ONE row with two permanently disagreeing
|
|
126
|
+
* fingerprints. A writes fp_A, B reads fp_A, sees a mismatch, writes fp_B, A
|
|
127
|
+
* reads fp_B, sees a mismatch... forever. The same holds for active-active
|
|
128
|
+
* deployments differing in cloud.region or cloud.account.id, and transiently
|
|
129
|
+
* for every rolling deploy (service.version). For those rows the throttle
|
|
130
|
+
* admitted not one write per minute but one write per BATCH — and with the
|
|
131
|
+
* autoscaler at 100 pods (~25K concurrent jobs) that was thousands of
|
|
132
|
+
* simultaneous UPDATEs onto ~2,300 rows. Postgres queues row-lock waiters
|
|
133
|
+
* strictly, so they did not collide and retry, they lined up: 1,017 active
|
|
134
|
+
* connections, 892 parked on locks, the tail waiting 3.7 hours. The database
|
|
135
|
+
* was never short of capacity.
|
|
136
|
+
*
|
|
137
|
+
* The shape below separates the two things that were tangled together:
|
|
138
|
+
*
|
|
139
|
+
* LIVENESS (`lastSeenAt`) is gated on key PRESENCE, never on metadata. One
|
|
140
|
+
* heartbeat per service per window, full stop — no payload can bust it, so
|
|
141
|
+
* disagreeing producers cannot flap it.
|
|
142
|
+
*
|
|
143
|
+
* ENRICHMENT (the descriptive columns) is gated on TWO keys: a fingerprint
|
|
144
|
+
* claim so a genuinely changed attribute still lands promptly, plus a
|
|
145
|
+
* presence-only rate limiter so a permanently-flapping service costs one
|
|
146
|
+
* enrichment write per rate-limit window instead of one per batch. The
|
|
147
|
+
* fingerprint key alone does not fix flapping — a longer TTL just makes
|
|
148
|
+
* each flap more expensive.
|
|
149
|
+
*
|
|
150
|
+
* Both claims are ATOMIC (SET NX / compare-and-claim in one round trip).
|
|
151
|
+
* The old read-then-write was check-then-act, and at this concurrency the
|
|
152
|
+
* losers of that race do not politely retry — they all proceed.
|
|
153
|
+
*
|
|
154
|
+
* Underneath both, the write itself is non-blocking (SKIP LOCKED), so a
|
|
155
|
+
* convoy cannot form even if every gate above fails open at once — which is
|
|
156
|
+
* exactly what a Redis outage does. That layering is what makes fail-open
|
|
157
|
+
* affordable here, and fail-open is required: a suppressed heartbeat strands
|
|
158
|
+
* a healthy service as "disconnected" while its telemetry keeps arriving.
|
|
105
159
|
*/
|
|
106
160
|
async updateLastSeen(serviceId, extra) {
|
|
161
|
+
const cacheKey = serviceId.toString();
|
|
107
162
|
/*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* immediately. With no extras the fingerprint is constant, preserving
|
|
113
|
-
* the original "refresh lastSeenAt at most once per window" behaviour.
|
|
163
|
+
* Collapse duplicate concurrent callers inside THIS process first. One
|
|
164
|
+
* OTLP batch resolves the same service once per resource — the shipped
|
|
165
|
+
* host-metrics collector config produces hundreds per batch — and without
|
|
166
|
+
* this every one of them independently asks Redis the same question.
|
|
114
167
|
*/
|
|
115
|
-
|
|
168
|
+
await SingleFlight.run(`${LAST_SEEN_CACHE_NAMESPACE}:${cacheKey}:${this.fingerprintLastSeenExtras(extra)}`, async () => {
|
|
169
|
+
await this.writeLastSeen(serviceId, cacheKey, extra);
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
async writeLastSeen(serviceId, cacheKey, extra) {
|
|
116
173
|
const fingerprint = this.fingerprintLastSeenExtras(extra);
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
174
|
+
/*
|
|
175
|
+
* Liveness gate. Presence-only and jittered: a fleet-wide restart would
|
|
176
|
+
* otherwise re-synchronise every service's window and rebuild the herd on
|
|
177
|
+
* a one-minute period.
|
|
178
|
+
*/
|
|
179
|
+
let writeLiveness = true;
|
|
180
|
+
try {
|
|
181
|
+
writeLiveness = await GlobalCache.setStringIfNotExists(LAST_SEEN_CACHE_NAMESPACE, cacheKey, "1", {
|
|
182
|
+
expiresInSeconds: GlobalCache.withJitter(LAST_SEEN_THROTTLE_SECONDS),
|
|
183
|
+
});
|
|
120
184
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
data.serviceVersion = extra.serviceVersion;
|
|
185
|
+
catch (_a) {
|
|
186
|
+
/*
|
|
187
|
+
* Cache unavailable — fail OPEN and refresh liveness anyway. Suppressing
|
|
188
|
+
* the heartbeat during a Redis outage is how a healthy service gets
|
|
189
|
+
* shown as disconnected while its data keeps arriving (issue #3006's
|
|
190
|
+
* failure mode). Affordable only because the write below cannot queue.
|
|
191
|
+
*/
|
|
192
|
+
writeLiveness = true;
|
|
130
193
|
}
|
|
131
|
-
|
|
132
|
-
|
|
194
|
+
/*
|
|
195
|
+
* Enrichment gate 1: has the payload actually changed? Compare-and-claim,
|
|
196
|
+
* so a deploy's new service.version is written promptly rather than
|
|
197
|
+
* waiting out a window.
|
|
198
|
+
*/
|
|
199
|
+
let metadataChanged = false;
|
|
200
|
+
try {
|
|
201
|
+
metadataChanged = await GlobalCache.setStringIfChanged(METADATA_FINGERPRINT_CACHE_NAMESPACE, cacheKey, fingerprint, {
|
|
202
|
+
expiresInSeconds: GlobalCache.withJitter(METADATA_FINGERPRINT_TTL_SECONDS),
|
|
203
|
+
});
|
|
133
204
|
}
|
|
134
|
-
|
|
135
|
-
|
|
205
|
+
catch (_b) {
|
|
206
|
+
/*
|
|
207
|
+
* Fail CLOSED for enrichment (unlike liveness). These columns are
|
|
208
|
+
* descriptive, not operational — nothing is mis-reported by writing them
|
|
209
|
+
* a little later, and letting a Redis outage open this gate would
|
|
210
|
+
* restore the per-batch write storm this method exists to prevent.
|
|
211
|
+
*/
|
|
212
|
+
metadataChanged = false;
|
|
136
213
|
}
|
|
137
|
-
|
|
138
|
-
|
|
214
|
+
/*
|
|
215
|
+
* Enrichment gate 2: rate limiter. A service whose producers permanently
|
|
216
|
+
* disagree (dev + prod under one name, active-active regions) reports a
|
|
217
|
+
* "change" on every batch forever. Gate 1 cannot tell that apart from a
|
|
218
|
+
* real deploy, so this bounds it regardless.
|
|
219
|
+
*/
|
|
220
|
+
let writeMetadata = false;
|
|
221
|
+
if (metadataChanged) {
|
|
222
|
+
try {
|
|
223
|
+
writeMetadata = await GlobalCache.setStringIfNotExists(METADATA_WRITE_WINDOW_CACHE_NAMESPACE, cacheKey, "1", {
|
|
224
|
+
expiresInSeconds: GlobalCache.withJitter(METADATA_WRITE_THROTTLE_SECONDS),
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
catch (_c) {
|
|
228
|
+
writeMetadata = false;
|
|
229
|
+
}
|
|
139
230
|
}
|
|
140
|
-
if (
|
|
141
|
-
|
|
231
|
+
if (!writeLiveness && !writeMetadata) {
|
|
232
|
+
// Nothing to say. Issue no statement at all.
|
|
233
|
+
return;
|
|
142
234
|
}
|
|
143
|
-
|
|
144
|
-
|
|
235
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
236
|
+
const liveness = {
|
|
237
|
+
lastSeenAt: OneUptimeDate.getCurrentDate(),
|
|
238
|
+
};
|
|
239
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
240
|
+
const metadata = {};
|
|
241
|
+
if (writeMetadata) {
|
|
242
|
+
if (extra === null || extra === void 0 ? void 0 : extra.serviceVersion) {
|
|
243
|
+
metadata.serviceVersion = extra.serviceVersion;
|
|
244
|
+
}
|
|
245
|
+
if (extra === null || extra === void 0 ? void 0 : extra.deploymentEnvironment) {
|
|
246
|
+
metadata.deploymentEnvironment = extra.deploymentEnvironment;
|
|
247
|
+
}
|
|
248
|
+
if (extra === null || extra === void 0 ? void 0 : extra.serviceNamespace) {
|
|
249
|
+
metadata.serviceNamespace = extra.serviceNamespace;
|
|
250
|
+
}
|
|
251
|
+
if (extra === null || extra === void 0 ? void 0 : extra.runtimeName) {
|
|
252
|
+
metadata.runtimeName = extra.runtimeName;
|
|
253
|
+
}
|
|
254
|
+
if (extra === null || extra === void 0 ? void 0 : extra.runtimeVersion) {
|
|
255
|
+
metadata.runtimeVersion = extra.runtimeVersion;
|
|
256
|
+
}
|
|
257
|
+
if (extra === null || extra === void 0 ? void 0 : extra.telemetrySdkLanguage) {
|
|
258
|
+
metadata.telemetrySdkLanguage = extra.telemetrySdkLanguage;
|
|
259
|
+
}
|
|
260
|
+
if (extra === null || extra === void 0 ? void 0 : extra.cloudProvider) {
|
|
261
|
+
metadata.cloudProvider = extra.cloudProvider;
|
|
262
|
+
}
|
|
263
|
+
if (extra === null || extra === void 0 ? void 0 : extra.cloudPlatform) {
|
|
264
|
+
metadata.cloudPlatform = extra.cloudPlatform;
|
|
265
|
+
}
|
|
266
|
+
if (extra === null || extra === void 0 ? void 0 : extra.cloudRegion) {
|
|
267
|
+
metadata.cloudRegion = extra.cloudRegion;
|
|
268
|
+
}
|
|
269
|
+
if (extra === null || extra === void 0 ? void 0 : extra.cloudAccountId) {
|
|
270
|
+
metadata.cloudAccountId = extra.cloudAccountId;
|
|
271
|
+
}
|
|
145
272
|
}
|
|
146
|
-
|
|
147
|
-
|
|
273
|
+
const hasMetadata = Object.keys(metadata).length > 0;
|
|
274
|
+
/*
|
|
275
|
+
* `lastSeenAt` rides along even on an enrichment-only write: refreshing it
|
|
276
|
+
* early is harmless (it is a liveness floor, not a sample), and splitting
|
|
277
|
+
* the two into separate statements would double the writes we just went to
|
|
278
|
+
* such lengths to halve.
|
|
279
|
+
*/
|
|
280
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
281
|
+
const data = Object.assign(Object.assign({}, liveness), metadata);
|
|
282
|
+
/*
|
|
283
|
+
* Heartbeat write. Deliberately NOT the full updateOneById pipeline
|
|
284
|
+
* (permission SELECT -> _findBy SELECT -> save() with `version = version +
|
|
285
|
+
* 1` -> workflow/realtime/audit hooks): that pipeline holds the row's
|
|
286
|
+
* write lock across several round trips inside an open transaction, which
|
|
287
|
+
* is precisely how a stalled event loop becomes the head of a lock convoy.
|
|
288
|
+
* It also intentionally drops the per-update workflow trigger that
|
|
289
|
+
* Service's @EnableWorkflow({ update: true }) fired on every heartbeat — a
|
|
290
|
+
* liveness ping should not trigger user workflows.
|
|
291
|
+
*
|
|
292
|
+
* And it is the SKIP LOCKED variant: contended writers yield instead of
|
|
293
|
+
* queueing, capping backends blocked on any one Service row at one.
|
|
294
|
+
*/
|
|
295
|
+
let wrote = false;
|
|
296
|
+
try {
|
|
297
|
+
wrote = await this.updateColumnsByIdIfUnlockedWithoutHooks({
|
|
298
|
+
id: serviceId,
|
|
299
|
+
data: data,
|
|
300
|
+
});
|
|
148
301
|
}
|
|
149
|
-
|
|
150
|
-
|
|
302
|
+
catch (err) {
|
|
303
|
+
/*
|
|
304
|
+
* Liveness must survive bad metadata. Every enrichment value is a
|
|
305
|
+
* collector-supplied resource attribute, so if one of them makes
|
|
306
|
+
* Postgres reject the statement it takes `lastSeenAt` down with it and
|
|
307
|
+
* the service silently stops looking alive. Retry with liveness only, so
|
|
308
|
+
* enrichment is what gets dropped, never the heartbeat. (Issue #3006 was
|
|
309
|
+
* this exact failure on Host.)
|
|
310
|
+
*/
|
|
311
|
+
logger.warn(`ServiceService.updateLastSeen: metadata write failed for service ${serviceId.toString()}, falling back to a liveness-only update: ${err instanceof Error ? err.message : String(err)}`);
|
|
312
|
+
await this.clearLastSeenGates(cacheKey, hasMetadata);
|
|
313
|
+
await this.updateColumnsByIdIfUnlockedWithoutHooks({
|
|
314
|
+
id: serviceId,
|
|
315
|
+
data: liveness,
|
|
316
|
+
});
|
|
317
|
+
return;
|
|
151
318
|
}
|
|
152
|
-
if (
|
|
153
|
-
|
|
319
|
+
if (!wrote && hasMetadata) {
|
|
320
|
+
/*
|
|
321
|
+
* The row was locked, so this write was SKIPPED rather than applied. For
|
|
322
|
+
* liveness that is a non-event — whoever holds the lock is storing the
|
|
323
|
+
* same timestamp. Enrichment is different: the holder may be writing an
|
|
324
|
+
* older payload, and the gates we just claimed would suppress ours for
|
|
325
|
+
* the rest of the window. Release them so the next batch re-attempts.
|
|
326
|
+
*
|
|
327
|
+
* This cannot become a retry storm — each attempt is a non-blocking
|
|
328
|
+
* statement that returns immediately when contended.
|
|
329
|
+
*/
|
|
330
|
+
await this.clearLastSeenGates(cacheKey, true);
|
|
154
331
|
}
|
|
155
|
-
|
|
156
|
-
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Re-open the gates so the next batch retries. Liveness is only re-opened
|
|
335
|
+
* when there was no enrichment at stake; otherwise the enrichment gates are
|
|
336
|
+
* the ones that must not stay claimed after a write that never happened.
|
|
337
|
+
*/
|
|
338
|
+
async clearLastSeenGates(cacheKey, hadMetadata) {
|
|
339
|
+
const namespaces = hadMetadata
|
|
340
|
+
? [
|
|
341
|
+
METADATA_FINGERPRINT_CACHE_NAMESPACE,
|
|
342
|
+
METADATA_WRITE_WINDOW_CACHE_NAMESPACE,
|
|
343
|
+
]
|
|
344
|
+
: [LAST_SEEN_CACHE_NAMESPACE];
|
|
345
|
+
for (const namespace of namespaces) {
|
|
346
|
+
try {
|
|
347
|
+
await GlobalCache.deleteKey(namespace, cacheKey);
|
|
348
|
+
}
|
|
349
|
+
catch (_a) {
|
|
350
|
+
// Best effort — the retry below is what actually matters.
|
|
351
|
+
}
|
|
157
352
|
}
|
|
158
|
-
/*
|
|
159
|
-
* Heartbeat write. This is the single hottest UPDATE in the system —
|
|
160
|
-
* every telemetry batch for a service re-resolves the same serviceId —
|
|
161
|
-
* so it must NOT go through the full updateOneById pipeline (permission
|
|
162
|
-
* SELECT -> _findBy SELECT -> save() with `version = version + 1` ->
|
|
163
|
-
* workflow/realtime/audit hooks). That pipeline holds the Service row's
|
|
164
|
-
* write lock across several round trips and, when an event loop stalls
|
|
165
|
-
* mid-`save()` transaction, leaves it open as the head of a lock convoy
|
|
166
|
-
* that starves the Postgres connection pool. lastSeenAt is an internal
|
|
167
|
-
* liveness timestamp with no onBeforeUpdate/onUpdateSuccess hooks on this
|
|
168
|
-
* model (connected / disconnected status is derived from it at read
|
|
169
|
-
* time), so a bare single-statement UPDATE is both correct and far
|
|
170
|
-
* cheaper. This also intentionally drops the per-update workflow trigger
|
|
171
|
-
* Service's @EnableWorkflow({ update: true }) fired on every heartbeat —
|
|
172
|
-
* a liveness ping should not trigger user workflows.
|
|
173
|
-
*/
|
|
174
|
-
await this.updateColumnsByIdWithoutHooks({
|
|
175
|
-
id: serviceId,
|
|
176
|
-
data: data,
|
|
177
|
-
});
|
|
178
353
|
}
|
|
179
354
|
fingerprintLastSeenExtras(extra) {
|
|
180
355
|
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ServiceService.js","sourceRoot":"","sources":["../../../../Server/Services/ServiceService.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,eAAe,MAAM,mBAAmB,CAAC;AAChD,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAC9C,OAAO,6BAA6B,MAAM,iCAAiC,CAAC;AAC5E,OAAO,6BAA6B,MAAM,iCAAiC,CAAC;AAC5E,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,gBAAgB,MAAM,wCAAwC,CAAC;AACtE,OAAO,QAAQ,MAAM,sBAAsB,CAAC;AAC5C,OAAO,aAAa,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,MAAM,qCAAqC,CAAC;AAGxD,OAAO,WAAW,MAAM,+BAA+B,CAAC;AACxD,OAAO,WAAW,MAAM,gCAAgC,CAAC;AACzD,OAAO,MAAyB,MAAM,iBAAiB,CAAC;AACxD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,mCAAmC,GAAW,EAAE,CAAC;AAEvD,MAAM,yBAAyB,GAAW,mBAAmB,CAAC;AAC9D,MAAM,0BAA0B,GAAW,EAAE,CAAC;AAE9C,MAAM,8BAA8B,GAAW,wBAAwB,CAAC;AACxE,MAAM,gCAAgC,GAAW,EAAE,CAAC;AAEpD,MAAM,OAAO,OAAQ,SAAQ,eAAsB;IACjD;QACE,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,CAAC;IAGwB,AAAN,KAAK,CAAC,cAAc,CACrC,QAAyB;QAEzB;;;;WAIG;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO;YACL,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,QAAQ;SACnB,CAAC;IACJ,CAAC;IAGwB,AAAN,KAAK,CAAC,eAAe,CACtC,SAA0B,EAC1B,WAAkB;QAElB,IAAI,WAAW,CAAC,SAAS,IAAI,WAAW,CAAC,EAAE,EAAE,CAAC;YAC5C,OAAO,CAAC,OAAO,EAAE;iBACd,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,6BAA6B,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;YACvE,CAAC,CAAC;iBACD,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,6BAA6B,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;YACvE,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;;gBACtB,MAAM,CAAC,KAAK,CACV,mEAAmE,KAAK,EAAE,EAC1E;oBACE,SAAS,EAAE,MAAA,WAAW,CAAC,SAAS,0CAAE,QAAQ,EAAE;oBAC5C,SAAS,EAAE,MAAA,WAAW,CAAC,EAAE,0CAAE,QAAQ,EAAE;iBACrB,CACnB,CAAC;YACJ,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAGY,AAAN,KAAK,CAAC,+BAA+B,CAC1C,SAAmB;QAEnB,MAAM,OAAO,GAAiB,MAAM,IAAI,CAAC,WAAW,CAAC;YACnD,EAAE,EAAE,SAAS;YACb,MAAM,EAAE;gBACN,SAAS,EAAE,IAAI;gBACf,0BAA0B,EAAE,IAAI;aACjC;YACD,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,0BAA0B,CAAC;QAC5C,CAAC;QAED,sCAAsC;QACtC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,OAAO,GAAmB,MAAM,cAAc,CAAC,WAAW,CAAC;gBAC/D,EAAE,EAAE,OAAO,CAAC,SAAS;gBACrB,MAAM,EAAE;oBACN,+BAA+B,EAAE,IAAI;iBACtC;gBACD,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEH,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,+BAA+B,EAAE,CAAC;gBAC7C,OAAO,OAAO,CAAC,+BAA+B,CAAC;YACjD,CAAC;QACH,CAAC;QAED,OAAO,mCAAmC,CAAC;IAC7C,CAAC;IAED
|
|
1
|
+
{"version":3,"file":"ServiceService.js","sourceRoot":"","sources":["../../../../Server/Services/ServiceService.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,eAAe,MAAM,mBAAmB,CAAC;AAChD,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAC9C,OAAO,6BAA6B,MAAM,iCAAiC,CAAC;AAC5E,OAAO,6BAA6B,MAAM,iCAAiC,CAAC;AAC5E,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,gBAAgB,MAAM,wCAAwC,CAAC;AACtE,OAAO,QAAQ,MAAM,sBAAsB,CAAC;AAC5C,OAAO,aAAa,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,MAAM,qCAAqC,CAAC;AAGxD,OAAO,WAAW,MAAM,+BAA+B,CAAC;AACxD,OAAO,YAAY,MAAM,uBAAuB,CAAC;AACjD,OAAO,WAAW,MAAM,gCAAgC,CAAC;AACzD,OAAO,MAAyB,MAAM,iBAAiB,CAAC;AACxD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,mCAAmC,GAAW,EAAE,CAAC;AAEvD,MAAM,yBAAyB,GAAW,mBAAmB,CAAC;AAC9D,MAAM,0BAA0B,GAAW,EAAE,CAAC;AAE9C;;;;;GAKG;AACH,MAAM,oCAAoC,GACxC,8BAA8B,CAAC;AACjC,MAAM,gCAAgC,GAAW,EAAE,GAAG,EAAE,CAAC;AAEzD,MAAM,qCAAqC,GACzC,+BAA+B,CAAC;AAClC,MAAM,+BAA+B,GAAW,CAAC,GAAG,EAAE,CAAC;AAEvD,MAAM,8BAA8B,GAAW,wBAAwB,CAAC;AACxE,MAAM,gCAAgC,GAAW,EAAE,CAAC;AAEpD,MAAM,OAAO,OAAQ,SAAQ,eAAsB;IACjD;QACE,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,CAAC;IAGwB,AAAN,KAAK,CAAC,cAAc,CACrC,QAAyB;QAEzB;;;;WAIG;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO;YACL,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,QAAQ;SACnB,CAAC;IACJ,CAAC;IAGwB,AAAN,KAAK,CAAC,eAAe,CACtC,SAA0B,EAC1B,WAAkB;QAElB,IAAI,WAAW,CAAC,SAAS,IAAI,WAAW,CAAC,EAAE,EAAE,CAAC;YAC5C,OAAO,CAAC,OAAO,EAAE;iBACd,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,6BAA6B,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;YACvE,CAAC,CAAC;iBACD,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,6BAA6B,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;YACvE,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;;gBACtB,MAAM,CAAC,KAAK,CACV,mEAAmE,KAAK,EAAE,EAC1E;oBACE,SAAS,EAAE,MAAA,WAAW,CAAC,SAAS,0CAAE,QAAQ,EAAE;oBAC5C,SAAS,EAAE,MAAA,WAAW,CAAC,EAAE,0CAAE,QAAQ,EAAE;iBACrB,CACnB,CAAC;YACJ,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAGY,AAAN,KAAK,CAAC,+BAA+B,CAC1C,SAAmB;QAEnB,MAAM,OAAO,GAAiB,MAAM,IAAI,CAAC,WAAW,CAAC;YACnD,EAAE,EAAE,SAAS;YACb,MAAM,EAAE;gBACN,SAAS,EAAE,IAAI;gBACf,0BAA0B,EAAE,IAAI;aACjC;YACD,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,0BAA0B,CAAC;QAC5C,CAAC;QAED,sCAAsC;QACtC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,OAAO,GAAmB,MAAM,cAAc,CAAC,WAAW,CAAC;gBAC/D,EAAE,EAAE,OAAO,CAAC,SAAS;gBACrB,MAAM,EAAE;oBACN,+BAA+B,EAAE,IAAI;iBACtC;gBACD,KAAK,EAAE;oBACL,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEH,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,+BAA+B,EAAE,CAAC;gBAC7C,OAAO,OAAO,CAAC,+BAA+B,CAAC;YACjD,CAAC;QACH,CAAC;QAED,OAAO,mCAAmC,CAAC;IAC7C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgDG;IAEU,AAAN,KAAK,CAAC,cAAc,CACzB,SAAmB,EACnB,KAWC;QAED,MAAM,QAAQ,GAAW,SAAS,CAAC,QAAQ,EAAE,CAAC;QAE9C;;;;;WAKG;QACH,MAAM,YAAY,CAAC,GAAG,CACpB,GAAG,yBAAyB,IAAI,QAAQ,IAAI,IAAI,CAAC,yBAAyB,CACxE,KAAK,CACN,EAAE,EACH,KAAK,IAAI,EAAE;YACT,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC,CACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,SAAmB,EACnB,QAAgB,EAChB,KAWC;QAED,MAAM,WAAW,GAAW,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC;QAElE;;;;WAIG;QACH,IAAI,aAAa,GAAY,IAAI,CAAC;QAClC,IAAI,CAAC;YACH,aAAa,GAAG,MAAM,WAAW,CAAC,oBAAoB,CACpD,yBAAyB,EACzB,QAAQ,EACR,GAAG,EACH;gBACE,gBAAgB,EAAE,WAAW,CAAC,UAAU,CAAC,0BAA0B,CAAC;aACrE,CACF,CAAC;QACJ,CAAC;QAAC,WAAM,CAAC;YACP;;;;;eAKG;YACH,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;QAED;;;;WAIG;QACH,IAAI,eAAe,GAAY,KAAK,CAAC;QACrC,IAAI,CAAC;YACH,eAAe,GAAG,MAAM,WAAW,CAAC,kBAAkB,CACpD,oCAAoC,EACpC,QAAQ,EACR,WAAW,EACX;gBACE,gBAAgB,EAAE,WAAW,CAAC,UAAU,CACtC,gCAAgC,CACjC;aACF,CACF,CAAC;QACJ,CAAC;QAAC,WAAM,CAAC;YACP;;;;;eAKG;YACH,eAAe,GAAG,KAAK,CAAC;QAC1B,CAAC;QAED;;;;;WAKG;QACH,IAAI,aAAa,GAAY,KAAK,CAAC;QACnC,IAAI,eAAe,EAAE,CAAC;YACpB,IAAI,CAAC;gBACH,aAAa,GAAG,MAAM,WAAW,CAAC,oBAAoB,CACpD,qCAAqC,EACrC,QAAQ,EACR,GAAG,EACH;oBACE,gBAAgB,EAAE,WAAW,CAAC,UAAU,CACtC,+BAA+B,CAChC;iBACF,CACF,CAAC;YACJ,CAAC;YAAC,WAAM,CAAC;gBACP,aAAa,GAAG,KAAK,CAAC;YACxB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,6CAA6C;YAC7C,OAAO;QACT,CAAC;QAED,8DAA8D;QAC9D,MAAM,QAAQ,GAAQ;YACpB,UAAU,EAAE,aAAa,CAAC,cAAc,EAAE;SAC3C,CAAC;QAEF,8DAA8D;QAC9D,MAAM,QAAQ,GAAQ,EAAE,CAAC;QAEzB,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,cAAc,EAAE,CAAC;gBAC1B,QAAQ,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC;YACjD,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,qBAAqB,EAAE,CAAC;gBACjC,QAAQ,CAAC,qBAAqB,GAAG,KAAK,CAAC,qBAAqB,CAAC;YAC/D,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,gBAAgB,EAAE,CAAC;gBAC5B,QAAQ,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB,CAAC;YACrD,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,EAAE,CAAC;gBACvB,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;YAC3C,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,cAAc,EAAE,CAAC;gBAC1B,QAAQ,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC;YACjD,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,oBAAoB,EAAE,CAAC;gBAChC,QAAQ,CAAC,oBAAoB,GAAG,KAAK,CAAC,oBAAoB,CAAC;YAC7D,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,aAAa,EAAE,CAAC;gBACzB,QAAQ,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;YAC/C,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,aAAa,EAAE,CAAC;gBACzB,QAAQ,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;YAC/C,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,EAAE,CAAC;gBACvB,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;YAC3C,CAAC;YACD,IAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,cAAc,EAAE,CAAC;gBAC1B,QAAQ,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC;YACjD,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAY,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAE9D;;;;;WAKG;QACH,8DAA8D;QAC9D,MAAM,IAAI,mCAAa,QAAQ,GAAK,QAAQ,CAAE,CAAC;QAE/C;;;;;;;;;;;;WAYG;QACH,IAAI,KAAK,GAAY,KAAK,CAAC;QAE3B,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,IAAI,CAAC,uCAAuC,CAAC;gBACzD,EAAE,EAAE,SAAS;gBACb,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb;;;;;;;eAOG;YACH,MAAM,CAAC,IAAI,CACT,oEAAoE,SAAS,CAAC,QAAQ,EAAE,6CACtF,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;YAEF,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;YAErD,MAAM,IAAI,CAAC,uCAAuC,CAAC;gBACjD,EAAE,EAAE,SAAS;gBACb,IAAI,EAAE,QAAQ;aACf,CAAC,CAAC;YAEH,OAAO;QACT,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,WAAW,EAAE,CAAC;YAC1B;;;;;;;;;eASG;YACH,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAgB,EAChB,WAAoB;QAEpB,MAAM,UAAU,GAAkB,WAAW;YAC3C,CAAC,CAAC;gBACE,oCAAoC;gBACpC,qCAAqC;aACtC;YACH,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC;QAEhC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,WAAW,CAAC,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACnD,CAAC;YAAC,WAAM,CAAC;gBACP,0DAA0D;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IAEO,yBAAyB,CAAC,KAWjC;;QACC,MAAM,UAAU,GAAkC;YAChD,cAAc,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,cAAc,mCAAI,IAAI;YAC7C,qBAAqB,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,qBAAqB,mCAAI,IAAI;YAC3D,gBAAgB,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,gBAAgB,mCAAI,IAAI;YACjD,WAAW,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,mCAAI,IAAI;YACvC,cAAc,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,cAAc,mCAAI,IAAI;YAC7C,oBAAoB,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,oBAAoB,mCAAI,IAAI;YACzD,aAAa,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,aAAa,mCAAI,IAAI;YAC3C,aAAa,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,aAAa,mCAAI,IAAI;YAC3C,WAAW,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,mCAAI,IAAI;YACvC,cAAc,EAAE,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,cAAc,mCAAI,IAAI;SAC9C,CAAC;QAEF,OAAO,MAAM;aACV,UAAU,CAAC,MAAM,CAAC;aAClB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;aAClC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAED;;;;;;OAMG;IAEU,AAAN,KAAK,CAAC,YAAY,CAAC,IAGzB;;QACC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;QACnD,MAAM,WAAW,GAAW,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAkB,MAAM,WAAW,CAAC,SAAS,CACvD,8BAA8B,EAC9B,QAAQ,CACT,CAAC;QACF,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,YAAY,GAAW,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YACvD,MAAM,cAAc,GAAiB,MAAM,IAAI,CAAC,aAAa,EAAE;iBAC5D,kBAAkB,EAAE;iBACpB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;iBACzB,EAAE,CAAC,YAAY,CAAC;iBAChB,QAAQ,EAAE,CAAC;YAEd,MAAM,WAAW,GAAgB,IAAI,GAAG,EAAE,CAAC;YAC3C,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAuB,MAAA,GAAG,CAAC,GAAG,0CAAE,QAAQ,EAAE,CAAC;gBACtD,IAAI,KAAK,EAAE,CAAC;oBACV,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAkB,EAAE,CAAC;YACnC,MAAM,IAAI,GAAgB,IAAI,GAAG,EAAE,CAAC;YACpC,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAAW,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACpC,IAAI,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9C,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAChB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,aAAa,EAAE;qBACvB,kBAAkB,EAAE;qBACpB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;qBACzB,EAAE,CAAC,YAAY,CAAC;qBAChB,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnB,CAAC;YAED,MAAM,WAAW,CAAC,SAAS,CACzB,8BAA8B,EAC9B,QAAQ,EACR,WAAW,EACX,EAAE,gBAAgB,EAAE,gCAAgC,EAAE,CACvD,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CACT,kDAAkD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,KACzE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAhgB0B;IADxB,WAAW,EAAE;;;;6CAiBb;AAGwB;IADxB,WAAW,EAAE;;6CAGC,KAAK;;8CAqBnB;AAGY;IADZ,WAAW,EAAE;;qCAED,QAAQ;;8DAuCpB;AAoDY;IADZ,WAAW,EAAE;;qCAED,QAAQ;;6CA8BpB;AAqRY;IADZ,WAAW,EAAE;;;;2CAmEb;AAGH,SAAS,mBAAmB,CAAC,QAAyB;IACpD,MAAM,MAAM,GAAkB,QAAQ;SACnC,GAAG,CAAC,CAAC,EAAY,EAAE,EAAE;QACpB,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;IACvB,CAAC,CAAC;SACD,IAAI,EAAE,CAAC;IACV,OAAO,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1E,CAAC;AAED,eAAe,IAAI,OAAO,EAAE,CAAC"}
|