@oneuptime/common 12.0.2 → 12.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel.ts +14 -0
- package/Models/DatabaseModels/Index.ts +2 -0
- package/Models/DatabaseModels/LogDropFilter.ts +9 -20
- package/Models/DatabaseModels/StatusPageMonitorRule.ts +911 -0
- package/Models/DatabaseModels/StatusPageResource.ts +73 -0
- package/Models/DatabaseModels/TraceDropFilter.ts +9 -20
- package/Server/API/BaseAPI.ts +13 -2
- package/Server/API/DashboardAPI.ts +39 -0
- package/Server/EnvironmentConfig.ts +31 -1
- package/Server/Infrastructure/GlobalCache.ts +115 -0
- package/Server/Infrastructure/Postgres/DataSourceOptions.ts +16 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1786005052769-AddStatusPageMonitorRule.ts +123 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +2 -0
- package/Server/Services/AlertStateTimelineService.ts +20 -0
- package/Server/Services/CephClusterService.ts +26 -86
- package/Server/Services/CloudResourceService.ts +23 -83
- package/Server/Services/DatabaseService.ts +111 -0
- package/Server/Services/DockerHostService.ts +19 -79
- package/Server/Services/DockerSwarmClusterService.ts +27 -88
- package/Server/Services/HostService.ts +46 -99
- package/Server/Services/IncidentStateTimelineService.ts +20 -0
- package/Server/Services/Index.ts +2 -0
- package/Server/Services/IoTFleetService.ts +19 -79
- package/Server/Services/KubernetesClusterService.ts +17 -78
- package/Server/Services/LabelService.ts +57 -10
- package/Server/Services/MetricTypeService.ts +84 -0
- package/Server/Services/MonitorService.ts +89 -14
- package/Server/Services/MonitorStatusTimelineService.ts +29 -0
- package/Server/Services/MonitorTemplateService.ts +65 -0
- package/Server/Services/PodmanHostService.ts +19 -79
- package/Server/Services/ProxmoxClusterService.ts +23 -83
- package/Server/Services/RumApplicationService.ts +19 -80
- package/Server/Services/ScheduledMaintenanceStateTimelineService.ts +21 -0
- package/Server/Services/ServerlessFunctionService.ts +24 -85
- package/Server/Services/ServiceService.ts +299 -73
- package/Server/Services/StatusPageMonitorRuleEngineService.ts +752 -0
- package/Server/Services/StatusPageMonitorRuleService.ts +471 -0
- package/Server/Services/TelemetryEntityRelationshipService.ts +1 -0
- package/Server/Services/TelemetryEntityService.ts +1 -0
- package/Server/Utils/Database/ProjectScopedReferenceValidator.ts +170 -39
- package/Server/Utils/Monitor/MonitorResource.ts +63 -36
- package/Server/Utils/Monitor/MonitorStatusTimeline.ts +36 -0
- package/Server/Utils/Monitor/MonitorStepsProjectValidator.ts +148 -127
- package/Server/Utils/Monitor/MonitorStepsReferenceExtractor.ts +434 -0
- package/Server/Utils/SingleFlight.ts +83 -0
- package/Server/Utils/StatusPage/MonitorRulePatternValidator.ts +46 -0
- package/Server/Utils/Telemetry/EntityRegistry.ts +107 -26
- package/Server/Utils/Telemetry/ResourceHeartbeat.ts +314 -0
- package/Server/Utils/Telemetry/Telemetry.ts +351 -122
- package/Tests/App/Dashboard/WidgetCatalog.test.ts +2 -0
- package/Tests/Models/DatabaseModels/ColumnTransformerInvariants.test.ts +306 -0
- package/Tests/Models/DatabaseModels/NumberColumnDeserialization.test.ts +333 -0
- package/Tests/Models/DatabaseModels/StatusPageMonitorRuleModel.test.ts +190 -0
- package/Tests/Server/API/DashboardPublicResourceListAPI.test.ts +26 -0
- package/Tests/Server/Infrastructure/GlobalCache.test.ts +226 -0
- package/Tests/Server/Infrastructure/PostgresLockTimeoutOptions.test.ts +181 -0
- package/Tests/Server/Services/DatabaseServiceUpdateColumnsIfUnlocked.test.ts +342 -0
- package/Tests/Server/Services/DropFilterSaveValidation.test.ts +245 -0
- package/Tests/Server/Services/HostServiceUpdateLastSeen.test.ts +163 -15
- package/Tests/Server/Services/MetricTypeAttachServices.test.ts +222 -0
- package/Tests/Server/Services/MissingReferenceWriteGuard.test.ts +614 -0
- package/Tests/Server/Services/MonitorStatusTimelineFastPath.test.ts +23 -0
- package/Tests/Server/Services/ResourceUpdateLastSeenLivenessFallback.test.ts +305 -56
- package/Tests/Server/Services/ServiceUpdateLastSeenSplitGates.test.ts +705 -0
- package/Tests/Server/Services/StatusPageMonitorRuleEngineService.test.ts +1719 -0
- package/Tests/Server/Services/StatusPageMonitorRuleHooks.test.ts +1212 -0
- package/Tests/Server/Services/StatusPageMonitorRuleMonitorCreateHook.test.ts +228 -0
- package/Tests/Server/Utils/APIKey/AccessPermission.test.ts +267 -0
- package/Tests/Server/Utils/Database/ProjectScopedReferenceValidator.test.ts +260 -13
- package/Tests/Server/Utils/EntityRegistryRowFence.test.ts +302 -0
- package/Tests/Server/Utils/MetricTypeIndexConvergence.test.ts +598 -0
- package/Tests/Server/Utils/Monitor/MonitorStatusTimelineUnusableStatus.test.ts +154 -0
- package/Tests/Server/Utils/Monitor/MonitorStepsProjectValidator.test.ts +638 -92
- package/Tests/Server/Utils/Monitor/MonitorStepsReferenceExtractor.test.ts +724 -0
- package/Tests/Server/Utils/SingleFlight.test.ts +221 -0
- package/Tests/Types/Dashboard/NetworkDashboardTemplate.test.ts +793 -0
- package/Tests/Types/Database/BigIntColumnTransformer.test.ts +147 -0
- package/Tests/Types/Database/DatabasePropertyTransformer.test.ts +140 -0
- package/Tests/Types/Database/NumericColumnValue.test.ts +127 -0
- package/Tests/Types/Decimal.test.ts +37 -0
- package/Tests/Types/Monitor/SnmpVersionParsing.test.ts +152 -0
- package/Tests/Types/Port.test.ts +38 -0
- package/Tests/UI/Components/LogsHistogram.test.tsx +276 -0
- package/Tests/UI/Components/UseLiveLogsRefresh.test.tsx +489 -0
- package/Tests/UI/Components/UseLogsHistogram.test.tsx +500 -0
- package/Tests/Utils/Dashboard/Components/DashboardComponentsUtil.test.ts +120 -0
- package/Tests/Utils/Dashboard/DashboardNetworkMapComponent.test.ts +327 -0
- package/Types/Dashboard/DashboardComponentType.ts +1 -0
- package/Types/Dashboard/DashboardComponents/ComponentArgument.ts +1 -0
- package/Types/Dashboard/DashboardComponents/DashboardNetworkMapComponent.ts +40 -0
- package/Types/Dashboard/DashboardTemplates.ts +397 -0
- package/Types/Database/BigIntColumnTransformer.ts +74 -0
- package/Types/Database/DatabaseProperty.ts +26 -0
- package/Types/Database/NumericColumnValue.ts +128 -0
- package/Types/Decimal.ts +16 -4
- package/Types/Permission.ts +48 -0
- package/Types/Port.ts +16 -4
- package/UI/Components/LogsViewer/useLiveLogsRefresh.ts +86 -0
- package/UI/Components/LogsViewer/useLogsHistogram.ts +113 -0
- package/Utils/Dashboard/Components/DashboardNetworkMapComponent.ts +136 -0
- package/Utils/Dashboard/Components/Index.ts +7 -0
- package/build/dist/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel.js +12 -0
- package/build/dist/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Index.js +2 -0
- package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
- package/build/dist/Models/DatabaseModels/LogDropFilter.js +9 -20
- package/build/dist/Models/DatabaseModels/LogDropFilter.js.map +1 -1
- package/build/dist/Models/DatabaseModels/StatusPageMonitorRule.js +925 -0
- package/build/dist/Models/DatabaseModels/StatusPageMonitorRule.js.map +1 -0
- package/build/dist/Models/DatabaseModels/StatusPageResource.js +72 -0
- package/build/dist/Models/DatabaseModels/StatusPageResource.js.map +1 -1
- package/build/dist/Models/DatabaseModels/TraceDropFilter.js +9 -20
- package/build/dist/Models/DatabaseModels/TraceDropFilter.js.map +1 -1
- package/build/dist/Server/API/BaseAPI.js +11 -1
- package/build/dist/Server/API/BaseAPI.js.map +1 -1
- package/build/dist/Server/API/DashboardAPI.js +39 -0
- package/build/dist/Server/API/DashboardAPI.js.map +1 -1
- package/build/dist/Server/EnvironmentConfig.js +27 -1
- package/build/dist/Server/EnvironmentConfig.js.map +1 -1
- package/build/dist/Server/Infrastructure/GlobalCache.js +91 -0
- package/build/dist/Server/Infrastructure/GlobalCache.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/DataSourceOptions.js +5 -3
- package/build/dist/Server/Infrastructure/Postgres/DataSourceOptions.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1786005052769-AddStatusPageMonitorRule.js +48 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1786005052769-AddStatusPageMonitorRule.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +2 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/AlertStateTimelineService.js +19 -0
- package/build/dist/Server/Services/AlertStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/CephClusterService.js +26 -76
- package/build/dist/Server/Services/CephClusterService.js.map +1 -1
- package/build/dist/Server/Services/CloudResourceService.js +23 -73
- package/build/dist/Server/Services/CloudResourceService.js.map +1 -1
- package/build/dist/Server/Services/DatabaseService.js +86 -0
- package/build/dist/Server/Services/DatabaseService.js.map +1 -1
- package/build/dist/Server/Services/DockerHostService.js +19 -69
- package/build/dist/Server/Services/DockerHostService.js.map +1 -1
- package/build/dist/Server/Services/DockerSwarmClusterService.js +27 -78
- package/build/dist/Server/Services/DockerSwarmClusterService.js.map +1 -1
- package/build/dist/Server/Services/HostService.js +58 -102
- package/build/dist/Server/Services/HostService.js.map +1 -1
- package/build/dist/Server/Services/IncidentStateTimelineService.js +19 -0
- package/build/dist/Server/Services/IncidentStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/Index.js +2 -0
- package/build/dist/Server/Services/Index.js.map +1 -1
- package/build/dist/Server/Services/IoTFleetService.js +19 -69
- package/build/dist/Server/Services/IoTFleetService.js.map +1 -1
- package/build/dist/Server/Services/KubernetesClusterService.js +17 -68
- package/build/dist/Server/Services/KubernetesClusterService.js.map +1 -1
- package/build/dist/Server/Services/LabelService.js +42 -7
- package/build/dist/Server/Services/LabelService.js.map +1 -1
- package/build/dist/Server/Services/MetricTypeService.js +74 -0
- package/build/dist/Server/Services/MetricTypeService.js.map +1 -1
- package/build/dist/Server/Services/MonitorService.js +81 -9
- package/build/dist/Server/Services/MonitorService.js.map +1 -1
- package/build/dist/Server/Services/MonitorStatusTimelineService.js +28 -0
- package/build/dist/Server/Services/MonitorStatusTimelineService.js.map +1 -1
- package/build/dist/Server/Services/MonitorTemplateService.js +60 -0
- package/build/dist/Server/Services/MonitorTemplateService.js.map +1 -1
- package/build/dist/Server/Services/PodmanHostService.js +19 -69
- package/build/dist/Server/Services/PodmanHostService.js.map +1 -1
- package/build/dist/Server/Services/ProxmoxClusterService.js +23 -73
- package/build/dist/Server/Services/ProxmoxClusterService.js.map +1 -1
- package/build/dist/Server/Services/RumApplicationService.js +19 -70
- package/build/dist/Server/Services/RumApplicationService.js.map +1 -1
- package/build/dist/Server/Services/ScheduledMaintenanceStateTimelineService.js +20 -0
- package/build/dist/Server/Services/ScheduledMaintenanceStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/ServerlessFunctionService.js +24 -75
- package/build/dist/Server/Services/ServerlessFunctionService.js.map +1 -1
- package/build/dist/Server/Services/ServiceService.js +236 -61
- package/build/dist/Server/Services/ServiceService.js.map +1 -1
- package/build/dist/Server/Services/StatusPageMonitorRuleEngineService.js +603 -0
- package/build/dist/Server/Services/StatusPageMonitorRuleEngineService.js.map +1 -0
- package/build/dist/Server/Services/StatusPageMonitorRuleService.js +401 -0
- package/build/dist/Server/Services/StatusPageMonitorRuleService.js.map +1 -0
- package/build/dist/Server/Services/TelemetryEntityRelationshipService.js +1 -0
- package/build/dist/Server/Services/TelemetryEntityRelationshipService.js.map +1 -1
- package/build/dist/Server/Services/TelemetryEntityService.js +1 -0
- package/build/dist/Server/Services/TelemetryEntityService.js.map +1 -1
- package/build/dist/Server/Utils/Database/ProjectScopedReferenceValidator.js +105 -29
- package/build/dist/Server/Utils/Database/ProjectScopedReferenceValidator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorResource.js +56 -32
- package/build/dist/Server/Utils/Monitor/MonitorResource.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStatusTimeline.js +30 -3
- package/build/dist/Server/Utils/Monitor/MonitorStatusTimeline.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStepsProjectValidator.js +110 -98
- package/build/dist/Server/Utils/Monitor/MonitorStepsProjectValidator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStepsReferenceExtractor.js +312 -0
- package/build/dist/Server/Utils/Monitor/MonitorStepsReferenceExtractor.js.map +1 -0
- package/build/dist/Server/Utils/SingleFlight.js +76 -0
- package/build/dist/Server/Utils/SingleFlight.js.map +1 -0
- package/build/dist/Server/Utils/StatusPage/MonitorRulePatternValidator.js +29 -0
- package/build/dist/Server/Utils/StatusPage/MonitorRulePatternValidator.js.map +1 -0
- package/build/dist/Server/Utils/Telemetry/EntityRegistry.js +78 -19
- package/build/dist/Server/Utils/Telemetry/EntityRegistry.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/ResourceHeartbeat.js +160 -0
- package/build/dist/Server/Utils/Telemetry/ResourceHeartbeat.js.map +1 -0
- package/build/dist/Server/Utils/Telemetry/Telemetry.js +279 -104
- package/build/dist/Server/Utils/Telemetry/Telemetry.js.map +1 -1
- package/build/dist/Types/Dashboard/DashboardComponentType.js +1 -0
- package/build/dist/Types/Dashboard/DashboardComponentType.js.map +1 -1
- package/build/dist/Types/Dashboard/DashboardComponents/ComponentArgument.js +1 -0
- package/build/dist/Types/Dashboard/DashboardComponents/ComponentArgument.js.map +1 -1
- package/build/dist/Types/Dashboard/DashboardComponents/DashboardNetworkMapComponent.js +2 -0
- package/build/dist/Types/Dashboard/DashboardComponents/DashboardNetworkMapComponent.js.map +1 -0
- package/build/dist/Types/Dashboard/DashboardTemplates.js +368 -0
- package/build/dist/Types/Dashboard/DashboardTemplates.js.map +1 -1
- package/build/dist/Types/Database/BigIntColumnTransformer.js +66 -0
- package/build/dist/Types/Database/BigIntColumnTransformer.js.map +1 -0
- package/build/dist/Types/Database/DatabaseProperty.js +25 -0
- package/build/dist/Types/Database/DatabaseProperty.js.map +1 -1
- package/build/dist/Types/Database/NumericColumnValue.js +97 -0
- package/build/dist/Types/Database/NumericColumnValue.js.map +1 -0
- package/build/dist/Types/Decimal.js +14 -3
- package/build/dist/Types/Decimal.js.map +1 -1
- package/build/dist/Types/Permission.js +42 -0
- package/build/dist/Types/Permission.js.map +1 -1
- package/build/dist/Types/Port.js +15 -5
- package/build/dist/Types/Port.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/useLiveLogsRefresh.js +44 -0
- package/build/dist/UI/Components/LogsViewer/useLiveLogsRefresh.js.map +1 -0
- package/build/dist/UI/Components/LogsViewer/useLogsHistogram.js +75 -0
- package/build/dist/UI/Components/LogsViewer/useLogsHistogram.js.map +1 -0
- package/build/dist/Utils/Dashboard/Components/DashboardNetworkMapComponent.js +114 -0
- package/build/dist/Utils/Dashboard/Components/DashboardNetworkMapComponent.js.map +1 -0
- package/build/dist/Utils/Dashboard/Components/Index.js +4 -0
- package/build/dist/Utils/Dashboard/Components/Index.js.map +1 -1
- package/package.json +1 -1
|
@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/PodmanHost";
|
|
|
5
5
|
import Label from "../../Models/DatabaseModels/Label";
|
|
6
6
|
import { OnCreate } from "../Types/Database/Hooks";
|
|
7
7
|
import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
|
|
8
|
+
import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
|
|
8
9
|
import ObjectID from "../../Types/ObjectID";
|
|
9
10
|
import QueryHelper from "../Types/Database/QueryHelper";
|
|
10
11
|
import OneUptimeDate from "../../Types/Date";
|
|
@@ -180,7 +181,6 @@ export class Service extends DatabaseService<Model> {
|
|
|
180
181
|
agentVersion?: string | undefined;
|
|
181
182
|
},
|
|
182
183
|
): Promise<void> {
|
|
183
|
-
const cacheKey: string = hostId.toString();
|
|
184
184
|
const extrasFingerprint: string = crypto
|
|
185
185
|
.createHash("sha1")
|
|
186
186
|
.update(
|
|
@@ -192,34 +192,6 @@ export class Service extends DatabaseService<Model> {
|
|
|
192
192
|
)
|
|
193
193
|
.digest("hex");
|
|
194
194
|
|
|
195
|
-
let cached: string | null = null;
|
|
196
|
-
try {
|
|
197
|
-
cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
|
|
198
|
-
} catch {
|
|
199
|
-
/*
|
|
200
|
-
* Cache unavailable — fail open and refresh lastSeenAt anyway. A
|
|
201
|
-
* cache error must never skip the DB write below, otherwise the
|
|
202
|
-
* resource is wrongly marked "disconnected" while telemetry is
|
|
203
|
-
* still flowing. Mirrors shouldRunMaintenance's fail-open stance.
|
|
204
|
-
*/
|
|
205
|
-
cached = null;
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
if (cached === extrasFingerprint) {
|
|
209
|
-
return; // same data was written recently
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
try {
|
|
213
|
-
await GlobalCache.setString(
|
|
214
|
-
LAST_SEEN_CACHE_NAMESPACE,
|
|
215
|
-
cacheKey,
|
|
216
|
-
extrasFingerprint,
|
|
217
|
-
{ expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
|
|
218
|
-
);
|
|
219
|
-
} catch {
|
|
220
|
-
// Best-effort throttle write; proceed with the DB update regardless.
|
|
221
|
-
}
|
|
222
|
-
|
|
223
195
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
224
196
|
const liveness: any = {
|
|
225
197
|
lastSeenAt: OneUptimeDate.getCurrentDate(),
|
|
@@ -227,66 +199,34 @@ export class Service extends DatabaseService<Model> {
|
|
|
227
199
|
};
|
|
228
200
|
|
|
229
201
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
230
|
-
const
|
|
202
|
+
const metadata: any = {};
|
|
231
203
|
|
|
232
204
|
if (extra?.osType) {
|
|
233
|
-
|
|
205
|
+
metadata.osType = extra.osType;
|
|
234
206
|
}
|
|
235
207
|
if (extra?.osVersion) {
|
|
236
|
-
|
|
208
|
+
metadata.osVersion = extra.osVersion;
|
|
237
209
|
}
|
|
238
210
|
if (extra?.agentVersion) {
|
|
239
|
-
|
|
211
|
+
metadata.agentVersion = extra.agentVersion;
|
|
240
212
|
}
|
|
241
213
|
|
|
242
214
|
/*
|
|
243
|
-
*
|
|
244
|
-
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
* Collector-supplied strings are clamped to their declared column
|
|
248
|
-
* widths inside updateColumnsByIdWithoutHooks — see
|
|
249
|
-
* DatabaseService.truncateStringColumnsToMaxLength.
|
|
215
|
+
* One gated, non-blocking heartbeat write. The gates, the fail-open /
|
|
216
|
+
* fail-closed split and the liveness-only fallback all live in
|
|
217
|
+
* ResourceHeartbeat — see there for why this row's throttle used to
|
|
218
|
+
* provide no throttling at all.
|
|
250
219
|
*/
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
* reject the statement, lastSeenAt silently stops advancing and
|
|
262
|
-
* markDisconnectedHosts flips a perfectly healthy Podman host to
|
|
263
|
-
* "disconnected" 15 minutes later — with telemetry still arriving the
|
|
264
|
-
* whole time. That was issue #3006 on the Host path (a host.ip list
|
|
265
|
-
* longer than the then-varchar(500) column). Retry with liveness only
|
|
266
|
-
* so the enrichment is what gets dropped, never the heartbeat.
|
|
267
|
-
*/
|
|
268
|
-
logger.warn(
|
|
269
|
-
`PodmanHostService.updateLastSeen: metadata write failed for Podman host ${hostId.toString()}, falling back to a liveness-only update: ${
|
|
270
|
-
err instanceof Error ? err.message : String(err)
|
|
271
|
-
}`,
|
|
272
|
-
);
|
|
273
|
-
|
|
274
|
-
/*
|
|
275
|
-
* The throttle fingerprint was already stored, so without this the
|
|
276
|
-
* next batch would short-circuit on a cache hit and skip the retry
|
|
277
|
-
* for the rest of the window.
|
|
278
|
-
*/
|
|
279
|
-
try {
|
|
280
|
-
await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
|
|
281
|
-
} catch {
|
|
282
|
-
// Best-effort; the fallback write below is what actually matters.
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
await this.updateColumnsByIdWithoutHooks({
|
|
286
|
-
id: hostId,
|
|
287
|
-
data: liveness,
|
|
288
|
-
});
|
|
289
|
-
}
|
|
220
|
+
await ResourceHeartbeat.write({
|
|
221
|
+
service: this,
|
|
222
|
+
id: hostId,
|
|
223
|
+
cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
|
|
224
|
+
throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
|
|
225
|
+
liveness: liveness,
|
|
226
|
+
metadata: metadata,
|
|
227
|
+
fingerprint: extrasFingerprint,
|
|
228
|
+
describe: `podman host ${hostId.toString()}`,
|
|
229
|
+
});
|
|
290
230
|
}
|
|
291
231
|
|
|
292
232
|
/**
|
|
@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/ProxmoxCluster";
|
|
|
5
5
|
import Label from "../../Models/DatabaseModels/Label";
|
|
6
6
|
import { OnCreate } from "../Types/Database/Hooks";
|
|
7
7
|
import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
|
|
8
|
+
import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
|
|
8
9
|
import ObjectID from "../../Types/ObjectID";
|
|
9
10
|
import QueryHelper from "../Types/Database/QueryHelper";
|
|
10
11
|
import OneUptimeDate from "../../Types/Date";
|
|
@@ -177,7 +178,6 @@ export class Service extends DatabaseService<Model> {
|
|
|
177
178
|
guestsWithoutBackupCount?: number | undefined;
|
|
178
179
|
},
|
|
179
180
|
): Promise<void> {
|
|
180
|
-
const cacheKey: string = clusterId.toString();
|
|
181
181
|
const extrasFingerprint: string = crypto
|
|
182
182
|
.createHash("sha1")
|
|
183
183
|
.update(
|
|
@@ -193,34 +193,6 @@ export class Service extends DatabaseService<Model> {
|
|
|
193
193
|
)
|
|
194
194
|
.digest("hex");
|
|
195
195
|
|
|
196
|
-
let cached: string | null = null;
|
|
197
|
-
try {
|
|
198
|
-
cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
|
|
199
|
-
} catch {
|
|
200
|
-
/*
|
|
201
|
-
* Cache unavailable — fail open and refresh lastSeenAt anyway. A
|
|
202
|
-
* cache error must never skip the DB write below, otherwise the
|
|
203
|
-
* resource is wrongly marked "disconnected" while telemetry is
|
|
204
|
-
* still flowing. Mirrors shouldRunMaintenance's fail-open stance.
|
|
205
|
-
*/
|
|
206
|
-
cached = null;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
if (cached === extrasFingerprint) {
|
|
210
|
-
return; // same data was written recently
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
try {
|
|
214
|
-
await GlobalCache.setString(
|
|
215
|
-
LAST_SEEN_CACHE_NAMESPACE,
|
|
216
|
-
cacheKey,
|
|
217
|
-
extrasFingerprint,
|
|
218
|
-
{ expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
|
|
219
|
-
);
|
|
220
|
-
} catch {
|
|
221
|
-
// Best-effort throttle write; proceed with the DB update regardless.
|
|
222
|
-
}
|
|
223
|
-
|
|
224
196
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
225
197
|
const liveness: any = {
|
|
226
198
|
lastSeenAt: OneUptimeDate.getCurrentDate(),
|
|
@@ -228,79 +200,47 @@ export class Service extends DatabaseService<Model> {
|
|
|
228
200
|
};
|
|
229
201
|
|
|
230
202
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
231
|
-
const
|
|
203
|
+
const metadata: any = {};
|
|
232
204
|
|
|
233
205
|
if (extra?.pveVersion) {
|
|
234
|
-
|
|
206
|
+
metadata.pveVersion = extra.pveVersion;
|
|
235
207
|
}
|
|
236
208
|
if (extra?.agentVersion) {
|
|
237
|
-
|
|
209
|
+
metadata.agentVersion = extra.agentVersion;
|
|
238
210
|
}
|
|
239
211
|
// Counts: 0 is a legitimate value — gate on undefined, not falsiness.
|
|
240
212
|
if (extra?.nodeCount !== undefined) {
|
|
241
|
-
|
|
213
|
+
metadata.nodeCount = extra.nodeCount;
|
|
242
214
|
}
|
|
243
215
|
if (extra?.onlineNodeCount !== undefined) {
|
|
244
|
-
|
|
216
|
+
metadata.onlineNodeCount = extra.onlineNodeCount;
|
|
245
217
|
}
|
|
246
218
|
if (extra?.guestCount !== undefined) {
|
|
247
|
-
|
|
219
|
+
metadata.guestCount = extra.guestCount;
|
|
248
220
|
}
|
|
249
221
|
if (extra?.storageCount !== undefined) {
|
|
250
|
-
|
|
222
|
+
metadata.storageCount = extra.storageCount;
|
|
251
223
|
}
|
|
252
224
|
if (extra?.guestsWithoutBackupCount !== undefined) {
|
|
253
|
-
|
|
225
|
+
metadata.guestsWithoutBackupCount = extra.guestsWithoutBackupCount;
|
|
254
226
|
}
|
|
255
227
|
|
|
256
228
|
/*
|
|
257
|
-
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
* Collector-supplied strings are clamped to their declared column
|
|
262
|
-
* widths inside updateColumnsByIdWithoutHooks — see
|
|
263
|
-
* DatabaseService.truncateStringColumnsToMaxLength.
|
|
229
|
+
* One gated, non-blocking heartbeat write. The gates, the fail-open /
|
|
230
|
+
* fail-closed split and the liveness-only fallback all live in
|
|
231
|
+
* ResourceHeartbeat — see there for why this row's throttle used to
|
|
232
|
+
* provide no throttling at all.
|
|
264
233
|
*/
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
* reject the statement, lastSeenAt silently stops advancing and
|
|
276
|
-
* markDisconnectedClusters flips a perfectly healthy Proxmox cluster
|
|
277
|
-
* to "disconnected" 15 minutes later — with telemetry still arriving
|
|
278
|
-
* the whole time. That was issue #3006 on the Host path (a host.ip
|
|
279
|
-
* list longer than the then-varchar(500) column). Retry with liveness
|
|
280
|
-
* only so the enrichment is what gets dropped, never the heartbeat.
|
|
281
|
-
*/
|
|
282
|
-
logger.warn(
|
|
283
|
-
`ProxmoxClusterService.updateLastSeen: metadata write failed for Proxmox cluster ${clusterId.toString()}, falling back to a liveness-only update: ${
|
|
284
|
-
err instanceof Error ? err.message : String(err)
|
|
285
|
-
}`,
|
|
286
|
-
);
|
|
287
|
-
|
|
288
|
-
/*
|
|
289
|
-
* The throttle fingerprint was already stored, so without this the
|
|
290
|
-
* next batch would short-circuit on a cache hit and skip the retry
|
|
291
|
-
* for the rest of the window.
|
|
292
|
-
*/
|
|
293
|
-
try {
|
|
294
|
-
await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
|
|
295
|
-
} catch {
|
|
296
|
-
// Best-effort; the fallback write below is what actually matters.
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
await this.updateColumnsByIdWithoutHooks({
|
|
300
|
-
id: clusterId,
|
|
301
|
-
data: liveness,
|
|
302
|
-
});
|
|
303
|
-
}
|
|
234
|
+
await ResourceHeartbeat.write({
|
|
235
|
+
service: this,
|
|
236
|
+
id: clusterId,
|
|
237
|
+
cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
|
|
238
|
+
throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
|
|
239
|
+
liveness: liveness,
|
|
240
|
+
metadata: metadata,
|
|
241
|
+
fingerprint: extrasFingerprint,
|
|
242
|
+
describe: `proxmox cluster ${clusterId.toString()}`,
|
|
243
|
+
});
|
|
304
244
|
}
|
|
305
245
|
|
|
306
246
|
/**
|
|
@@ -2,6 +2,7 @@ import DatabaseService from "./DatabaseService";
|
|
|
2
2
|
import Model from "../../Models/DatabaseModels/RumApplication";
|
|
3
3
|
import Label from "../../Models/DatabaseModels/Label";
|
|
4
4
|
import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
|
|
5
|
+
import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
|
|
5
6
|
import ObjectID from "../../Types/ObjectID";
|
|
6
7
|
import QueryHelper from "../Types/Database/QueryHelper";
|
|
7
8
|
import OneUptimeDate from "../../Types/Date";
|
|
@@ -276,7 +277,6 @@ export class Service extends DatabaseService<Model> {
|
|
|
276
277
|
sdkLanguage?: string | undefined;
|
|
277
278
|
},
|
|
278
279
|
): Promise<void> {
|
|
279
|
-
const cacheKey: string = rumApplicationId.toString();
|
|
280
280
|
const extrasFingerprint: string = crypto
|
|
281
281
|
.createHash("sha1")
|
|
282
282
|
.update(
|
|
@@ -288,34 +288,6 @@ export class Service extends DatabaseService<Model> {
|
|
|
288
288
|
)
|
|
289
289
|
.digest("hex");
|
|
290
290
|
|
|
291
|
-
let cached: string | null = null;
|
|
292
|
-
try {
|
|
293
|
-
cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
|
|
294
|
-
} catch {
|
|
295
|
-
/*
|
|
296
|
-
* Cache unavailable — fail open and refresh lastSeenAt anyway. A
|
|
297
|
-
* cache error must never skip the DB write below, otherwise the
|
|
298
|
-
* resource is wrongly marked "disconnected" while telemetry is
|
|
299
|
-
* still flowing. Mirrors shouldRunMaintenance's fail-open stance.
|
|
300
|
-
*/
|
|
301
|
-
cached = null;
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
if (cached === extrasFingerprint) {
|
|
305
|
-
return; // same data was written recently
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
try {
|
|
309
|
-
await GlobalCache.setString(
|
|
310
|
-
LAST_SEEN_CACHE_NAMESPACE,
|
|
311
|
-
cacheKey,
|
|
312
|
-
extrasFingerprint,
|
|
313
|
-
{ expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
|
|
314
|
-
);
|
|
315
|
-
} catch {
|
|
316
|
-
// Best-effort throttle write; proceed with the DB update regardless.
|
|
317
|
-
}
|
|
318
|
-
|
|
319
291
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
320
292
|
const liveness: any = {
|
|
321
293
|
lastSeenAt: OneUptimeDate.getCurrentDate(),
|
|
@@ -323,67 +295,34 @@ export class Service extends DatabaseService<Model> {
|
|
|
323
295
|
};
|
|
324
296
|
|
|
325
297
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
326
|
-
const
|
|
298
|
+
const metadata: any = {};
|
|
327
299
|
|
|
328
300
|
if (extra?.agentVersion) {
|
|
329
|
-
|
|
301
|
+
metadata.agentVersion = extra.agentVersion;
|
|
330
302
|
}
|
|
331
303
|
if (extra?.clientType) {
|
|
332
|
-
|
|
304
|
+
metadata.clientType = extra.clientType;
|
|
333
305
|
}
|
|
334
306
|
if (extra?.sdkLanguage) {
|
|
335
|
-
|
|
307
|
+
metadata.sdkLanguage = extra.sdkLanguage;
|
|
336
308
|
}
|
|
337
309
|
|
|
338
310
|
/*
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
* Collector-supplied strings are clamped to their declared column
|
|
344
|
-
* widths inside updateColumnsByIdWithoutHooks — see
|
|
345
|
-
* DatabaseService.truncateStringColumnsToMaxLength.
|
|
311
|
+
* One gated, non-blocking heartbeat write. The gates, the fail-open /
|
|
312
|
+
* fail-closed split and the liveness-only fallback all live in
|
|
313
|
+
* ResourceHeartbeat — see there for why this row's throttle used to
|
|
314
|
+
* provide no throttling at all.
|
|
346
315
|
*/
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
* reject the statement, lastSeenAt silently stops advancing and
|
|
358
|
-
* markDisconnectedApplications flips a perfectly healthy RUM
|
|
359
|
-
* application to "disconnected" 15 minutes later — with telemetry
|
|
360
|
-
* still arriving the whole time. That was issue #3006 on the Host
|
|
361
|
-
* path (a host.ip list longer than the then-varchar(500) column).
|
|
362
|
-
* Retry with liveness only so the enrichment is what gets dropped,
|
|
363
|
-
* never the heartbeat.
|
|
364
|
-
*/
|
|
365
|
-
logger.warn(
|
|
366
|
-
`RumApplicationService.updateLastSeen: metadata write failed for RUM application ${rumApplicationId.toString()}, falling back to a liveness-only update: ${
|
|
367
|
-
err instanceof Error ? err.message : String(err)
|
|
368
|
-
}`,
|
|
369
|
-
);
|
|
370
|
-
|
|
371
|
-
/*
|
|
372
|
-
* The throttle fingerprint was already stored, so without this the
|
|
373
|
-
* next batch would short-circuit on a cache hit and skip the retry
|
|
374
|
-
* for the rest of the window.
|
|
375
|
-
*/
|
|
376
|
-
try {
|
|
377
|
-
await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
|
|
378
|
-
} catch {
|
|
379
|
-
// Best-effort; the fallback write below is what actually matters.
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
await this.updateColumnsByIdWithoutHooks({
|
|
383
|
-
id: rumApplicationId,
|
|
384
|
-
data: liveness,
|
|
385
|
-
});
|
|
386
|
-
}
|
|
316
|
+
await ResourceHeartbeat.write({
|
|
317
|
+
service: this,
|
|
318
|
+
id: rumApplicationId,
|
|
319
|
+
cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
|
|
320
|
+
throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
|
|
321
|
+
liveness: liveness,
|
|
322
|
+
metadata: metadata,
|
|
323
|
+
fingerprint: extrasFingerprint,
|
|
324
|
+
describe: `rum application ${rumApplicationId.toString()}`,
|
|
325
|
+
});
|
|
387
326
|
}
|
|
388
327
|
|
|
389
328
|
/*
|
|
@@ -26,6 +26,7 @@ import ScheduledMaintenanceStateTimeline from "../../Models/DatabaseModels/Sched
|
|
|
26
26
|
import { IsBillingEnabled } from "../EnvironmentConfig";
|
|
27
27
|
import ScheduledMaintenanceFeedService from "./ScheduledMaintenanceFeedService";
|
|
28
28
|
import { ScheduledMaintenanceFeedEventType } from "../../Models/DatabaseModels/ScheduledMaintenanceFeed";
|
|
29
|
+
import ProjectScopedReferenceValidator from "../Utils/Database/ProjectScopedReferenceValidator";
|
|
29
30
|
import logger, { LogAttributes } from "../Utils/Logger";
|
|
30
31
|
import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
|
|
31
32
|
import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
|
|
@@ -122,6 +123,26 @@ export class Service extends DatabaseService<ScheduledMaintenanceStateTimeline>
|
|
|
122
123
|
throw new BadDataException("scheduledMaintenanceStateId is null");
|
|
123
124
|
}
|
|
124
125
|
|
|
126
|
+
/*
|
|
127
|
+
* Same guard, same reason, as MonitorStatusTimelineService: an id that
|
|
128
|
+
* exists nowhere (or belongs to another project) used to reach Postgres
|
|
129
|
+
* and come back as a raw foreign key violation. The ordering check below
|
|
130
|
+
* does look the state up, but only when there IS a preceding row with an
|
|
131
|
+
* order, so the first timeline row for a scheduled maintenance event went
|
|
132
|
+
* through unchecked.
|
|
133
|
+
*/
|
|
134
|
+
await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
|
|
135
|
+
projectId: createBy.props.tenantId || createBy.data.projectId,
|
|
136
|
+
subject: "scheduled maintenance state timeline",
|
|
137
|
+
references: [
|
|
138
|
+
{
|
|
139
|
+
modelName: "Scheduled Maintenance State",
|
|
140
|
+
id: scheduledMaintenanceStateId,
|
|
141
|
+
service: ScheduledMaintenanceStateService,
|
|
142
|
+
},
|
|
143
|
+
],
|
|
144
|
+
});
|
|
145
|
+
|
|
125
146
|
const stateBeforeThis: ScheduledMaintenanceStateTimeline | null =
|
|
126
147
|
await this.findOneBy({
|
|
127
148
|
query: {
|
|
@@ -2,6 +2,7 @@ import DatabaseService from "./DatabaseService";
|
|
|
2
2
|
import Model from "../../Models/DatabaseModels/ServerlessFunction";
|
|
3
3
|
import Label from "../../Models/DatabaseModels/Label";
|
|
4
4
|
import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
|
|
5
|
+
import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
|
|
5
6
|
import ObjectID from "../../Types/ObjectID";
|
|
6
7
|
import QueryHelper from "../Types/Database/QueryHelper";
|
|
7
8
|
import OneUptimeDate from "../../Types/Date";
|
|
@@ -153,7 +154,6 @@ export class Service extends DatabaseService<Model> {
|
|
|
153
154
|
runtimeVersion?: string | undefined;
|
|
154
155
|
},
|
|
155
156
|
): Promise<void> {
|
|
156
|
-
const cacheKey: string = serverlessFunctionId.toString();
|
|
157
157
|
const extrasFingerprint: string = crypto
|
|
158
158
|
.createHash("sha1")
|
|
159
159
|
.update(
|
|
@@ -170,34 +170,6 @@ export class Service extends DatabaseService<Model> {
|
|
|
170
170
|
)
|
|
171
171
|
.digest("hex");
|
|
172
172
|
|
|
173
|
-
let cached: string | null = null;
|
|
174
|
-
try {
|
|
175
|
-
cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
|
|
176
|
-
} catch {
|
|
177
|
-
/*
|
|
178
|
-
* Cache unavailable — fail open and refresh lastSeenAt anyway. A
|
|
179
|
-
* cache error must never skip the DB write below, otherwise the
|
|
180
|
-
* resource is wrongly marked "disconnected" while telemetry is
|
|
181
|
-
* still flowing. Mirrors shouldRunMaintenance's fail-open stance.
|
|
182
|
-
*/
|
|
183
|
-
cached = null;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
if (cached === extrasFingerprint) {
|
|
187
|
-
return; // same data was written recently
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
try {
|
|
191
|
-
await GlobalCache.setString(
|
|
192
|
-
LAST_SEEN_CACHE_NAMESPACE,
|
|
193
|
-
cacheKey,
|
|
194
|
-
extrasFingerprint,
|
|
195
|
-
{ expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
|
|
196
|
-
);
|
|
197
|
-
} catch {
|
|
198
|
-
// Best-effort throttle write; proceed with the DB update regardless.
|
|
199
|
-
}
|
|
200
|
-
|
|
201
173
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
202
174
|
const liveness: any = {
|
|
203
175
|
lastSeenAt: OneUptimeDate.getCurrentDate(),
|
|
@@ -205,82 +177,49 @@ export class Service extends DatabaseService<Model> {
|
|
|
205
177
|
};
|
|
206
178
|
|
|
207
179
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
208
|
-
const
|
|
180
|
+
const metadata: any = {};
|
|
209
181
|
|
|
210
182
|
if (extra?.agentVersion) {
|
|
211
|
-
|
|
183
|
+
metadata.agentVersion = extra.agentVersion;
|
|
212
184
|
}
|
|
213
185
|
if (extra?.cloudPlatform) {
|
|
214
|
-
|
|
186
|
+
metadata.cloudPlatform = extra.cloudPlatform;
|
|
215
187
|
}
|
|
216
188
|
if (extra?.cloudProvider) {
|
|
217
|
-
|
|
189
|
+
metadata.cloudProvider = extra.cloudProvider;
|
|
218
190
|
}
|
|
219
191
|
if (extra?.cloudRegion) {
|
|
220
|
-
|
|
192
|
+
metadata.cloudRegion = extra.cloudRegion;
|
|
221
193
|
}
|
|
222
194
|
if (extra?.cloudAccountId) {
|
|
223
|
-
|
|
195
|
+
metadata.cloudAccountId = extra.cloudAccountId;
|
|
224
196
|
}
|
|
225
197
|
if (extra?.functionVersion) {
|
|
226
|
-
|
|
198
|
+
metadata.functionVersion = extra.functionVersion;
|
|
227
199
|
}
|
|
228
200
|
if (extra?.runtimeName) {
|
|
229
|
-
|
|
201
|
+
metadata.runtimeName = extra.runtimeName;
|
|
230
202
|
}
|
|
231
203
|
if (extra?.runtimeVersion) {
|
|
232
|
-
|
|
204
|
+
metadata.runtimeVersion = extra.runtimeVersion;
|
|
233
205
|
}
|
|
234
206
|
|
|
235
207
|
/*
|
|
236
|
-
*
|
|
237
|
-
*
|
|
238
|
-
*
|
|
239
|
-
*
|
|
240
|
-
* Collector-supplied strings are clamped to their declared column
|
|
241
|
-
* widths inside updateColumnsByIdWithoutHooks — see
|
|
242
|
-
* DatabaseService.truncateStringColumnsToMaxLength.
|
|
208
|
+
* One gated, non-blocking heartbeat write. The gates, the fail-open /
|
|
209
|
+
* fail-closed split and the liveness-only fallback all live in
|
|
210
|
+
* ResourceHeartbeat — see there for why this row's throttle used to
|
|
211
|
+
* provide no throttling at all.
|
|
243
212
|
*/
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
* reject the statement, lastSeenAt silently stops advancing and
|
|
255
|
-
* markDisconnectedFunctions flips a perfectly healthy serverless
|
|
256
|
-
* function to "disconnected" 15 minutes later — with telemetry still
|
|
257
|
-
* arriving the whole time. That was issue #3006 on the Host path (a
|
|
258
|
-
* host.ip list longer than the then-varchar(500) column). Retry with
|
|
259
|
-
* liveness only so the enrichment is what gets dropped, never the
|
|
260
|
-
* heartbeat.
|
|
261
|
-
*/
|
|
262
|
-
logger.warn(
|
|
263
|
-
`ServerlessFunctionService.updateLastSeen: metadata write failed for serverless function ${serverlessFunctionId.toString()}, falling back to a liveness-only update: ${
|
|
264
|
-
err instanceof Error ? err.message : String(err)
|
|
265
|
-
}`,
|
|
266
|
-
);
|
|
267
|
-
|
|
268
|
-
/*
|
|
269
|
-
* The throttle fingerprint was already stored, so without this the
|
|
270
|
-
* next batch would short-circuit on a cache hit and skip the retry
|
|
271
|
-
* for the rest of the window.
|
|
272
|
-
*/
|
|
273
|
-
try {
|
|
274
|
-
await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
|
|
275
|
-
} catch {
|
|
276
|
-
// Best-effort; the fallback write below is what actually matters.
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
await this.updateColumnsByIdWithoutHooks({
|
|
280
|
-
id: serverlessFunctionId,
|
|
281
|
-
data: liveness,
|
|
282
|
-
});
|
|
283
|
-
}
|
|
213
|
+
await ResourceHeartbeat.write({
|
|
214
|
+
service: this,
|
|
215
|
+
id: serverlessFunctionId,
|
|
216
|
+
cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
|
|
217
|
+
throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
|
|
218
|
+
liveness: liveness,
|
|
219
|
+
metadata: metadata,
|
|
220
|
+
fingerprint: extrasFingerprint,
|
|
221
|
+
describe: `serverless function ${serverlessFunctionId.toString()}`,
|
|
222
|
+
});
|
|
284
223
|
}
|
|
285
224
|
|
|
286
225
|
/**
|