@cadenza.io/service 2.6.1 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +35 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +1126 -17
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1126 -17
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -250,6 +250,8 @@ var isBrowser = typeof window !== "undefined" && typeof window.document !== "und
|
|
|
250
250
|
// src/utils/inquiry.ts
|
|
251
251
|
var META_INTENT_PREFIX = "meta-";
|
|
252
252
|
var META_RUNTIME_TRANSPORT_DIAGNOSTICS_INTENT = "meta-runtime-transport-diagnostics";
|
|
253
|
+
var META_RUNTIME_STATUS_INTENT = "meta-runtime-status";
|
|
254
|
+
var META_READINESS_INTENT = "meta-readiness";
|
|
253
255
|
function isPlainObject(value) {
|
|
254
256
|
return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
|
|
255
257
|
}
|
|
@@ -313,8 +315,200 @@ function summarizeResponderStatuses(statuses) {
|
|
|
313
315
|
return { responded, failed, timedOut, pending };
|
|
314
316
|
}
|
|
315
317
|
|
|
318
|
+
// src/utils/readiness.ts
|
|
319
|
+
function evaluateDependencyReadiness(input) {
|
|
320
|
+
const missedHeartbeats = Math.max(
|
|
321
|
+
0,
|
|
322
|
+
Math.trunc(Number(input.missedHeartbeats) || 0)
|
|
323
|
+
);
|
|
324
|
+
const stale = missedHeartbeats > 0;
|
|
325
|
+
const timeoutReached = missedHeartbeats >= Math.max(1, input.missThreshold);
|
|
326
|
+
if (!input.exists) {
|
|
327
|
+
return {
|
|
328
|
+
state: "unavailable",
|
|
329
|
+
stale: true,
|
|
330
|
+
blocked: true,
|
|
331
|
+
reason: "missing"
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
if (timeoutReached) {
|
|
335
|
+
return {
|
|
336
|
+
state: "unavailable",
|
|
337
|
+
stale: true,
|
|
338
|
+
blocked: true,
|
|
339
|
+
reason: "heartbeat-timeout"
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
if (input.runtimeState === "unavailable" || !input.acceptingWork) {
|
|
343
|
+
return {
|
|
344
|
+
state: "unavailable",
|
|
345
|
+
stale,
|
|
346
|
+
blocked: true,
|
|
347
|
+
reason: "runtime-unavailable"
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
if (stale) {
|
|
351
|
+
return {
|
|
352
|
+
state: "degraded",
|
|
353
|
+
stale: true,
|
|
354
|
+
blocked: false,
|
|
355
|
+
reason: "heartbeat-stale"
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
if (input.runtimeState === "overloaded") {
|
|
359
|
+
return {
|
|
360
|
+
state: "overloaded",
|
|
361
|
+
stale: false,
|
|
362
|
+
blocked: false,
|
|
363
|
+
reason: "runtime-overloaded"
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
if (input.runtimeState === "degraded") {
|
|
367
|
+
return {
|
|
368
|
+
state: "degraded",
|
|
369
|
+
stale: false,
|
|
370
|
+
blocked: false,
|
|
371
|
+
reason: "runtime-degraded"
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
return {
|
|
375
|
+
state: "ready",
|
|
376
|
+
stale: false,
|
|
377
|
+
blocked: false,
|
|
378
|
+
reason: "runtime-healthy"
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
function summarizeDependencyReadiness(evaluations) {
|
|
382
|
+
const summary = {
|
|
383
|
+
total: evaluations.length,
|
|
384
|
+
ready: 0,
|
|
385
|
+
degraded: 0,
|
|
386
|
+
overloaded: 0,
|
|
387
|
+
unavailable: 0,
|
|
388
|
+
stale: 0
|
|
389
|
+
};
|
|
390
|
+
for (const evaluation of evaluations) {
|
|
391
|
+
if (evaluation.state === "ready") summary.ready++;
|
|
392
|
+
if (evaluation.state === "degraded") summary.degraded++;
|
|
393
|
+
if (evaluation.state === "overloaded") summary.overloaded++;
|
|
394
|
+
if (evaluation.state === "unavailable") summary.unavailable++;
|
|
395
|
+
if (evaluation.stale) summary.stale++;
|
|
396
|
+
}
|
|
397
|
+
return summary;
|
|
398
|
+
}
|
|
399
|
+
function resolveServiceReadinessState(localRuntimeState, localAcceptingWork, dependencySummary) {
|
|
400
|
+
if (localRuntimeState === "unavailable" || !localAcceptingWork) {
|
|
401
|
+
return "blocked";
|
|
402
|
+
}
|
|
403
|
+
if (dependencySummary.unavailable > 0) {
|
|
404
|
+
return "blocked";
|
|
405
|
+
}
|
|
406
|
+
if (dependencySummary.degraded > 0 || dependencySummary.overloaded > 0 || dependencySummary.stale > 0) {
|
|
407
|
+
return "degraded";
|
|
408
|
+
}
|
|
409
|
+
return "ready";
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// src/utils/runtimeStatus.ts
|
|
413
|
+
function resolveRuntimeStatus(input) {
|
|
414
|
+
const numberOfRunningGraphs = Math.max(
|
|
415
|
+
0,
|
|
416
|
+
Math.trunc(Number(input.numberOfRunningGraphs) || 0)
|
|
417
|
+
);
|
|
418
|
+
const isActive = Boolean(input.isActive);
|
|
419
|
+
const isNonResponsive = Boolean(input.isNonResponsive);
|
|
420
|
+
const isBlocked = Boolean(input.isBlocked);
|
|
421
|
+
if (!isActive || isNonResponsive || isBlocked) {
|
|
422
|
+
return {
|
|
423
|
+
state: "unavailable",
|
|
424
|
+
acceptingWork: false,
|
|
425
|
+
numberOfRunningGraphs,
|
|
426
|
+
isActive,
|
|
427
|
+
isNonResponsive,
|
|
428
|
+
isBlocked
|
|
429
|
+
};
|
|
430
|
+
}
|
|
431
|
+
if (numberOfRunningGraphs >= input.overloadedGraphThreshold) {
|
|
432
|
+
return {
|
|
433
|
+
state: "overloaded",
|
|
434
|
+
acceptingWork: true,
|
|
435
|
+
numberOfRunningGraphs,
|
|
436
|
+
isActive,
|
|
437
|
+
isNonResponsive,
|
|
438
|
+
isBlocked
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
if (numberOfRunningGraphs >= input.degradedGraphThreshold) {
|
|
442
|
+
return {
|
|
443
|
+
state: "degraded",
|
|
444
|
+
acceptingWork: true,
|
|
445
|
+
numberOfRunningGraphs,
|
|
446
|
+
isActive,
|
|
447
|
+
isNonResponsive,
|
|
448
|
+
isBlocked
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
return {
|
|
452
|
+
state: "healthy",
|
|
453
|
+
acceptingWork: true,
|
|
454
|
+
numberOfRunningGraphs,
|
|
455
|
+
isActive,
|
|
456
|
+
isNonResponsive,
|
|
457
|
+
isBlocked
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
function runtimeStatusPriority(state) {
|
|
461
|
+
switch (state) {
|
|
462
|
+
case "healthy":
|
|
463
|
+
return 0;
|
|
464
|
+
case "degraded":
|
|
465
|
+
return 1;
|
|
466
|
+
case "overloaded":
|
|
467
|
+
return 2;
|
|
468
|
+
case "unavailable":
|
|
469
|
+
return 3;
|
|
470
|
+
default:
|
|
471
|
+
return 4;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
function hasSignificantRuntimeStatusChange(previous, next) {
|
|
475
|
+
if (!previous) {
|
|
476
|
+
return true;
|
|
477
|
+
}
|
|
478
|
+
return previous.state !== next.state || previous.acceptingWork !== next.acceptingWork || previous.isActive !== next.isActive || previous.isNonResponsive !== next.isNonResponsive || previous.isBlocked !== next.isBlocked;
|
|
479
|
+
}
|
|
480
|
+
|
|
316
481
|
// src/registry/ServiceRegistry.ts
|
|
317
482
|
var META_SERVICE_REGISTRY_FULL_SYNC_INTENT = "meta-service-registry-full-sync";
|
|
483
|
+
var META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL = "meta.service_registry.runtime_status.heartbeat_tick";
|
|
484
|
+
var META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL = "meta.service_registry.runtime_status.monitor_tick";
|
|
485
|
+
var INTERNAL_RUNTIME_STATUS_TASK_NAMES = /* @__PURE__ */ new Set([
|
|
486
|
+
"Track local routine start",
|
|
487
|
+
"Track local routine end",
|
|
488
|
+
"Start runtime status sharing intervals",
|
|
489
|
+
"Broadcast runtime status",
|
|
490
|
+
"Monitor dependee heartbeat freshness",
|
|
491
|
+
"Resolve runtime status fallback inquiry",
|
|
492
|
+
"Respond runtime status inquiry",
|
|
493
|
+
"Respond readiness inquiry",
|
|
494
|
+
"Collect distributed readiness",
|
|
495
|
+
"Get status"
|
|
496
|
+
]);
|
|
497
|
+
function readPositiveIntegerEnv(name, fallback) {
|
|
498
|
+
if (typeof process === "undefined") {
|
|
499
|
+
return fallback;
|
|
500
|
+
}
|
|
501
|
+
const raw = process.env?.[name];
|
|
502
|
+
const parsed = Number(raw);
|
|
503
|
+
if (!Number.isFinite(parsed)) {
|
|
504
|
+
return fallback;
|
|
505
|
+
}
|
|
506
|
+
const normalized = Math.trunc(parsed);
|
|
507
|
+
if (normalized <= 0) {
|
|
508
|
+
return fallback;
|
|
509
|
+
}
|
|
510
|
+
return normalized;
|
|
511
|
+
}
|
|
318
512
|
var ServiceRegistry = class _ServiceRegistry {
|
|
319
513
|
/**
|
|
320
514
|
* Initializes a private constructor for managing service instances, remote signals,
|
|
@@ -332,6 +526,36 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
332
526
|
this.remoteIntents = /* @__PURE__ */ new Map();
|
|
333
527
|
this.remoteIntentDeputiesByKey = /* @__PURE__ */ new Map();
|
|
334
528
|
this.remoteIntentDeputiesByTask = /* @__PURE__ */ new Map();
|
|
529
|
+
this.dependeesByService = /* @__PURE__ */ new Map();
|
|
530
|
+
this.dependeeByInstance = /* @__PURE__ */ new Map();
|
|
531
|
+
this.readinessDependeesByService = /* @__PURE__ */ new Map();
|
|
532
|
+
this.readinessDependeeByInstance = /* @__PURE__ */ new Map();
|
|
533
|
+
this.lastHeartbeatAtByInstance = /* @__PURE__ */ new Map();
|
|
534
|
+
this.missedHeartbeatsByInstance = /* @__PURE__ */ new Map();
|
|
535
|
+
this.runtimeStatusFallbackInFlightByInstance = /* @__PURE__ */ new Set();
|
|
536
|
+
this.activeRoutineExecutionIds = /* @__PURE__ */ new Set();
|
|
537
|
+
this.runtimeStatusHeartbeatStarted = false;
|
|
538
|
+
this.lastRuntimeStatusSnapshot = null;
|
|
539
|
+
this.runtimeStatusHeartbeatIntervalMs = readPositiveIntegerEnv(
|
|
540
|
+
"CADENZA_RUNTIME_STATUS_HEARTBEAT_MS",
|
|
541
|
+
3e4
|
|
542
|
+
);
|
|
543
|
+
this.runtimeStatusMissThreshold = readPositiveIntegerEnv(
|
|
544
|
+
"CADENZA_RUNTIME_STATUS_MISSED_HEARTBEATS",
|
|
545
|
+
3
|
|
546
|
+
);
|
|
547
|
+
this.runtimeStatusFallbackTimeoutMs = readPositiveIntegerEnv(
|
|
548
|
+
"CADENZA_RUNTIME_STATUS_FALLBACK_TIMEOUT_MS",
|
|
549
|
+
1500
|
|
550
|
+
);
|
|
551
|
+
this.degradedGraphThreshold = readPositiveIntegerEnv(
|
|
552
|
+
"CADENZA_RUNTIME_STATUS_DEGRADED_GRAPH_THRESHOLD",
|
|
553
|
+
10
|
|
554
|
+
);
|
|
555
|
+
this.overloadedGraphThreshold = readPositiveIntegerEnv(
|
|
556
|
+
"CADENZA_RUNTIME_STATUS_OVERLOADED_GRAPH_THRESHOLD",
|
|
557
|
+
20
|
|
558
|
+
);
|
|
335
559
|
this.serviceName = null;
|
|
336
560
|
this.serviceInstanceId = null;
|
|
337
561
|
this.numberOfRunningGraphs = 0;
|
|
@@ -370,10 +594,137 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
370
594
|
}
|
|
371
595
|
}
|
|
372
596
|
});
|
|
597
|
+
CadenzaService.defineIntent({
|
|
598
|
+
name: META_RUNTIME_STATUS_INTENT,
|
|
599
|
+
description: "Gather lightweight runtime status reports from services in the distributed runtime.",
|
|
600
|
+
input: {
|
|
601
|
+
type: "object",
|
|
602
|
+
properties: {
|
|
603
|
+
detailLevel: {
|
|
604
|
+
type: "string",
|
|
605
|
+
constraints: {
|
|
606
|
+
oneOf: ["minimal", "full"]
|
|
607
|
+
}
|
|
608
|
+
},
|
|
609
|
+
targetServiceName: {
|
|
610
|
+
type: "string"
|
|
611
|
+
},
|
|
612
|
+
targetServiceInstanceId: {
|
|
613
|
+
type: "string"
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
},
|
|
617
|
+
output: {
|
|
618
|
+
type: "object",
|
|
619
|
+
properties: {
|
|
620
|
+
runtimeStatusReports: {
|
|
621
|
+
type: "array"
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
});
|
|
626
|
+
CadenzaService.createMetaTask(
|
|
627
|
+
"Respond runtime status inquiry",
|
|
628
|
+
(ctx) => {
|
|
629
|
+
const targetServiceName = ctx.targetServiceName;
|
|
630
|
+
const targetServiceInstanceId = ctx.targetServiceInstanceId;
|
|
631
|
+
const detailLevel = ctx.detailLevel === "full" ? "full" : "minimal";
|
|
632
|
+
const report = this.buildLocalRuntimeStatusReport(detailLevel);
|
|
633
|
+
if (!report) {
|
|
634
|
+
return {};
|
|
635
|
+
}
|
|
636
|
+
if (targetServiceName && targetServiceName !== report.serviceName) {
|
|
637
|
+
return {};
|
|
638
|
+
}
|
|
639
|
+
if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
|
|
640
|
+
return {};
|
|
641
|
+
}
|
|
642
|
+
return {
|
|
643
|
+
runtimeStatusReports: [report]
|
|
644
|
+
};
|
|
645
|
+
},
|
|
646
|
+
"Responds to runtime-status inquiries with local service instance status."
|
|
647
|
+
).respondsTo(META_RUNTIME_STATUS_INTENT);
|
|
648
|
+
CadenzaService.defineIntent({
|
|
649
|
+
name: META_READINESS_INTENT,
|
|
650
|
+
description: "Gather service readiness reports derived from local runtime status and required dependees.",
|
|
651
|
+
input: {
|
|
652
|
+
type: "object",
|
|
653
|
+
properties: {
|
|
654
|
+
detailLevel: {
|
|
655
|
+
type: "string",
|
|
656
|
+
constraints: {
|
|
657
|
+
oneOf: ["minimal", "full"]
|
|
658
|
+
}
|
|
659
|
+
},
|
|
660
|
+
includeDependencies: {
|
|
661
|
+
type: "boolean"
|
|
662
|
+
},
|
|
663
|
+
refreshStaleDependencies: {
|
|
664
|
+
type: "boolean"
|
|
665
|
+
},
|
|
666
|
+
targetServiceName: {
|
|
667
|
+
type: "string"
|
|
668
|
+
},
|
|
669
|
+
targetServiceInstanceId: {
|
|
670
|
+
type: "string"
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
},
|
|
674
|
+
output: {
|
|
675
|
+
type: "object",
|
|
676
|
+
properties: {
|
|
677
|
+
readinessReports: {
|
|
678
|
+
type: "array"
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
});
|
|
683
|
+
CadenzaService.createMetaTask(
|
|
684
|
+
"Respond readiness inquiry",
|
|
685
|
+
async (ctx) => {
|
|
686
|
+
const targetServiceName = ctx.targetServiceName;
|
|
687
|
+
const targetServiceInstanceId = ctx.targetServiceInstanceId;
|
|
688
|
+
const report = await this.buildLocalReadinessReport({
|
|
689
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
690
|
+
includeDependencies: ctx.includeDependencies,
|
|
691
|
+
refreshStaleDependencies: ctx.refreshStaleDependencies
|
|
692
|
+
});
|
|
693
|
+
if (!report) {
|
|
694
|
+
return {};
|
|
695
|
+
}
|
|
696
|
+
if (targetServiceName && targetServiceName !== report.serviceName) {
|
|
697
|
+
return {};
|
|
698
|
+
}
|
|
699
|
+
if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
|
|
700
|
+
return {};
|
|
701
|
+
}
|
|
702
|
+
return {
|
|
703
|
+
readinessReports: [report]
|
|
704
|
+
};
|
|
705
|
+
},
|
|
706
|
+
"Responds to distributed readiness inquiries using required dependee health."
|
|
707
|
+
).respondsTo(META_READINESS_INTENT);
|
|
373
708
|
this.handleInstanceUpdateTask = CadenzaService.createMetaTask(
|
|
374
709
|
"Handle Instance Update",
|
|
375
710
|
(ctx, emit) => {
|
|
376
|
-
const
|
|
711
|
+
const serviceInstance = ctx.serviceInstance ?? (ctx.__serviceInstanceId || ctx.serviceInstanceId ? {
|
|
712
|
+
uuid: ctx.__serviceInstanceId ?? ctx.serviceInstanceId,
|
|
713
|
+
serviceName: ctx.__serviceName ?? ctx.serviceName,
|
|
714
|
+
address: ctx.serviceAddress ?? "",
|
|
715
|
+
port: ctx.servicePort ?? 0,
|
|
716
|
+
exposed: !!ctx.exposed,
|
|
717
|
+
isFrontend: !!ctx.isFrontend,
|
|
718
|
+
isActive: typeof ctx.isActive === "boolean" ? ctx.isActive : typeof ctx.__active === "boolean" ? ctx.__active : true,
|
|
719
|
+
isNonResponsive: !!ctx.isNonResponsive,
|
|
720
|
+
isBlocked: !!ctx.isBlocked,
|
|
721
|
+
health: ctx.health ?? ctx.__health ?? {},
|
|
722
|
+
numberOfRunningGraphs: ctx.numberOfRunningGraphs ?? ctx.__numberOfRunningGraphs ?? 0,
|
|
723
|
+
isPrimary: false
|
|
724
|
+
} : void 0);
|
|
725
|
+
if (!serviceInstance?.uuid || !serviceInstance?.serviceName) {
|
|
726
|
+
return false;
|
|
727
|
+
}
|
|
377
728
|
const {
|
|
378
729
|
uuid: uuid4,
|
|
379
730
|
serviceName,
|
|
@@ -395,6 +746,7 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
395
746
|
emit(`meta.socket_shutdown_requested:${address}_${port}`, {});
|
|
396
747
|
emit(`meta.fetch.destroy_requested:${address}_${port}`, {});
|
|
397
748
|
}
|
|
749
|
+
this.unregisterDependee(uuid4, serviceName);
|
|
398
750
|
return;
|
|
399
751
|
}
|
|
400
752
|
if (!this.instances.has(serviceName))
|
|
@@ -406,6 +758,18 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
406
758
|
} else {
|
|
407
759
|
instances.push(serviceInstance);
|
|
408
760
|
}
|
|
761
|
+
const trackedInstance = existing ?? instances.find((instance) => instance.uuid === uuid4);
|
|
762
|
+
if (trackedInstance) {
|
|
763
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
764
|
+
trackedInstance.numberOfRunningGraphs ?? 0,
|
|
765
|
+
trackedInstance.isActive,
|
|
766
|
+
trackedInstance.isNonResponsive,
|
|
767
|
+
trackedInstance.isBlocked
|
|
768
|
+
);
|
|
769
|
+
trackedInstance.runtimeState = snapshot.state;
|
|
770
|
+
trackedInstance.acceptingWork = snapshot.acceptingWork;
|
|
771
|
+
trackedInstance.reportedAt = trackedInstance.reportedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
772
|
+
}
|
|
409
773
|
if (this.serviceName === serviceName) {
|
|
410
774
|
return false;
|
|
411
775
|
}
|
|
@@ -445,13 +809,27 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
445
809
|
"global.meta.service_instance.inserted",
|
|
446
810
|
"global.meta.service_instance.updated",
|
|
447
811
|
"meta.service_instance.inserted",
|
|
448
|
-
"meta.service_instance.updated"
|
|
449
|
-
"meta.socket_client.status_received"
|
|
812
|
+
"meta.service_instance.updated"
|
|
450
813
|
).attachSignal(
|
|
451
814
|
"meta.service_registry.dependee_registered",
|
|
452
815
|
"meta.socket_shutdown_requested",
|
|
453
816
|
"meta.fetch.destroy_requested"
|
|
454
817
|
);
|
|
818
|
+
CadenzaService.createMetaTask(
|
|
819
|
+
"Track dependee registration",
|
|
820
|
+
(ctx) => {
|
|
821
|
+
if (!ctx.serviceName || !ctx.serviceInstanceId) {
|
|
822
|
+
return false;
|
|
823
|
+
}
|
|
824
|
+
this.registerDependee(ctx.serviceName, ctx.serviceInstanceId, {
|
|
825
|
+
requiredForReadiness: this.shouldRequireReadinessFromCommunicationTypes(
|
|
826
|
+
ctx.communicationTypes
|
|
827
|
+
)
|
|
828
|
+
});
|
|
829
|
+
return true;
|
|
830
|
+
},
|
|
831
|
+
"Tracks remote dependency instances for runtime heartbeat monitoring."
|
|
832
|
+
).doOn("meta.service_registry.dependee_registered");
|
|
455
833
|
CadenzaService.createMetaTask("Split service instances", function* (ctx) {
|
|
456
834
|
if (!ctx.serviceInstances) {
|
|
457
835
|
return;
|
|
@@ -551,6 +929,15 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
551
929
|
for (const instance of instances ?? []) {
|
|
552
930
|
instance.isActive = false;
|
|
553
931
|
instance.isNonResponsive = true;
|
|
932
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
933
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
934
|
+
instance.isActive,
|
|
935
|
+
instance.isNonResponsive,
|
|
936
|
+
instance.isBlocked
|
|
937
|
+
);
|
|
938
|
+
instance.runtimeState = snapshot.state;
|
|
939
|
+
instance.acceptingWork = snapshot.acceptingWork;
|
|
940
|
+
instance.reportedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
554
941
|
emit("global.meta.service_registry.service_not_responding", {
|
|
555
942
|
data: {
|
|
556
943
|
isActive: false,
|
|
@@ -568,7 +955,8 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
568
955
|
"meta.fetch.handshake_failed",
|
|
569
956
|
"meta.fetch.handshake_failed.*",
|
|
570
957
|
"meta.socket_client.disconnected",
|
|
571
|
-
"meta.socket_client.disconnected.*"
|
|
958
|
+
"meta.socket_client.disconnected.*",
|
|
959
|
+
"meta.service_registry.runtime_status_unreachable"
|
|
572
960
|
).attachSignal("global.meta.service_registry.service_not_responding");
|
|
573
961
|
this.handleServiceHandshakeTask = CadenzaService.createMetaTask(
|
|
574
962
|
"Handle service handshake",
|
|
@@ -583,6 +971,15 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
583
971
|
}
|
|
584
972
|
instance.isActive = true;
|
|
585
973
|
instance.isNonResponsive = false;
|
|
974
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
975
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
976
|
+
instance.isActive,
|
|
977
|
+
instance.isNonResponsive,
|
|
978
|
+
instance.isBlocked
|
|
979
|
+
);
|
|
980
|
+
instance.runtimeState = snapshot.state;
|
|
981
|
+
instance.acceptingWork = snapshot.acceptingWork;
|
|
982
|
+
instance.reportedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
586
983
|
emit("global.meta.service_registry.service_handshake", {
|
|
587
984
|
data: {
|
|
588
985
|
isActive: instance.isActive,
|
|
@@ -600,6 +997,7 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
600
997
|
if (indexToDelete >= 0) {
|
|
601
998
|
this.instances.get(serviceName)?.splice(indexToDelete, 1);
|
|
602
999
|
}
|
|
1000
|
+
this.unregisterDependee(i.uuid, serviceName);
|
|
603
1001
|
emit("global.meta.service_registry.deleted", {
|
|
604
1002
|
data: {
|
|
605
1003
|
isActive: false,
|
|
@@ -621,14 +1019,46 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
621
1019
|
this.handleSocketStatusUpdateTask = CadenzaService.createMetaTask(
|
|
622
1020
|
"Handle Socket Status Update",
|
|
623
1021
|
(ctx) => {
|
|
624
|
-
const
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
if (
|
|
629
|
-
|
|
630
|
-
|
|
1022
|
+
const report = this.normalizeRuntimeStatusReport(ctx);
|
|
1023
|
+
if (!report) {
|
|
1024
|
+
return false;
|
|
1025
|
+
}
|
|
1026
|
+
if (report.serviceName === this.serviceName && report.serviceInstanceId === this.serviceInstanceId) {
|
|
1027
|
+
return false;
|
|
1028
|
+
}
|
|
1029
|
+
let applied = this.applyRuntimeStatusReport(report);
|
|
1030
|
+
if (!applied && report.serviceAddress && typeof report.servicePort === "number") {
|
|
1031
|
+
if (!this.instances.has(report.serviceName)) {
|
|
1032
|
+
this.instances.set(report.serviceName, []);
|
|
1033
|
+
}
|
|
1034
|
+
this.instances.get(report.serviceName).push({
|
|
1035
|
+
uuid: report.serviceInstanceId,
|
|
1036
|
+
serviceName: report.serviceName,
|
|
1037
|
+
address: report.serviceAddress,
|
|
1038
|
+
port: report.servicePort,
|
|
1039
|
+
exposed: !!report.exposed,
|
|
1040
|
+
isFrontend: !!report.isFrontend,
|
|
1041
|
+
isActive: report.isActive,
|
|
1042
|
+
isNonResponsive: report.isNonResponsive,
|
|
1043
|
+
isBlocked: report.isBlocked,
|
|
1044
|
+
numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1045
|
+
runtimeState: report.state,
|
|
1046
|
+
acceptingWork: report.acceptingWork,
|
|
1047
|
+
reportedAt: report.reportedAt,
|
|
1048
|
+
health: report.health ?? {},
|
|
1049
|
+
isPrimary: false
|
|
1050
|
+
});
|
|
1051
|
+
applied = true;
|
|
631
1052
|
}
|
|
1053
|
+
if (!applied) {
|
|
1054
|
+
return false;
|
|
1055
|
+
}
|
|
1056
|
+
this.registerDependee(report.serviceName, report.serviceInstanceId);
|
|
1057
|
+
this.lastHeartbeatAtByInstance.set(report.serviceInstanceId, Date.now());
|
|
1058
|
+
this.missedHeartbeatsByInstance.set(report.serviceInstanceId, 0);
|
|
1059
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(
|
|
1060
|
+
report.serviceInstanceId
|
|
1061
|
+
);
|
|
632
1062
|
return true;
|
|
633
1063
|
},
|
|
634
1064
|
"Handles status update from socket broadcast"
|
|
@@ -758,7 +1188,25 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
758
1188
|
const { __serviceName, __triedInstances, __retries, __broadcast } = context;
|
|
759
1189
|
let retries = __retries ?? 0;
|
|
760
1190
|
let triedInstances = __triedInstances ?? [];
|
|
761
|
-
const instances = this.instances.get(__serviceName)?.filter((i) => i.isActive && !i.isNonResponsive && !i.isBlocked).sort((a, b) =>
|
|
1191
|
+
const instances = this.instances.get(__serviceName)?.filter((i) => i.isActive && !i.isNonResponsive && !i.isBlocked).sort((a, b) => {
|
|
1192
|
+
const leftStatus = this.resolveRuntimeStatusSnapshot(
|
|
1193
|
+
a.numberOfRunningGraphs ?? 0,
|
|
1194
|
+
a.isActive,
|
|
1195
|
+
a.isNonResponsive,
|
|
1196
|
+
a.isBlocked
|
|
1197
|
+
);
|
|
1198
|
+
const rightStatus = this.resolveRuntimeStatusSnapshot(
|
|
1199
|
+
b.numberOfRunningGraphs ?? 0,
|
|
1200
|
+
b.isActive,
|
|
1201
|
+
b.isNonResponsive,
|
|
1202
|
+
b.isBlocked
|
|
1203
|
+
);
|
|
1204
|
+
const priorityDelta = runtimeStatusPriority(leftStatus.state) - runtimeStatusPriority(rightStatus.state);
|
|
1205
|
+
if (priorityDelta !== 0) {
|
|
1206
|
+
return priorityDelta;
|
|
1207
|
+
}
|
|
1208
|
+
return (a.numberOfRunningGraphs ?? 0) - (b.numberOfRunningGraphs ?? 0);
|
|
1209
|
+
});
|
|
762
1210
|
if (!instances || instances.length === 0 || retries > this.retryCount) {
|
|
763
1211
|
context.errored = true;
|
|
764
1212
|
context.__error = `No active instances for ${__serviceName}. Retries: ${retries}. ${this.instances.get(
|
|
@@ -844,15 +1292,292 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
844
1292
|
errored: true
|
|
845
1293
|
};
|
|
846
1294
|
}
|
|
847
|
-
const
|
|
1295
|
+
const report = this.buildLocalRuntimeStatusReport("full");
|
|
1296
|
+
if (!report) {
|
|
1297
|
+
return {
|
|
1298
|
+
...ctx,
|
|
1299
|
+
__status: "error",
|
|
1300
|
+
__error: "No local service instance available for status check",
|
|
1301
|
+
errored: true
|
|
1302
|
+
};
|
|
1303
|
+
}
|
|
848
1304
|
return {
|
|
849
1305
|
...ctx,
|
|
850
1306
|
__status: "ok",
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
1307
|
+
__serviceName: report.serviceName,
|
|
1308
|
+
__serviceInstanceId: report.serviceInstanceId,
|
|
1309
|
+
__numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1310
|
+
__health: report.health ?? {},
|
|
1311
|
+
__active: report.isActive,
|
|
1312
|
+
reportedAt: report.reportedAt,
|
|
1313
|
+
serviceName: report.serviceName,
|
|
1314
|
+
serviceInstanceId: report.serviceInstanceId,
|
|
1315
|
+
numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1316
|
+
health: report.health ?? {},
|
|
1317
|
+
isActive: report.isActive,
|
|
1318
|
+
isNonResponsive: report.isNonResponsive,
|
|
1319
|
+
isBlocked: report.isBlocked,
|
|
1320
|
+
state: report.state,
|
|
1321
|
+
acceptingWork: report.acceptingWork
|
|
854
1322
|
};
|
|
855
|
-
}).doOn(
|
|
1323
|
+
}).doOn(
|
|
1324
|
+
"meta.socket.status_check_requested",
|
|
1325
|
+
"meta.rest.status_check_requested"
|
|
1326
|
+
);
|
|
1327
|
+
CadenzaService.createMetaTask(
|
|
1328
|
+
"Track local routine start",
|
|
1329
|
+
(ctx, emit) => {
|
|
1330
|
+
const sourceTaskName = String(ctx.__signalEmission?.taskName ?? "");
|
|
1331
|
+
if (INTERNAL_RUNTIME_STATUS_TASK_NAMES.has(sourceTaskName)) {
|
|
1332
|
+
return false;
|
|
1333
|
+
}
|
|
1334
|
+
const routineId = String(
|
|
1335
|
+
ctx.filter?.uuid ?? ctx.__routineExecId ?? ""
|
|
1336
|
+
);
|
|
1337
|
+
if (!routineId) {
|
|
1338
|
+
return false;
|
|
1339
|
+
}
|
|
1340
|
+
this.activeRoutineExecutionIds.add(routineId);
|
|
1341
|
+
this.numberOfRunningGraphs = this.activeRoutineExecutionIds.size;
|
|
1342
|
+
const localInstance = this.getLocalInstance();
|
|
1343
|
+
if (!localInstance) {
|
|
1344
|
+
return true;
|
|
1345
|
+
}
|
|
1346
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
1347
|
+
this.numberOfRunningGraphs,
|
|
1348
|
+
localInstance.isActive,
|
|
1349
|
+
localInstance.isNonResponsive,
|
|
1350
|
+
localInstance.isBlocked
|
|
1351
|
+
);
|
|
1352
|
+
if (hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
|
|
1353
|
+
emit("meta.service_registry.runtime_status_broadcast_requested", {
|
|
1354
|
+
reason: "runtime-state-change"
|
|
1355
|
+
});
|
|
1356
|
+
}
|
|
1357
|
+
return true;
|
|
1358
|
+
},
|
|
1359
|
+
"Tracks local routine starts for runtime load status."
|
|
1360
|
+
).doOn("meta.node.started_routine_execution");
|
|
1361
|
+
CadenzaService.createMetaTask(
|
|
1362
|
+
"Track local routine end",
|
|
1363
|
+
(ctx, emit) => {
|
|
1364
|
+
const sourceTaskName = String(ctx.__signalEmission?.taskName ?? "");
|
|
1365
|
+
if (INTERNAL_RUNTIME_STATUS_TASK_NAMES.has(sourceTaskName)) {
|
|
1366
|
+
return false;
|
|
1367
|
+
}
|
|
1368
|
+
const routineId = String(
|
|
1369
|
+
ctx.filter?.uuid ?? ctx.__routineExecId ?? ""
|
|
1370
|
+
);
|
|
1371
|
+
if (!routineId) {
|
|
1372
|
+
return false;
|
|
1373
|
+
}
|
|
1374
|
+
this.activeRoutineExecutionIds.delete(routineId);
|
|
1375
|
+
this.numberOfRunningGraphs = this.activeRoutineExecutionIds.size;
|
|
1376
|
+
const localInstance = this.getLocalInstance();
|
|
1377
|
+
if (!localInstance) {
|
|
1378
|
+
return true;
|
|
1379
|
+
}
|
|
1380
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
1381
|
+
this.numberOfRunningGraphs,
|
|
1382
|
+
localInstance.isActive,
|
|
1383
|
+
localInstance.isNonResponsive,
|
|
1384
|
+
localInstance.isBlocked
|
|
1385
|
+
);
|
|
1386
|
+
if (hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
|
|
1387
|
+
emit("meta.service_registry.runtime_status_broadcast_requested", {
|
|
1388
|
+
reason: "runtime-state-change"
|
|
1389
|
+
});
|
|
1390
|
+
}
|
|
1391
|
+
return true;
|
|
1392
|
+
},
|
|
1393
|
+
"Tracks local routine completion for runtime load status."
|
|
1394
|
+
).doOn("meta.node.ended_routine_execution");
|
|
1395
|
+
CadenzaService.createMetaTask(
|
|
1396
|
+
"Start runtime status sharing intervals",
|
|
1397
|
+
() => {
|
|
1398
|
+
if (this.runtimeStatusHeartbeatStarted) {
|
|
1399
|
+
return false;
|
|
1400
|
+
}
|
|
1401
|
+
this.runtimeStatusHeartbeatStarted = true;
|
|
1402
|
+
CadenzaService.interval(
|
|
1403
|
+
META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL,
|
|
1404
|
+
{ reason: "heartbeat" },
|
|
1405
|
+
this.runtimeStatusHeartbeatIntervalMs,
|
|
1406
|
+
true
|
|
1407
|
+
);
|
|
1408
|
+
CadenzaService.interval(
|
|
1409
|
+
META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL,
|
|
1410
|
+
{},
|
|
1411
|
+
this.runtimeStatusHeartbeatIntervalMs
|
|
1412
|
+
);
|
|
1413
|
+
return true;
|
|
1414
|
+
},
|
|
1415
|
+
"Starts runtime status heartbeat and heartbeat-monitor loops once per service instance."
|
|
1416
|
+
).doOn("meta.service_registry.instance_inserted");
|
|
1417
|
+
CadenzaService.createMetaTask(
|
|
1418
|
+
"Broadcast runtime status",
|
|
1419
|
+
(ctx, emit) => {
|
|
1420
|
+
const report = this.buildLocalRuntimeStatusReport(
|
|
1421
|
+
ctx.detailLevel === "full" ? "full" : "minimal"
|
|
1422
|
+
);
|
|
1423
|
+
if (!report) {
|
|
1424
|
+
return false;
|
|
1425
|
+
}
|
|
1426
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
1427
|
+
report.numberOfRunningGraphs,
|
|
1428
|
+
report.isActive,
|
|
1429
|
+
report.isNonResponsive,
|
|
1430
|
+
report.isBlocked
|
|
1431
|
+
);
|
|
1432
|
+
const force = ctx.reason === "heartbeat" || ctx.force === true || this.lastRuntimeStatusSnapshot === null;
|
|
1433
|
+
if (!force && !hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
|
|
1434
|
+
return false;
|
|
1435
|
+
}
|
|
1436
|
+
this.lastRuntimeStatusSnapshot = snapshot;
|
|
1437
|
+
emit("meta.service.updated", {
|
|
1438
|
+
__serviceName: report.serviceName,
|
|
1439
|
+
__serviceInstanceId: report.serviceInstanceId,
|
|
1440
|
+
__reportedAt: report.reportedAt,
|
|
1441
|
+
__numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1442
|
+
__health: report.health ?? {},
|
|
1443
|
+
__active: report.isActive,
|
|
1444
|
+
serviceName: report.serviceName,
|
|
1445
|
+
serviceInstanceId: report.serviceInstanceId,
|
|
1446
|
+
serviceAddress: report.serviceAddress,
|
|
1447
|
+
servicePort: report.servicePort,
|
|
1448
|
+
exposed: report.exposed,
|
|
1449
|
+
isFrontend: report.isFrontend,
|
|
1450
|
+
reportedAt: report.reportedAt,
|
|
1451
|
+
numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1452
|
+
health: report.health ?? {},
|
|
1453
|
+
isActive: report.isActive,
|
|
1454
|
+
isNonResponsive: report.isNonResponsive,
|
|
1455
|
+
isBlocked: report.isBlocked,
|
|
1456
|
+
state: report.state,
|
|
1457
|
+
acceptingWork: report.acceptingWork
|
|
1458
|
+
});
|
|
1459
|
+
return true;
|
|
1460
|
+
},
|
|
1461
|
+
"Broadcasts local runtime status to connected dependees."
|
|
1462
|
+
).doOn(
|
|
1463
|
+
META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL,
|
|
1464
|
+
"meta.service_registry.runtime_status_broadcast_requested"
|
|
1465
|
+
);
|
|
1466
|
+
CadenzaService.createMetaTask(
|
|
1467
|
+
"Monitor dependee heartbeat freshness",
|
|
1468
|
+
(ctx, emit) => {
|
|
1469
|
+
if (!this.useSocket) {
|
|
1470
|
+
return false;
|
|
1471
|
+
}
|
|
1472
|
+
const now = Date.now();
|
|
1473
|
+
for (const [serviceName, instanceIds] of this.dependeesByService) {
|
|
1474
|
+
for (const serviceInstanceId of instanceIds) {
|
|
1475
|
+
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
1476
|
+
if (!instance || !instance.isActive || instance.isBlocked) {
|
|
1477
|
+
continue;
|
|
1478
|
+
}
|
|
1479
|
+
const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
|
|
1480
|
+
const misses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
|
|
1481
|
+
const heartbeatBudget = this.runtimeStatusHeartbeatIntervalMs * (misses + 1);
|
|
1482
|
+
if (lastHeartbeat > 0 && now - lastHeartbeat < heartbeatBudget) {
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1485
|
+
const nextMisses = misses + 1;
|
|
1486
|
+
this.missedHeartbeatsByInstance.set(serviceInstanceId, nextMisses);
|
|
1487
|
+
if (nextMisses < this.runtimeStatusMissThreshold || this.runtimeStatusFallbackInFlightByInstance.has(serviceInstanceId)) {
|
|
1488
|
+
continue;
|
|
1489
|
+
}
|
|
1490
|
+
this.runtimeStatusFallbackInFlightByInstance.add(serviceInstanceId);
|
|
1491
|
+
emit("meta.service_registry.runtime_status_fallback_requested", {
|
|
1492
|
+
...ctx,
|
|
1493
|
+
serviceName,
|
|
1494
|
+
serviceInstanceId,
|
|
1495
|
+
serviceAddress: instance.address,
|
|
1496
|
+
servicePort: instance.port
|
|
1497
|
+
});
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
return true;
|
|
1501
|
+
},
|
|
1502
|
+
"Monitors dependee heartbeat freshness and requests inquiry fallback after repeated misses."
|
|
1503
|
+
).doOn(META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL);
|
|
1504
|
+
CadenzaService.createMetaTask(
|
|
1505
|
+
"Resolve runtime status fallback inquiry",
|
|
1506
|
+
async (ctx, emit) => {
|
|
1507
|
+
const serviceName = ctx.serviceName;
|
|
1508
|
+
const serviceInstanceId = ctx.serviceInstanceId;
|
|
1509
|
+
if (!serviceName || !serviceInstanceId) {
|
|
1510
|
+
return false;
|
|
1511
|
+
}
|
|
1512
|
+
try {
|
|
1513
|
+
const { report, inquiryMeta } = await this.resolveRuntimeStatusFallbackInquiry(
|
|
1514
|
+
serviceName,
|
|
1515
|
+
serviceInstanceId,
|
|
1516
|
+
{
|
|
1517
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
1518
|
+
overallTimeoutMs: ctx.overallTimeoutMs,
|
|
1519
|
+
perResponderTimeoutMs: ctx.perResponderTimeoutMs,
|
|
1520
|
+
requireComplete: ctx.requireComplete
|
|
1521
|
+
}
|
|
1522
|
+
);
|
|
1523
|
+
return {
|
|
1524
|
+
...ctx,
|
|
1525
|
+
runtimeStatusReport: report,
|
|
1526
|
+
__inquiryMeta: inquiryMeta
|
|
1527
|
+
};
|
|
1528
|
+
} catch (error) {
|
|
1529
|
+
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
1530
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1531
|
+
CadenzaService.log(
|
|
1532
|
+
"Runtime status fallback inquiry failed.",
|
|
1533
|
+
{
|
|
1534
|
+
serviceName,
|
|
1535
|
+
serviceInstanceId,
|
|
1536
|
+
error: message
|
|
1537
|
+
},
|
|
1538
|
+
"warning"
|
|
1539
|
+
);
|
|
1540
|
+
emit("meta.service_registry.runtime_status_unreachable", {
|
|
1541
|
+
...ctx,
|
|
1542
|
+
serviceName,
|
|
1543
|
+
serviceInstanceId,
|
|
1544
|
+
serviceAddress: instance?.address ?? ctx.serviceAddress,
|
|
1545
|
+
servicePort: instance?.port ?? ctx.servicePort,
|
|
1546
|
+
__error: message,
|
|
1547
|
+
errored: true
|
|
1548
|
+
});
|
|
1549
|
+
return {
|
|
1550
|
+
...ctx,
|
|
1551
|
+
__error: message,
|
|
1552
|
+
errored: true
|
|
1553
|
+
};
|
|
1554
|
+
} finally {
|
|
1555
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
|
|
1556
|
+
}
|
|
1557
|
+
},
|
|
1558
|
+
"Runs runtime-status inquiry fallback for a dependee instance after missed heartbeats."
|
|
1559
|
+
).doOn("meta.service_registry.runtime_status_fallback_requested").emits("meta.service_registry.runtime_status_fallback_resolved").emitsOnFail("meta.service_registry.runtime_status_fallback_failed");
|
|
1560
|
+
this.collectReadinessTask = CadenzaService.createMetaTask(
|
|
1561
|
+
"Collect distributed readiness",
|
|
1562
|
+
async (ctx) => {
|
|
1563
|
+
const inquiryResult = await CadenzaService.inquire(
|
|
1564
|
+
META_READINESS_INTENT,
|
|
1565
|
+
{
|
|
1566
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
1567
|
+
includeDependencies: ctx.includeDependencies,
|
|
1568
|
+
refreshStaleDependencies: ctx.refreshStaleDependencies,
|
|
1569
|
+
targetServiceName: ctx.targetServiceName,
|
|
1570
|
+
targetServiceInstanceId: ctx.targetServiceInstanceId
|
|
1571
|
+
},
|
|
1572
|
+
ctx.inquiryOptions ?? ctx.__inquiryOptions ?? {}
|
|
1573
|
+
);
|
|
1574
|
+
return {
|
|
1575
|
+
...ctx,
|
|
1576
|
+
...inquiryResult
|
|
1577
|
+
};
|
|
1578
|
+
},
|
|
1579
|
+
"Collects distributed readiness reports from services."
|
|
1580
|
+
).doOn("meta.service_registry.readiness_requested").emits("meta.service_registry.readiness_collected").emitsOnFail("meta.service_registry.readiness_failed");
|
|
856
1581
|
this.collectTransportDiagnosticsTask = CadenzaService.createMetaTask(
|
|
857
1582
|
"Collect transport diagnostics",
|
|
858
1583
|
async (ctx) => {
|
|
@@ -1192,6 +1917,379 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1192
1917
|
localTaskName: task.name
|
|
1193
1918
|
};
|
|
1194
1919
|
}
|
|
1920
|
+
getInstance(serviceName, instanceId) {
|
|
1921
|
+
return this.instances.get(serviceName)?.find((instance) => instance.uuid === instanceId);
|
|
1922
|
+
}
|
|
1923
|
+
getLocalInstance() {
|
|
1924
|
+
if (!this.serviceName || !this.serviceInstanceId) {
|
|
1925
|
+
return void 0;
|
|
1926
|
+
}
|
|
1927
|
+
return this.getInstance(this.serviceName, this.serviceInstanceId);
|
|
1928
|
+
}
|
|
1929
|
+
registerDependee(serviceName, serviceInstanceId, options = {}) {
|
|
1930
|
+
if (!serviceName || !serviceInstanceId) {
|
|
1931
|
+
return;
|
|
1932
|
+
}
|
|
1933
|
+
if (!this.dependeesByService.has(serviceName)) {
|
|
1934
|
+
this.dependeesByService.set(serviceName, /* @__PURE__ */ new Set());
|
|
1935
|
+
}
|
|
1936
|
+
this.dependeesByService.get(serviceName).add(serviceInstanceId);
|
|
1937
|
+
this.dependeeByInstance.set(serviceInstanceId, serviceName);
|
|
1938
|
+
if (options.requiredForReadiness) {
|
|
1939
|
+
if (!this.readinessDependeesByService.has(serviceName)) {
|
|
1940
|
+
this.readinessDependeesByService.set(serviceName, /* @__PURE__ */ new Set());
|
|
1941
|
+
}
|
|
1942
|
+
this.readinessDependeesByService.get(serviceName).add(serviceInstanceId);
|
|
1943
|
+
this.readinessDependeeByInstance.set(serviceInstanceId, serviceName);
|
|
1944
|
+
}
|
|
1945
|
+
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
1946
|
+
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
1947
|
+
}
|
|
1948
|
+
unregisterDependee(serviceInstanceId, serviceName) {
|
|
1949
|
+
const dependeeServiceName = serviceName ?? this.dependeeByInstance.get(serviceInstanceId);
|
|
1950
|
+
if (dependeeServiceName) {
|
|
1951
|
+
this.dependeesByService.get(dependeeServiceName)?.delete(serviceInstanceId);
|
|
1952
|
+
if (!this.dependeesByService.get(dependeeServiceName)?.size) {
|
|
1953
|
+
this.dependeesByService.delete(dependeeServiceName);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
this.dependeeByInstance.delete(serviceInstanceId);
|
|
1957
|
+
const readinessDependeeServiceName = serviceName ?? this.readinessDependeeByInstance.get(serviceInstanceId);
|
|
1958
|
+
if (readinessDependeeServiceName) {
|
|
1959
|
+
this.readinessDependeesByService.get(readinessDependeeServiceName)?.delete(serviceInstanceId);
|
|
1960
|
+
if (!this.readinessDependeesByService.get(readinessDependeeServiceName)?.size) {
|
|
1961
|
+
this.readinessDependeesByService.delete(readinessDependeeServiceName);
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1964
|
+
this.readinessDependeeByInstance.delete(serviceInstanceId);
|
|
1965
|
+
this.lastHeartbeatAtByInstance.delete(serviceInstanceId);
|
|
1966
|
+
this.missedHeartbeatsByInstance.delete(serviceInstanceId);
|
|
1967
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
|
|
1968
|
+
}
|
|
1969
|
+
getHeartbeatMisses(serviceInstanceId, now = Date.now()) {
|
|
1970
|
+
const observedMisses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
|
|
1971
|
+
const lastHeartbeatAt = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
|
|
1972
|
+
if (lastHeartbeatAt <= 0) {
|
|
1973
|
+
return Math.max(observedMisses, this.runtimeStatusMissThreshold);
|
|
1974
|
+
}
|
|
1975
|
+
const estimatedMisses = Math.max(
|
|
1976
|
+
0,
|
|
1977
|
+
Math.floor((now - lastHeartbeatAt) / this.runtimeStatusHeartbeatIntervalMs)
|
|
1978
|
+
);
|
|
1979
|
+
return Math.max(observedMisses, estimatedMisses);
|
|
1980
|
+
}
|
|
1981
|
+
shouldRequireReadinessFromCommunicationTypes(communicationTypes) {
|
|
1982
|
+
if (!Array.isArray(communicationTypes)) {
|
|
1983
|
+
return false;
|
|
1984
|
+
}
|
|
1985
|
+
return communicationTypes.some((type) => {
|
|
1986
|
+
const normalized = String(type).toLowerCase();
|
|
1987
|
+
return normalized === "delegation" || normalized === "inquiry";
|
|
1988
|
+
});
|
|
1989
|
+
}
|
|
1990
|
+
resolveRuntimeStatusSnapshot(numberOfRunningGraphs, isActive, isNonResponsive, isBlocked) {
|
|
1991
|
+
return resolveRuntimeStatus({
|
|
1992
|
+
numberOfRunningGraphs,
|
|
1993
|
+
isActive,
|
|
1994
|
+
isNonResponsive,
|
|
1995
|
+
isBlocked,
|
|
1996
|
+
degradedGraphThreshold: this.degradedGraphThreshold,
|
|
1997
|
+
overloadedGraphThreshold: this.overloadedGraphThreshold
|
|
1998
|
+
});
|
|
1999
|
+
}
|
|
2000
|
+
normalizeRuntimeStatusReport(ctx) {
|
|
2001
|
+
const serviceName = ctx.serviceName ?? ctx.__serviceName ?? ctx.serviceInstance?.serviceName;
|
|
2002
|
+
const serviceInstanceId = ctx.serviceInstanceId ?? ctx.__serviceInstanceId ?? ctx.serviceInstance?.uuid;
|
|
2003
|
+
if (!serviceName || !serviceInstanceId) {
|
|
2004
|
+
return null;
|
|
2005
|
+
}
|
|
2006
|
+
const servicePort = ctx.servicePort ?? ctx.port ?? ctx.serviceInstance?.port;
|
|
2007
|
+
const numberOfRunningGraphs = Math.max(
|
|
2008
|
+
0,
|
|
2009
|
+
Math.trunc(
|
|
2010
|
+
Number(ctx.numberOfRunningGraphs ?? ctx.__numberOfRunningGraphs ?? 0)
|
|
2011
|
+
)
|
|
2012
|
+
);
|
|
2013
|
+
const isActive = Boolean(ctx.isActive ?? ctx.__active ?? true);
|
|
2014
|
+
const isNonResponsive = Boolean(ctx.isNonResponsive ?? false);
|
|
2015
|
+
const isBlocked = Boolean(ctx.isBlocked ?? false);
|
|
2016
|
+
const resolved = this.resolveRuntimeStatusSnapshot(
|
|
2017
|
+
numberOfRunningGraphs,
|
|
2018
|
+
isActive,
|
|
2019
|
+
isNonResponsive,
|
|
2020
|
+
isBlocked
|
|
2021
|
+
);
|
|
2022
|
+
return {
|
|
2023
|
+
serviceName,
|
|
2024
|
+
serviceInstanceId,
|
|
2025
|
+
serviceAddress: ctx.serviceAddress ?? ctx.address ?? ctx.serviceInstance?.address,
|
|
2026
|
+
servicePort: typeof servicePort === "number" ? servicePort : void 0,
|
|
2027
|
+
exposed: typeof ctx.exposed === "boolean" ? ctx.exposed : typeof ctx.serviceInstance?.exposed === "boolean" ? ctx.serviceInstance.exposed : void 0,
|
|
2028
|
+
isFrontend: typeof ctx.isFrontend === "boolean" ? ctx.isFrontend : typeof ctx.serviceInstance?.isFrontend === "boolean" ? ctx.serviceInstance.isFrontend : void 0,
|
|
2029
|
+
reportedAt: ctx.reportedAt ?? (typeof ctx.__reportedAt === "string" ? ctx.__reportedAt : void 0) ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
2030
|
+
state: ctx.state === "healthy" || ctx.state === "degraded" || ctx.state === "overloaded" || ctx.state === "unavailable" ? ctx.state : resolved.state,
|
|
2031
|
+
acceptingWork: typeof ctx.acceptingWork === "boolean" ? ctx.acceptingWork : resolved.acceptingWork,
|
|
2032
|
+
numberOfRunningGraphs,
|
|
2033
|
+
isActive,
|
|
2034
|
+
isNonResponsive,
|
|
2035
|
+
isBlocked,
|
|
2036
|
+
health: ctx.health ?? ctx.__health ?? {}
|
|
2037
|
+
};
|
|
2038
|
+
}
|
|
2039
|
+
applyRuntimeStatusReport(report) {
|
|
2040
|
+
const instance = this.getInstance(report.serviceName, report.serviceInstanceId);
|
|
2041
|
+
if (!instance) {
|
|
2042
|
+
return false;
|
|
2043
|
+
}
|
|
2044
|
+
if (report.serviceAddress) {
|
|
2045
|
+
instance.address = report.serviceAddress;
|
|
2046
|
+
}
|
|
2047
|
+
if (typeof report.servicePort === "number") {
|
|
2048
|
+
instance.port = report.servicePort;
|
|
2049
|
+
}
|
|
2050
|
+
if (typeof report.exposed === "boolean") {
|
|
2051
|
+
instance.exposed = report.exposed;
|
|
2052
|
+
}
|
|
2053
|
+
if (typeof report.isFrontend === "boolean") {
|
|
2054
|
+
instance.isFrontend = report.isFrontend;
|
|
2055
|
+
}
|
|
2056
|
+
instance.numberOfRunningGraphs = report.numberOfRunningGraphs;
|
|
2057
|
+
instance.isActive = report.isActive;
|
|
2058
|
+
instance.isNonResponsive = report.isNonResponsive;
|
|
2059
|
+
instance.isBlocked = report.isBlocked;
|
|
2060
|
+
instance.runtimeState = report.state;
|
|
2061
|
+
instance.acceptingWork = report.acceptingWork;
|
|
2062
|
+
instance.reportedAt = report.reportedAt;
|
|
2063
|
+
instance.health = {
|
|
2064
|
+
...instance.health ?? {},
|
|
2065
|
+
...report.health ?? {},
|
|
2066
|
+
runtimeStatus: {
|
|
2067
|
+
state: report.state,
|
|
2068
|
+
acceptingWork: report.acceptingWork,
|
|
2069
|
+
reportedAt: report.reportedAt
|
|
2070
|
+
}
|
|
2071
|
+
};
|
|
2072
|
+
return true;
|
|
2073
|
+
}
|
|
2074
|
+
buildLocalRuntimeStatusReport(detailLevel = "minimal") {
|
|
2075
|
+
if (!this.serviceName || !this.serviceInstanceId) {
|
|
2076
|
+
return null;
|
|
2077
|
+
}
|
|
2078
|
+
const localInstance = this.getLocalInstance();
|
|
2079
|
+
if (!localInstance) {
|
|
2080
|
+
return null;
|
|
2081
|
+
}
|
|
2082
|
+
const numberOfRunningGraphs = this.activeRoutineExecutionIds.size || this.numberOfRunningGraphs || 0;
|
|
2083
|
+
this.numberOfRunningGraphs = numberOfRunningGraphs;
|
|
2084
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
2085
|
+
numberOfRunningGraphs,
|
|
2086
|
+
localInstance.isActive,
|
|
2087
|
+
localInstance.isNonResponsive,
|
|
2088
|
+
localInstance.isBlocked
|
|
2089
|
+
);
|
|
2090
|
+
const reportedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2091
|
+
const report = {
|
|
2092
|
+
serviceName: this.serviceName,
|
|
2093
|
+
serviceInstanceId: this.serviceInstanceId,
|
|
2094
|
+
serviceAddress: localInstance.address,
|
|
2095
|
+
servicePort: localInstance.port,
|
|
2096
|
+
exposed: localInstance.exposed,
|
|
2097
|
+
isFrontend: localInstance.isFrontend,
|
|
2098
|
+
reportedAt,
|
|
2099
|
+
state: snapshot.state,
|
|
2100
|
+
acceptingWork: snapshot.acceptingWork,
|
|
2101
|
+
numberOfRunningGraphs: snapshot.numberOfRunningGraphs,
|
|
2102
|
+
isActive: snapshot.isActive,
|
|
2103
|
+
isNonResponsive: snapshot.isNonResponsive,
|
|
2104
|
+
isBlocked: snapshot.isBlocked,
|
|
2105
|
+
health: {
|
|
2106
|
+
...localInstance.health ?? {},
|
|
2107
|
+
runtimeStatus: {
|
|
2108
|
+
state: snapshot.state,
|
|
2109
|
+
acceptingWork: snapshot.acceptingWork,
|
|
2110
|
+
reportedAt
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
};
|
|
2114
|
+
this.applyRuntimeStatusReport(report);
|
|
2115
|
+
if (detailLevel !== "full") {
|
|
2116
|
+
delete report.health;
|
|
2117
|
+
}
|
|
2118
|
+
return report;
|
|
2119
|
+
}
|
|
2120
|
+
selectRuntimeStatusReportForTarget(inquiryResult, targetServiceName, targetServiceInstanceId) {
|
|
2121
|
+
const reports = Array.isArray(inquiryResult.runtimeStatusReports) ? inquiryResult.runtimeStatusReports : [];
|
|
2122
|
+
for (const candidate of reports) {
|
|
2123
|
+
const report = this.normalizeRuntimeStatusReport(candidate);
|
|
2124
|
+
if (!report) {
|
|
2125
|
+
continue;
|
|
2126
|
+
}
|
|
2127
|
+
if (report.serviceName === targetServiceName && report.serviceInstanceId === targetServiceInstanceId) {
|
|
2128
|
+
return report;
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
return null;
|
|
2132
|
+
}
|
|
2133
|
+
async resolveRuntimeStatusFallbackInquiry(serviceName, serviceInstanceId, options = {}) {
|
|
2134
|
+
const inquiryResult = await CadenzaService.inquire(
|
|
2135
|
+
META_RUNTIME_STATUS_INTENT,
|
|
2136
|
+
{
|
|
2137
|
+
targetServiceName: serviceName,
|
|
2138
|
+
targetServiceInstanceId: serviceInstanceId,
|
|
2139
|
+
detailLevel: options.detailLevel ?? "minimal"
|
|
2140
|
+
},
|
|
2141
|
+
{
|
|
2142
|
+
overallTimeoutMs: options.overallTimeoutMs ?? this.runtimeStatusFallbackTimeoutMs,
|
|
2143
|
+
perResponderTimeoutMs: options.perResponderTimeoutMs ?? Math.max(250, Math.floor(this.runtimeStatusFallbackTimeoutMs * 0.75)),
|
|
2144
|
+
requireComplete: options.requireComplete ?? false
|
|
2145
|
+
}
|
|
2146
|
+
);
|
|
2147
|
+
const report = this.selectRuntimeStatusReportForTarget(
|
|
2148
|
+
inquiryResult,
|
|
2149
|
+
serviceName,
|
|
2150
|
+
serviceInstanceId
|
|
2151
|
+
);
|
|
2152
|
+
if (!report) {
|
|
2153
|
+
throw new Error(
|
|
2154
|
+
`No runtime status report for ${serviceName}/${serviceInstanceId}`
|
|
2155
|
+
);
|
|
2156
|
+
}
|
|
2157
|
+
if (!this.applyRuntimeStatusReport(report)) {
|
|
2158
|
+
throw new Error(
|
|
2159
|
+
`No tracked instance for runtime fallback ${serviceName}/${serviceInstanceId}`
|
|
2160
|
+
);
|
|
2161
|
+
}
|
|
2162
|
+
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
2163
|
+
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
2164
|
+
return {
|
|
2165
|
+
report,
|
|
2166
|
+
inquiryMeta: inquiryResult.__inquiryMeta ?? {}
|
|
2167
|
+
};
|
|
2168
|
+
}
|
|
2169
|
+
evaluateDependencyReadinessDetail(serviceName, serviceInstanceId, now = Date.now()) {
|
|
2170
|
+
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
2171
|
+
const missedHeartbeats = this.getHeartbeatMisses(serviceInstanceId, now);
|
|
2172
|
+
const runtimeState = instance ? instance.runtimeState ?? this.resolveRuntimeStatusSnapshot(
|
|
2173
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
2174
|
+
instance.isActive,
|
|
2175
|
+
instance.isNonResponsive,
|
|
2176
|
+
instance.isBlocked
|
|
2177
|
+
).state : "unavailable";
|
|
2178
|
+
const acceptingWork = instance ? typeof instance.acceptingWork === "boolean" ? instance.acceptingWork : this.resolveRuntimeStatusSnapshot(
|
|
2179
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
2180
|
+
instance.isActive,
|
|
2181
|
+
instance.isNonResponsive,
|
|
2182
|
+
instance.isBlocked
|
|
2183
|
+
).acceptingWork : false;
|
|
2184
|
+
const evaluation = evaluateDependencyReadiness({
|
|
2185
|
+
exists: Boolean(instance),
|
|
2186
|
+
runtimeState,
|
|
2187
|
+
acceptingWork,
|
|
2188
|
+
missedHeartbeats,
|
|
2189
|
+
missThreshold: this.runtimeStatusMissThreshold
|
|
2190
|
+
});
|
|
2191
|
+
const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId);
|
|
2192
|
+
return {
|
|
2193
|
+
serviceName,
|
|
2194
|
+
serviceInstanceId,
|
|
2195
|
+
dependencyState: evaluation.state,
|
|
2196
|
+
runtimeState,
|
|
2197
|
+
acceptingWork,
|
|
2198
|
+
missedHeartbeats,
|
|
2199
|
+
stale: evaluation.stale,
|
|
2200
|
+
blocked: evaluation.blocked,
|
|
2201
|
+
reason: evaluation.reason,
|
|
2202
|
+
lastHeartbeatAt: lastHeartbeat ? new Date(lastHeartbeat).toISOString() : null,
|
|
2203
|
+
reportedAt: instance?.reportedAt ?? null
|
|
2204
|
+
};
|
|
2205
|
+
}
|
|
2206
|
+
async buildLocalReadinessReport(options = {}) {
|
|
2207
|
+
const localRuntime = this.buildLocalRuntimeStatusReport("minimal");
|
|
2208
|
+
if (!localRuntime) {
|
|
2209
|
+
return null;
|
|
2210
|
+
}
|
|
2211
|
+
const detailLevel = options.detailLevel ?? "minimal";
|
|
2212
|
+
const includeDependencies = options.includeDependencies ?? detailLevel === "full";
|
|
2213
|
+
const refreshStaleDependencies = options.refreshStaleDependencies ?? true;
|
|
2214
|
+
const dependencyPairs = Array.from(this.readinessDependeesByService.entries()).flatMap(
|
|
2215
|
+
([serviceName, instanceIds]) => Array.from(instanceIds).map((serviceInstanceId) => ({
|
|
2216
|
+
serviceName,
|
|
2217
|
+
serviceInstanceId
|
|
2218
|
+
}))
|
|
2219
|
+
).sort((left, right) => {
|
|
2220
|
+
if (left.serviceName !== right.serviceName) {
|
|
2221
|
+
return left.serviceName.localeCompare(right.serviceName);
|
|
2222
|
+
}
|
|
2223
|
+
return left.serviceInstanceId.localeCompare(right.serviceInstanceId);
|
|
2224
|
+
});
|
|
2225
|
+
if (refreshStaleDependencies) {
|
|
2226
|
+
for (const dependency of dependencyPairs) {
|
|
2227
|
+
const misses = this.getHeartbeatMisses(dependency.serviceInstanceId);
|
|
2228
|
+
if (misses < this.runtimeStatusMissThreshold) {
|
|
2229
|
+
continue;
|
|
2230
|
+
}
|
|
2231
|
+
if (this.runtimeStatusFallbackInFlightByInstance.has(
|
|
2232
|
+
dependency.serviceInstanceId
|
|
2233
|
+
)) {
|
|
2234
|
+
continue;
|
|
2235
|
+
}
|
|
2236
|
+
this.runtimeStatusFallbackInFlightByInstance.add(
|
|
2237
|
+
dependency.serviceInstanceId
|
|
2238
|
+
);
|
|
2239
|
+
try {
|
|
2240
|
+
await this.resolveRuntimeStatusFallbackInquiry(
|
|
2241
|
+
dependency.serviceName,
|
|
2242
|
+
dependency.serviceInstanceId
|
|
2243
|
+
);
|
|
2244
|
+
} catch (error) {
|
|
2245
|
+
CadenzaService.log(
|
|
2246
|
+
"Readiness dependency fallback failed.",
|
|
2247
|
+
{
|
|
2248
|
+
serviceName: dependency.serviceName,
|
|
2249
|
+
serviceInstanceId: dependency.serviceInstanceId,
|
|
2250
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2251
|
+
},
|
|
2252
|
+
"warning"
|
|
2253
|
+
);
|
|
2254
|
+
} finally {
|
|
2255
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(
|
|
2256
|
+
dependency.serviceInstanceId
|
|
2257
|
+
);
|
|
2258
|
+
}
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
const now = Date.now();
|
|
2262
|
+
const dependencyDetails = dependencyPairs.map(
|
|
2263
|
+
(dependency) => this.evaluateDependencyReadinessDetail(
|
|
2264
|
+
dependency.serviceName,
|
|
2265
|
+
dependency.serviceInstanceId,
|
|
2266
|
+
now
|
|
2267
|
+
)
|
|
2268
|
+
);
|
|
2269
|
+
const dependencySummary = summarizeDependencyReadiness(
|
|
2270
|
+
dependencyDetails.map((detail) => ({
|
|
2271
|
+
state: detail.dependencyState,
|
|
2272
|
+
stale: detail.stale,
|
|
2273
|
+
blocked: detail.blocked,
|
|
2274
|
+
reason: detail.reason
|
|
2275
|
+
}))
|
|
2276
|
+
);
|
|
2277
|
+
const readinessState = resolveServiceReadinessState(
|
|
2278
|
+
localRuntime.state,
|
|
2279
|
+
localRuntime.acceptingWork,
|
|
2280
|
+
dependencySummary
|
|
2281
|
+
);
|
|
2282
|
+
return {
|
|
2283
|
+
serviceName: localRuntime.serviceName,
|
|
2284
|
+
serviceInstanceId: localRuntime.serviceInstanceId,
|
|
2285
|
+
reportedAt: new Date(now).toISOString(),
|
|
2286
|
+
readinessState,
|
|
2287
|
+
runtimeState: localRuntime.state,
|
|
2288
|
+
acceptingWork: localRuntime.acceptingWork,
|
|
2289
|
+
dependencySummary,
|
|
2290
|
+
...includeDependencies ? { dependencies: dependencyDetails } : {}
|
|
2291
|
+
};
|
|
2292
|
+
}
|
|
1195
2293
|
reset() {
|
|
1196
2294
|
this.instances.clear();
|
|
1197
2295
|
this.deputies.clear();
|
|
@@ -1199,6 +2297,17 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1199
2297
|
this.remoteIntents.clear();
|
|
1200
2298
|
this.remoteIntentDeputiesByKey.clear();
|
|
1201
2299
|
this.remoteIntentDeputiesByTask.clear();
|
|
2300
|
+
this.dependeesByService.clear();
|
|
2301
|
+
this.dependeeByInstance.clear();
|
|
2302
|
+
this.readinessDependeesByService.clear();
|
|
2303
|
+
this.readinessDependeeByInstance.clear();
|
|
2304
|
+
this.lastHeartbeatAtByInstance.clear();
|
|
2305
|
+
this.missedHeartbeatsByInstance.clear();
|
|
2306
|
+
this.runtimeStatusFallbackInFlightByInstance.clear();
|
|
2307
|
+
this.activeRoutineExecutionIds.clear();
|
|
2308
|
+
this.numberOfRunningGraphs = 0;
|
|
2309
|
+
this.runtimeStatusHeartbeatStarted = false;
|
|
2310
|
+
this.lastRuntimeStatusSnapshot = null;
|
|
1202
2311
|
}
|
|
1203
2312
|
};
|
|
1204
2313
|
|