@cadenza.io/service 2.6.1 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +35 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +1126 -17
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1126 -17
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -298,6 +298,8 @@ var isBrowser = typeof window !== "undefined" && typeof window.document !== "und
|
|
|
298
298
|
// src/utils/inquiry.ts
|
|
299
299
|
var META_INTENT_PREFIX = "meta-";
|
|
300
300
|
var META_RUNTIME_TRANSPORT_DIAGNOSTICS_INTENT = "meta-runtime-transport-diagnostics";
|
|
301
|
+
var META_RUNTIME_STATUS_INTENT = "meta-runtime-status";
|
|
302
|
+
var META_READINESS_INTENT = "meta-readiness";
|
|
301
303
|
function isPlainObject(value) {
|
|
302
304
|
return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
|
|
303
305
|
}
|
|
@@ -361,8 +363,200 @@ function summarizeResponderStatuses(statuses) {
|
|
|
361
363
|
return { responded, failed, timedOut, pending };
|
|
362
364
|
}
|
|
363
365
|
|
|
366
|
+
// src/utils/readiness.ts
|
|
367
|
+
function evaluateDependencyReadiness(input) {
|
|
368
|
+
const missedHeartbeats = Math.max(
|
|
369
|
+
0,
|
|
370
|
+
Math.trunc(Number(input.missedHeartbeats) || 0)
|
|
371
|
+
);
|
|
372
|
+
const stale = missedHeartbeats > 0;
|
|
373
|
+
const timeoutReached = missedHeartbeats >= Math.max(1, input.missThreshold);
|
|
374
|
+
if (!input.exists) {
|
|
375
|
+
return {
|
|
376
|
+
state: "unavailable",
|
|
377
|
+
stale: true,
|
|
378
|
+
blocked: true,
|
|
379
|
+
reason: "missing"
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
if (timeoutReached) {
|
|
383
|
+
return {
|
|
384
|
+
state: "unavailable",
|
|
385
|
+
stale: true,
|
|
386
|
+
blocked: true,
|
|
387
|
+
reason: "heartbeat-timeout"
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
if (input.runtimeState === "unavailable" || !input.acceptingWork) {
|
|
391
|
+
return {
|
|
392
|
+
state: "unavailable",
|
|
393
|
+
stale,
|
|
394
|
+
blocked: true,
|
|
395
|
+
reason: "runtime-unavailable"
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
if (stale) {
|
|
399
|
+
return {
|
|
400
|
+
state: "degraded",
|
|
401
|
+
stale: true,
|
|
402
|
+
blocked: false,
|
|
403
|
+
reason: "heartbeat-stale"
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
if (input.runtimeState === "overloaded") {
|
|
407
|
+
return {
|
|
408
|
+
state: "overloaded",
|
|
409
|
+
stale: false,
|
|
410
|
+
blocked: false,
|
|
411
|
+
reason: "runtime-overloaded"
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
if (input.runtimeState === "degraded") {
|
|
415
|
+
return {
|
|
416
|
+
state: "degraded",
|
|
417
|
+
stale: false,
|
|
418
|
+
blocked: false,
|
|
419
|
+
reason: "runtime-degraded"
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
return {
|
|
423
|
+
state: "ready",
|
|
424
|
+
stale: false,
|
|
425
|
+
blocked: false,
|
|
426
|
+
reason: "runtime-healthy"
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
function summarizeDependencyReadiness(evaluations) {
|
|
430
|
+
const summary = {
|
|
431
|
+
total: evaluations.length,
|
|
432
|
+
ready: 0,
|
|
433
|
+
degraded: 0,
|
|
434
|
+
overloaded: 0,
|
|
435
|
+
unavailable: 0,
|
|
436
|
+
stale: 0
|
|
437
|
+
};
|
|
438
|
+
for (const evaluation of evaluations) {
|
|
439
|
+
if (evaluation.state === "ready") summary.ready++;
|
|
440
|
+
if (evaluation.state === "degraded") summary.degraded++;
|
|
441
|
+
if (evaluation.state === "overloaded") summary.overloaded++;
|
|
442
|
+
if (evaluation.state === "unavailable") summary.unavailable++;
|
|
443
|
+
if (evaluation.stale) summary.stale++;
|
|
444
|
+
}
|
|
445
|
+
return summary;
|
|
446
|
+
}
|
|
447
|
+
function resolveServiceReadinessState(localRuntimeState, localAcceptingWork, dependencySummary) {
|
|
448
|
+
if (localRuntimeState === "unavailable" || !localAcceptingWork) {
|
|
449
|
+
return "blocked";
|
|
450
|
+
}
|
|
451
|
+
if (dependencySummary.unavailable > 0) {
|
|
452
|
+
return "blocked";
|
|
453
|
+
}
|
|
454
|
+
if (dependencySummary.degraded > 0 || dependencySummary.overloaded > 0 || dependencySummary.stale > 0) {
|
|
455
|
+
return "degraded";
|
|
456
|
+
}
|
|
457
|
+
return "ready";
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// src/utils/runtimeStatus.ts
|
|
461
|
+
function resolveRuntimeStatus(input) {
|
|
462
|
+
const numberOfRunningGraphs = Math.max(
|
|
463
|
+
0,
|
|
464
|
+
Math.trunc(Number(input.numberOfRunningGraphs) || 0)
|
|
465
|
+
);
|
|
466
|
+
const isActive = Boolean(input.isActive);
|
|
467
|
+
const isNonResponsive = Boolean(input.isNonResponsive);
|
|
468
|
+
const isBlocked = Boolean(input.isBlocked);
|
|
469
|
+
if (!isActive || isNonResponsive || isBlocked) {
|
|
470
|
+
return {
|
|
471
|
+
state: "unavailable",
|
|
472
|
+
acceptingWork: false,
|
|
473
|
+
numberOfRunningGraphs,
|
|
474
|
+
isActive,
|
|
475
|
+
isNonResponsive,
|
|
476
|
+
isBlocked
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
if (numberOfRunningGraphs >= input.overloadedGraphThreshold) {
|
|
480
|
+
return {
|
|
481
|
+
state: "overloaded",
|
|
482
|
+
acceptingWork: true,
|
|
483
|
+
numberOfRunningGraphs,
|
|
484
|
+
isActive,
|
|
485
|
+
isNonResponsive,
|
|
486
|
+
isBlocked
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
if (numberOfRunningGraphs >= input.degradedGraphThreshold) {
|
|
490
|
+
return {
|
|
491
|
+
state: "degraded",
|
|
492
|
+
acceptingWork: true,
|
|
493
|
+
numberOfRunningGraphs,
|
|
494
|
+
isActive,
|
|
495
|
+
isNonResponsive,
|
|
496
|
+
isBlocked
|
|
497
|
+
};
|
|
498
|
+
}
|
|
499
|
+
return {
|
|
500
|
+
state: "healthy",
|
|
501
|
+
acceptingWork: true,
|
|
502
|
+
numberOfRunningGraphs,
|
|
503
|
+
isActive,
|
|
504
|
+
isNonResponsive,
|
|
505
|
+
isBlocked
|
|
506
|
+
};
|
|
507
|
+
}
|
|
508
|
+
function runtimeStatusPriority(state) {
|
|
509
|
+
switch (state) {
|
|
510
|
+
case "healthy":
|
|
511
|
+
return 0;
|
|
512
|
+
case "degraded":
|
|
513
|
+
return 1;
|
|
514
|
+
case "overloaded":
|
|
515
|
+
return 2;
|
|
516
|
+
case "unavailable":
|
|
517
|
+
return 3;
|
|
518
|
+
default:
|
|
519
|
+
return 4;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
function hasSignificantRuntimeStatusChange(previous, next) {
|
|
523
|
+
if (!previous) {
|
|
524
|
+
return true;
|
|
525
|
+
}
|
|
526
|
+
return previous.state !== next.state || previous.acceptingWork !== next.acceptingWork || previous.isActive !== next.isActive || previous.isNonResponsive !== next.isNonResponsive || previous.isBlocked !== next.isBlocked;
|
|
527
|
+
}
|
|
528
|
+
|
|
364
529
|
// src/registry/ServiceRegistry.ts
|
|
365
530
|
var META_SERVICE_REGISTRY_FULL_SYNC_INTENT = "meta-service-registry-full-sync";
|
|
531
|
+
var META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL = "meta.service_registry.runtime_status.heartbeat_tick";
|
|
532
|
+
var META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL = "meta.service_registry.runtime_status.monitor_tick";
|
|
533
|
+
var INTERNAL_RUNTIME_STATUS_TASK_NAMES = /* @__PURE__ */ new Set([
|
|
534
|
+
"Track local routine start",
|
|
535
|
+
"Track local routine end",
|
|
536
|
+
"Start runtime status sharing intervals",
|
|
537
|
+
"Broadcast runtime status",
|
|
538
|
+
"Monitor dependee heartbeat freshness",
|
|
539
|
+
"Resolve runtime status fallback inquiry",
|
|
540
|
+
"Respond runtime status inquiry",
|
|
541
|
+
"Respond readiness inquiry",
|
|
542
|
+
"Collect distributed readiness",
|
|
543
|
+
"Get status"
|
|
544
|
+
]);
|
|
545
|
+
function readPositiveIntegerEnv(name, fallback) {
|
|
546
|
+
if (typeof process === "undefined") {
|
|
547
|
+
return fallback;
|
|
548
|
+
}
|
|
549
|
+
const raw = process.env?.[name];
|
|
550
|
+
const parsed = Number(raw);
|
|
551
|
+
if (!Number.isFinite(parsed)) {
|
|
552
|
+
return fallback;
|
|
553
|
+
}
|
|
554
|
+
const normalized = Math.trunc(parsed);
|
|
555
|
+
if (normalized <= 0) {
|
|
556
|
+
return fallback;
|
|
557
|
+
}
|
|
558
|
+
return normalized;
|
|
559
|
+
}
|
|
366
560
|
var ServiceRegistry = class _ServiceRegistry {
|
|
367
561
|
/**
|
|
368
562
|
* Initializes a private constructor for managing service instances, remote signals,
|
|
@@ -380,6 +574,36 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
380
574
|
this.remoteIntents = /* @__PURE__ */ new Map();
|
|
381
575
|
this.remoteIntentDeputiesByKey = /* @__PURE__ */ new Map();
|
|
382
576
|
this.remoteIntentDeputiesByTask = /* @__PURE__ */ new Map();
|
|
577
|
+
this.dependeesByService = /* @__PURE__ */ new Map();
|
|
578
|
+
this.dependeeByInstance = /* @__PURE__ */ new Map();
|
|
579
|
+
this.readinessDependeesByService = /* @__PURE__ */ new Map();
|
|
580
|
+
this.readinessDependeeByInstance = /* @__PURE__ */ new Map();
|
|
581
|
+
this.lastHeartbeatAtByInstance = /* @__PURE__ */ new Map();
|
|
582
|
+
this.missedHeartbeatsByInstance = /* @__PURE__ */ new Map();
|
|
583
|
+
this.runtimeStatusFallbackInFlightByInstance = /* @__PURE__ */ new Set();
|
|
584
|
+
this.activeRoutineExecutionIds = /* @__PURE__ */ new Set();
|
|
585
|
+
this.runtimeStatusHeartbeatStarted = false;
|
|
586
|
+
this.lastRuntimeStatusSnapshot = null;
|
|
587
|
+
this.runtimeStatusHeartbeatIntervalMs = readPositiveIntegerEnv(
|
|
588
|
+
"CADENZA_RUNTIME_STATUS_HEARTBEAT_MS",
|
|
589
|
+
3e4
|
|
590
|
+
);
|
|
591
|
+
this.runtimeStatusMissThreshold = readPositiveIntegerEnv(
|
|
592
|
+
"CADENZA_RUNTIME_STATUS_MISSED_HEARTBEATS",
|
|
593
|
+
3
|
|
594
|
+
);
|
|
595
|
+
this.runtimeStatusFallbackTimeoutMs = readPositiveIntegerEnv(
|
|
596
|
+
"CADENZA_RUNTIME_STATUS_FALLBACK_TIMEOUT_MS",
|
|
597
|
+
1500
|
|
598
|
+
);
|
|
599
|
+
this.degradedGraphThreshold = readPositiveIntegerEnv(
|
|
600
|
+
"CADENZA_RUNTIME_STATUS_DEGRADED_GRAPH_THRESHOLD",
|
|
601
|
+
10
|
|
602
|
+
);
|
|
603
|
+
this.overloadedGraphThreshold = readPositiveIntegerEnv(
|
|
604
|
+
"CADENZA_RUNTIME_STATUS_OVERLOADED_GRAPH_THRESHOLD",
|
|
605
|
+
20
|
|
606
|
+
);
|
|
383
607
|
this.serviceName = null;
|
|
384
608
|
this.serviceInstanceId = null;
|
|
385
609
|
this.numberOfRunningGraphs = 0;
|
|
@@ -418,10 +642,137 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
418
642
|
}
|
|
419
643
|
}
|
|
420
644
|
});
|
|
645
|
+
CadenzaService.defineIntent({
|
|
646
|
+
name: META_RUNTIME_STATUS_INTENT,
|
|
647
|
+
description: "Gather lightweight runtime status reports from services in the distributed runtime.",
|
|
648
|
+
input: {
|
|
649
|
+
type: "object",
|
|
650
|
+
properties: {
|
|
651
|
+
detailLevel: {
|
|
652
|
+
type: "string",
|
|
653
|
+
constraints: {
|
|
654
|
+
oneOf: ["minimal", "full"]
|
|
655
|
+
}
|
|
656
|
+
},
|
|
657
|
+
targetServiceName: {
|
|
658
|
+
type: "string"
|
|
659
|
+
},
|
|
660
|
+
targetServiceInstanceId: {
|
|
661
|
+
type: "string"
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
},
|
|
665
|
+
output: {
|
|
666
|
+
type: "object",
|
|
667
|
+
properties: {
|
|
668
|
+
runtimeStatusReports: {
|
|
669
|
+
type: "array"
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
});
|
|
674
|
+
CadenzaService.createMetaTask(
|
|
675
|
+
"Respond runtime status inquiry",
|
|
676
|
+
(ctx) => {
|
|
677
|
+
const targetServiceName = ctx.targetServiceName;
|
|
678
|
+
const targetServiceInstanceId = ctx.targetServiceInstanceId;
|
|
679
|
+
const detailLevel = ctx.detailLevel === "full" ? "full" : "minimal";
|
|
680
|
+
const report = this.buildLocalRuntimeStatusReport(detailLevel);
|
|
681
|
+
if (!report) {
|
|
682
|
+
return {};
|
|
683
|
+
}
|
|
684
|
+
if (targetServiceName && targetServiceName !== report.serviceName) {
|
|
685
|
+
return {};
|
|
686
|
+
}
|
|
687
|
+
if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
|
|
688
|
+
return {};
|
|
689
|
+
}
|
|
690
|
+
return {
|
|
691
|
+
runtimeStatusReports: [report]
|
|
692
|
+
};
|
|
693
|
+
},
|
|
694
|
+
"Responds to runtime-status inquiries with local service instance status."
|
|
695
|
+
).respondsTo(META_RUNTIME_STATUS_INTENT);
|
|
696
|
+
CadenzaService.defineIntent({
|
|
697
|
+
name: META_READINESS_INTENT,
|
|
698
|
+
description: "Gather service readiness reports derived from local runtime status and required dependees.",
|
|
699
|
+
input: {
|
|
700
|
+
type: "object",
|
|
701
|
+
properties: {
|
|
702
|
+
detailLevel: {
|
|
703
|
+
type: "string",
|
|
704
|
+
constraints: {
|
|
705
|
+
oneOf: ["minimal", "full"]
|
|
706
|
+
}
|
|
707
|
+
},
|
|
708
|
+
includeDependencies: {
|
|
709
|
+
type: "boolean"
|
|
710
|
+
},
|
|
711
|
+
refreshStaleDependencies: {
|
|
712
|
+
type: "boolean"
|
|
713
|
+
},
|
|
714
|
+
targetServiceName: {
|
|
715
|
+
type: "string"
|
|
716
|
+
},
|
|
717
|
+
targetServiceInstanceId: {
|
|
718
|
+
type: "string"
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
},
|
|
722
|
+
output: {
|
|
723
|
+
type: "object",
|
|
724
|
+
properties: {
|
|
725
|
+
readinessReports: {
|
|
726
|
+
type: "array"
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
});
|
|
731
|
+
CadenzaService.createMetaTask(
|
|
732
|
+
"Respond readiness inquiry",
|
|
733
|
+
async (ctx) => {
|
|
734
|
+
const targetServiceName = ctx.targetServiceName;
|
|
735
|
+
const targetServiceInstanceId = ctx.targetServiceInstanceId;
|
|
736
|
+
const report = await this.buildLocalReadinessReport({
|
|
737
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
738
|
+
includeDependencies: ctx.includeDependencies,
|
|
739
|
+
refreshStaleDependencies: ctx.refreshStaleDependencies
|
|
740
|
+
});
|
|
741
|
+
if (!report) {
|
|
742
|
+
return {};
|
|
743
|
+
}
|
|
744
|
+
if (targetServiceName && targetServiceName !== report.serviceName) {
|
|
745
|
+
return {};
|
|
746
|
+
}
|
|
747
|
+
if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
|
|
748
|
+
return {};
|
|
749
|
+
}
|
|
750
|
+
return {
|
|
751
|
+
readinessReports: [report]
|
|
752
|
+
};
|
|
753
|
+
},
|
|
754
|
+
"Responds to distributed readiness inquiries using required dependee health."
|
|
755
|
+
).respondsTo(META_READINESS_INTENT);
|
|
421
756
|
this.handleInstanceUpdateTask = CadenzaService.createMetaTask(
|
|
422
757
|
"Handle Instance Update",
|
|
423
758
|
(ctx, emit) => {
|
|
424
|
-
const
|
|
759
|
+
const serviceInstance = ctx.serviceInstance ?? (ctx.__serviceInstanceId || ctx.serviceInstanceId ? {
|
|
760
|
+
uuid: ctx.__serviceInstanceId ?? ctx.serviceInstanceId,
|
|
761
|
+
serviceName: ctx.__serviceName ?? ctx.serviceName,
|
|
762
|
+
address: ctx.serviceAddress ?? "",
|
|
763
|
+
port: ctx.servicePort ?? 0,
|
|
764
|
+
exposed: !!ctx.exposed,
|
|
765
|
+
isFrontend: !!ctx.isFrontend,
|
|
766
|
+
isActive: typeof ctx.isActive === "boolean" ? ctx.isActive : typeof ctx.__active === "boolean" ? ctx.__active : true,
|
|
767
|
+
isNonResponsive: !!ctx.isNonResponsive,
|
|
768
|
+
isBlocked: !!ctx.isBlocked,
|
|
769
|
+
health: ctx.health ?? ctx.__health ?? {},
|
|
770
|
+
numberOfRunningGraphs: ctx.numberOfRunningGraphs ?? ctx.__numberOfRunningGraphs ?? 0,
|
|
771
|
+
isPrimary: false
|
|
772
|
+
} : void 0);
|
|
773
|
+
if (!serviceInstance?.uuid || !serviceInstance?.serviceName) {
|
|
774
|
+
return false;
|
|
775
|
+
}
|
|
425
776
|
const {
|
|
426
777
|
uuid: uuid4,
|
|
427
778
|
serviceName,
|
|
@@ -443,6 +794,7 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
443
794
|
emit(`meta.socket_shutdown_requested:${address}_${port}`, {});
|
|
444
795
|
emit(`meta.fetch.destroy_requested:${address}_${port}`, {});
|
|
445
796
|
}
|
|
797
|
+
this.unregisterDependee(uuid4, serviceName);
|
|
446
798
|
return;
|
|
447
799
|
}
|
|
448
800
|
if (!this.instances.has(serviceName))
|
|
@@ -454,6 +806,18 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
454
806
|
} else {
|
|
455
807
|
instances.push(serviceInstance);
|
|
456
808
|
}
|
|
809
|
+
const trackedInstance = existing ?? instances.find((instance) => instance.uuid === uuid4);
|
|
810
|
+
if (trackedInstance) {
|
|
811
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
812
|
+
trackedInstance.numberOfRunningGraphs ?? 0,
|
|
813
|
+
trackedInstance.isActive,
|
|
814
|
+
trackedInstance.isNonResponsive,
|
|
815
|
+
trackedInstance.isBlocked
|
|
816
|
+
);
|
|
817
|
+
trackedInstance.runtimeState = snapshot.state;
|
|
818
|
+
trackedInstance.acceptingWork = snapshot.acceptingWork;
|
|
819
|
+
trackedInstance.reportedAt = trackedInstance.reportedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
820
|
+
}
|
|
457
821
|
if (this.serviceName === serviceName) {
|
|
458
822
|
return false;
|
|
459
823
|
}
|
|
@@ -493,13 +857,27 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
493
857
|
"global.meta.service_instance.inserted",
|
|
494
858
|
"global.meta.service_instance.updated",
|
|
495
859
|
"meta.service_instance.inserted",
|
|
496
|
-
"meta.service_instance.updated"
|
|
497
|
-
"meta.socket_client.status_received"
|
|
860
|
+
"meta.service_instance.updated"
|
|
498
861
|
).attachSignal(
|
|
499
862
|
"meta.service_registry.dependee_registered",
|
|
500
863
|
"meta.socket_shutdown_requested",
|
|
501
864
|
"meta.fetch.destroy_requested"
|
|
502
865
|
);
|
|
866
|
+
CadenzaService.createMetaTask(
|
|
867
|
+
"Track dependee registration",
|
|
868
|
+
(ctx) => {
|
|
869
|
+
if (!ctx.serviceName || !ctx.serviceInstanceId) {
|
|
870
|
+
return false;
|
|
871
|
+
}
|
|
872
|
+
this.registerDependee(ctx.serviceName, ctx.serviceInstanceId, {
|
|
873
|
+
requiredForReadiness: this.shouldRequireReadinessFromCommunicationTypes(
|
|
874
|
+
ctx.communicationTypes
|
|
875
|
+
)
|
|
876
|
+
});
|
|
877
|
+
return true;
|
|
878
|
+
},
|
|
879
|
+
"Tracks remote dependency instances for runtime heartbeat monitoring."
|
|
880
|
+
).doOn("meta.service_registry.dependee_registered");
|
|
503
881
|
CadenzaService.createMetaTask("Split service instances", function* (ctx) {
|
|
504
882
|
if (!ctx.serviceInstances) {
|
|
505
883
|
return;
|
|
@@ -599,6 +977,15 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
599
977
|
for (const instance of instances ?? []) {
|
|
600
978
|
instance.isActive = false;
|
|
601
979
|
instance.isNonResponsive = true;
|
|
980
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
981
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
982
|
+
instance.isActive,
|
|
983
|
+
instance.isNonResponsive,
|
|
984
|
+
instance.isBlocked
|
|
985
|
+
);
|
|
986
|
+
instance.runtimeState = snapshot.state;
|
|
987
|
+
instance.acceptingWork = snapshot.acceptingWork;
|
|
988
|
+
instance.reportedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
602
989
|
emit("global.meta.service_registry.service_not_responding", {
|
|
603
990
|
data: {
|
|
604
991
|
isActive: false,
|
|
@@ -616,7 +1003,8 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
616
1003
|
"meta.fetch.handshake_failed",
|
|
617
1004
|
"meta.fetch.handshake_failed.*",
|
|
618
1005
|
"meta.socket_client.disconnected",
|
|
619
|
-
"meta.socket_client.disconnected.*"
|
|
1006
|
+
"meta.socket_client.disconnected.*",
|
|
1007
|
+
"meta.service_registry.runtime_status_unreachable"
|
|
620
1008
|
).attachSignal("global.meta.service_registry.service_not_responding");
|
|
621
1009
|
this.handleServiceHandshakeTask = CadenzaService.createMetaTask(
|
|
622
1010
|
"Handle service handshake",
|
|
@@ -631,6 +1019,15 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
631
1019
|
}
|
|
632
1020
|
instance.isActive = true;
|
|
633
1021
|
instance.isNonResponsive = false;
|
|
1022
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
1023
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
1024
|
+
instance.isActive,
|
|
1025
|
+
instance.isNonResponsive,
|
|
1026
|
+
instance.isBlocked
|
|
1027
|
+
);
|
|
1028
|
+
instance.runtimeState = snapshot.state;
|
|
1029
|
+
instance.acceptingWork = snapshot.acceptingWork;
|
|
1030
|
+
instance.reportedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
634
1031
|
emit("global.meta.service_registry.service_handshake", {
|
|
635
1032
|
data: {
|
|
636
1033
|
isActive: instance.isActive,
|
|
@@ -648,6 +1045,7 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
648
1045
|
if (indexToDelete >= 0) {
|
|
649
1046
|
this.instances.get(serviceName)?.splice(indexToDelete, 1);
|
|
650
1047
|
}
|
|
1048
|
+
this.unregisterDependee(i.uuid, serviceName);
|
|
651
1049
|
emit("global.meta.service_registry.deleted", {
|
|
652
1050
|
data: {
|
|
653
1051
|
isActive: false,
|
|
@@ -669,14 +1067,46 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
669
1067
|
this.handleSocketStatusUpdateTask = CadenzaService.createMetaTask(
|
|
670
1068
|
"Handle Socket Status Update",
|
|
671
1069
|
(ctx) => {
|
|
672
|
-
const
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
if (
|
|
677
|
-
|
|
678
|
-
|
|
1070
|
+
const report = this.normalizeRuntimeStatusReport(ctx);
|
|
1071
|
+
if (!report) {
|
|
1072
|
+
return false;
|
|
1073
|
+
}
|
|
1074
|
+
if (report.serviceName === this.serviceName && report.serviceInstanceId === this.serviceInstanceId) {
|
|
1075
|
+
return false;
|
|
1076
|
+
}
|
|
1077
|
+
let applied = this.applyRuntimeStatusReport(report);
|
|
1078
|
+
if (!applied && report.serviceAddress && typeof report.servicePort === "number") {
|
|
1079
|
+
if (!this.instances.has(report.serviceName)) {
|
|
1080
|
+
this.instances.set(report.serviceName, []);
|
|
1081
|
+
}
|
|
1082
|
+
this.instances.get(report.serviceName).push({
|
|
1083
|
+
uuid: report.serviceInstanceId,
|
|
1084
|
+
serviceName: report.serviceName,
|
|
1085
|
+
address: report.serviceAddress,
|
|
1086
|
+
port: report.servicePort,
|
|
1087
|
+
exposed: !!report.exposed,
|
|
1088
|
+
isFrontend: !!report.isFrontend,
|
|
1089
|
+
isActive: report.isActive,
|
|
1090
|
+
isNonResponsive: report.isNonResponsive,
|
|
1091
|
+
isBlocked: report.isBlocked,
|
|
1092
|
+
numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1093
|
+
runtimeState: report.state,
|
|
1094
|
+
acceptingWork: report.acceptingWork,
|
|
1095
|
+
reportedAt: report.reportedAt,
|
|
1096
|
+
health: report.health ?? {},
|
|
1097
|
+
isPrimary: false
|
|
1098
|
+
});
|
|
1099
|
+
applied = true;
|
|
679
1100
|
}
|
|
1101
|
+
if (!applied) {
|
|
1102
|
+
return false;
|
|
1103
|
+
}
|
|
1104
|
+
this.registerDependee(report.serviceName, report.serviceInstanceId);
|
|
1105
|
+
this.lastHeartbeatAtByInstance.set(report.serviceInstanceId, Date.now());
|
|
1106
|
+
this.missedHeartbeatsByInstance.set(report.serviceInstanceId, 0);
|
|
1107
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(
|
|
1108
|
+
report.serviceInstanceId
|
|
1109
|
+
);
|
|
680
1110
|
return true;
|
|
681
1111
|
},
|
|
682
1112
|
"Handles status update from socket broadcast"
|
|
@@ -806,7 +1236,25 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
806
1236
|
const { __serviceName, __triedInstances, __retries, __broadcast } = context;
|
|
807
1237
|
let retries = __retries ?? 0;
|
|
808
1238
|
let triedInstances = __triedInstances ?? [];
|
|
809
|
-
const instances = this.instances.get(__serviceName)?.filter((i) => i.isActive && !i.isNonResponsive && !i.isBlocked).sort((a, b) =>
|
|
1239
|
+
const instances = this.instances.get(__serviceName)?.filter((i) => i.isActive && !i.isNonResponsive && !i.isBlocked).sort((a, b) => {
|
|
1240
|
+
const leftStatus = this.resolveRuntimeStatusSnapshot(
|
|
1241
|
+
a.numberOfRunningGraphs ?? 0,
|
|
1242
|
+
a.isActive,
|
|
1243
|
+
a.isNonResponsive,
|
|
1244
|
+
a.isBlocked
|
|
1245
|
+
);
|
|
1246
|
+
const rightStatus = this.resolveRuntimeStatusSnapshot(
|
|
1247
|
+
b.numberOfRunningGraphs ?? 0,
|
|
1248
|
+
b.isActive,
|
|
1249
|
+
b.isNonResponsive,
|
|
1250
|
+
b.isBlocked
|
|
1251
|
+
);
|
|
1252
|
+
const priorityDelta = runtimeStatusPriority(leftStatus.state) - runtimeStatusPriority(rightStatus.state);
|
|
1253
|
+
if (priorityDelta !== 0) {
|
|
1254
|
+
return priorityDelta;
|
|
1255
|
+
}
|
|
1256
|
+
return (a.numberOfRunningGraphs ?? 0) - (b.numberOfRunningGraphs ?? 0);
|
|
1257
|
+
});
|
|
810
1258
|
if (!instances || instances.length === 0 || retries > this.retryCount) {
|
|
811
1259
|
context.errored = true;
|
|
812
1260
|
context.__error = `No active instances for ${__serviceName}. Retries: ${retries}. ${this.instances.get(
|
|
@@ -892,15 +1340,292 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
892
1340
|
errored: true
|
|
893
1341
|
};
|
|
894
1342
|
}
|
|
895
|
-
const
|
|
1343
|
+
const report = this.buildLocalRuntimeStatusReport("full");
|
|
1344
|
+
if (!report) {
|
|
1345
|
+
return {
|
|
1346
|
+
...ctx,
|
|
1347
|
+
__status: "error",
|
|
1348
|
+
__error: "No local service instance available for status check",
|
|
1349
|
+
errored: true
|
|
1350
|
+
};
|
|
1351
|
+
}
|
|
896
1352
|
return {
|
|
897
1353
|
...ctx,
|
|
898
1354
|
__status: "ok",
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
1355
|
+
__serviceName: report.serviceName,
|
|
1356
|
+
__serviceInstanceId: report.serviceInstanceId,
|
|
1357
|
+
__numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1358
|
+
__health: report.health ?? {},
|
|
1359
|
+
__active: report.isActive,
|
|
1360
|
+
reportedAt: report.reportedAt,
|
|
1361
|
+
serviceName: report.serviceName,
|
|
1362
|
+
serviceInstanceId: report.serviceInstanceId,
|
|
1363
|
+
numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1364
|
+
health: report.health ?? {},
|
|
1365
|
+
isActive: report.isActive,
|
|
1366
|
+
isNonResponsive: report.isNonResponsive,
|
|
1367
|
+
isBlocked: report.isBlocked,
|
|
1368
|
+
state: report.state,
|
|
1369
|
+
acceptingWork: report.acceptingWork
|
|
902
1370
|
};
|
|
903
|
-
}).doOn(
|
|
1371
|
+
}).doOn(
|
|
1372
|
+
"meta.socket.status_check_requested",
|
|
1373
|
+
"meta.rest.status_check_requested"
|
|
1374
|
+
);
|
|
1375
|
+
CadenzaService.createMetaTask(
|
|
1376
|
+
"Track local routine start",
|
|
1377
|
+
(ctx, emit) => {
|
|
1378
|
+
const sourceTaskName = String(ctx.__signalEmission?.taskName ?? "");
|
|
1379
|
+
if (INTERNAL_RUNTIME_STATUS_TASK_NAMES.has(sourceTaskName)) {
|
|
1380
|
+
return false;
|
|
1381
|
+
}
|
|
1382
|
+
const routineId = String(
|
|
1383
|
+
ctx.filter?.uuid ?? ctx.__routineExecId ?? ""
|
|
1384
|
+
);
|
|
1385
|
+
if (!routineId) {
|
|
1386
|
+
return false;
|
|
1387
|
+
}
|
|
1388
|
+
this.activeRoutineExecutionIds.add(routineId);
|
|
1389
|
+
this.numberOfRunningGraphs = this.activeRoutineExecutionIds.size;
|
|
1390
|
+
const localInstance = this.getLocalInstance();
|
|
1391
|
+
if (!localInstance) {
|
|
1392
|
+
return true;
|
|
1393
|
+
}
|
|
1394
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
1395
|
+
this.numberOfRunningGraphs,
|
|
1396
|
+
localInstance.isActive,
|
|
1397
|
+
localInstance.isNonResponsive,
|
|
1398
|
+
localInstance.isBlocked
|
|
1399
|
+
);
|
|
1400
|
+
if (hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
|
|
1401
|
+
emit("meta.service_registry.runtime_status_broadcast_requested", {
|
|
1402
|
+
reason: "runtime-state-change"
|
|
1403
|
+
});
|
|
1404
|
+
}
|
|
1405
|
+
return true;
|
|
1406
|
+
},
|
|
1407
|
+
"Tracks local routine starts for runtime load status."
|
|
1408
|
+
).doOn("meta.node.started_routine_execution");
|
|
1409
|
+
CadenzaService.createMetaTask(
|
|
1410
|
+
"Track local routine end",
|
|
1411
|
+
(ctx, emit) => {
|
|
1412
|
+
const sourceTaskName = String(ctx.__signalEmission?.taskName ?? "");
|
|
1413
|
+
if (INTERNAL_RUNTIME_STATUS_TASK_NAMES.has(sourceTaskName)) {
|
|
1414
|
+
return false;
|
|
1415
|
+
}
|
|
1416
|
+
const routineId = String(
|
|
1417
|
+
ctx.filter?.uuid ?? ctx.__routineExecId ?? ""
|
|
1418
|
+
);
|
|
1419
|
+
if (!routineId) {
|
|
1420
|
+
return false;
|
|
1421
|
+
}
|
|
1422
|
+
this.activeRoutineExecutionIds.delete(routineId);
|
|
1423
|
+
this.numberOfRunningGraphs = this.activeRoutineExecutionIds.size;
|
|
1424
|
+
const localInstance = this.getLocalInstance();
|
|
1425
|
+
if (!localInstance) {
|
|
1426
|
+
return true;
|
|
1427
|
+
}
|
|
1428
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
1429
|
+
this.numberOfRunningGraphs,
|
|
1430
|
+
localInstance.isActive,
|
|
1431
|
+
localInstance.isNonResponsive,
|
|
1432
|
+
localInstance.isBlocked
|
|
1433
|
+
);
|
|
1434
|
+
if (hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
|
|
1435
|
+
emit("meta.service_registry.runtime_status_broadcast_requested", {
|
|
1436
|
+
reason: "runtime-state-change"
|
|
1437
|
+
});
|
|
1438
|
+
}
|
|
1439
|
+
return true;
|
|
1440
|
+
},
|
|
1441
|
+
"Tracks local routine completion for runtime load status."
|
|
1442
|
+
).doOn("meta.node.ended_routine_execution");
|
|
1443
|
+
CadenzaService.createMetaTask(
|
|
1444
|
+
"Start runtime status sharing intervals",
|
|
1445
|
+
() => {
|
|
1446
|
+
if (this.runtimeStatusHeartbeatStarted) {
|
|
1447
|
+
return false;
|
|
1448
|
+
}
|
|
1449
|
+
this.runtimeStatusHeartbeatStarted = true;
|
|
1450
|
+
CadenzaService.interval(
|
|
1451
|
+
META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL,
|
|
1452
|
+
{ reason: "heartbeat" },
|
|
1453
|
+
this.runtimeStatusHeartbeatIntervalMs,
|
|
1454
|
+
true
|
|
1455
|
+
);
|
|
1456
|
+
CadenzaService.interval(
|
|
1457
|
+
META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL,
|
|
1458
|
+
{},
|
|
1459
|
+
this.runtimeStatusHeartbeatIntervalMs
|
|
1460
|
+
);
|
|
1461
|
+
return true;
|
|
1462
|
+
},
|
|
1463
|
+
"Starts runtime status heartbeat and heartbeat-monitor loops once per service instance."
|
|
1464
|
+
).doOn("meta.service_registry.instance_inserted");
|
|
1465
|
+
CadenzaService.createMetaTask(
|
|
1466
|
+
"Broadcast runtime status",
|
|
1467
|
+
(ctx, emit) => {
|
|
1468
|
+
const report = this.buildLocalRuntimeStatusReport(
|
|
1469
|
+
ctx.detailLevel === "full" ? "full" : "minimal"
|
|
1470
|
+
);
|
|
1471
|
+
if (!report) {
|
|
1472
|
+
return false;
|
|
1473
|
+
}
|
|
1474
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
1475
|
+
report.numberOfRunningGraphs,
|
|
1476
|
+
report.isActive,
|
|
1477
|
+
report.isNonResponsive,
|
|
1478
|
+
report.isBlocked
|
|
1479
|
+
);
|
|
1480
|
+
const force = ctx.reason === "heartbeat" || ctx.force === true || this.lastRuntimeStatusSnapshot === null;
|
|
1481
|
+
if (!force && !hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
|
|
1482
|
+
return false;
|
|
1483
|
+
}
|
|
1484
|
+
this.lastRuntimeStatusSnapshot = snapshot;
|
|
1485
|
+
emit("meta.service.updated", {
|
|
1486
|
+
__serviceName: report.serviceName,
|
|
1487
|
+
__serviceInstanceId: report.serviceInstanceId,
|
|
1488
|
+
__reportedAt: report.reportedAt,
|
|
1489
|
+
__numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1490
|
+
__health: report.health ?? {},
|
|
1491
|
+
__active: report.isActive,
|
|
1492
|
+
serviceName: report.serviceName,
|
|
1493
|
+
serviceInstanceId: report.serviceInstanceId,
|
|
1494
|
+
serviceAddress: report.serviceAddress,
|
|
1495
|
+
servicePort: report.servicePort,
|
|
1496
|
+
exposed: report.exposed,
|
|
1497
|
+
isFrontend: report.isFrontend,
|
|
1498
|
+
reportedAt: report.reportedAt,
|
|
1499
|
+
numberOfRunningGraphs: report.numberOfRunningGraphs,
|
|
1500
|
+
health: report.health ?? {},
|
|
1501
|
+
isActive: report.isActive,
|
|
1502
|
+
isNonResponsive: report.isNonResponsive,
|
|
1503
|
+
isBlocked: report.isBlocked,
|
|
1504
|
+
state: report.state,
|
|
1505
|
+
acceptingWork: report.acceptingWork
|
|
1506
|
+
});
|
|
1507
|
+
return true;
|
|
1508
|
+
},
|
|
1509
|
+
"Broadcasts local runtime status to connected dependees."
|
|
1510
|
+
).doOn(
|
|
1511
|
+
META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL,
|
|
1512
|
+
"meta.service_registry.runtime_status_broadcast_requested"
|
|
1513
|
+
);
|
|
1514
|
+
CadenzaService.createMetaTask(
|
|
1515
|
+
"Monitor dependee heartbeat freshness",
|
|
1516
|
+
(ctx, emit) => {
|
|
1517
|
+
if (!this.useSocket) {
|
|
1518
|
+
return false;
|
|
1519
|
+
}
|
|
1520
|
+
const now = Date.now();
|
|
1521
|
+
for (const [serviceName, instanceIds] of this.dependeesByService) {
|
|
1522
|
+
for (const serviceInstanceId of instanceIds) {
|
|
1523
|
+
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
1524
|
+
if (!instance || !instance.isActive || instance.isBlocked) {
|
|
1525
|
+
continue;
|
|
1526
|
+
}
|
|
1527
|
+
const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
|
|
1528
|
+
const misses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
|
|
1529
|
+
const heartbeatBudget = this.runtimeStatusHeartbeatIntervalMs * (misses + 1);
|
|
1530
|
+
if (lastHeartbeat > 0 && now - lastHeartbeat < heartbeatBudget) {
|
|
1531
|
+
continue;
|
|
1532
|
+
}
|
|
1533
|
+
const nextMisses = misses + 1;
|
|
1534
|
+
this.missedHeartbeatsByInstance.set(serviceInstanceId, nextMisses);
|
|
1535
|
+
if (nextMisses < this.runtimeStatusMissThreshold || this.runtimeStatusFallbackInFlightByInstance.has(serviceInstanceId)) {
|
|
1536
|
+
continue;
|
|
1537
|
+
}
|
|
1538
|
+
this.runtimeStatusFallbackInFlightByInstance.add(serviceInstanceId);
|
|
1539
|
+
emit("meta.service_registry.runtime_status_fallback_requested", {
|
|
1540
|
+
...ctx,
|
|
1541
|
+
serviceName,
|
|
1542
|
+
serviceInstanceId,
|
|
1543
|
+
serviceAddress: instance.address,
|
|
1544
|
+
servicePort: instance.port
|
|
1545
|
+
});
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
return true;
|
|
1549
|
+
},
|
|
1550
|
+
"Monitors dependee heartbeat freshness and requests inquiry fallback after repeated misses."
|
|
1551
|
+
).doOn(META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL);
|
|
1552
|
+
CadenzaService.createMetaTask(
|
|
1553
|
+
"Resolve runtime status fallback inquiry",
|
|
1554
|
+
async (ctx, emit) => {
|
|
1555
|
+
const serviceName = ctx.serviceName;
|
|
1556
|
+
const serviceInstanceId = ctx.serviceInstanceId;
|
|
1557
|
+
if (!serviceName || !serviceInstanceId) {
|
|
1558
|
+
return false;
|
|
1559
|
+
}
|
|
1560
|
+
try {
|
|
1561
|
+
const { report, inquiryMeta } = await this.resolveRuntimeStatusFallbackInquiry(
|
|
1562
|
+
serviceName,
|
|
1563
|
+
serviceInstanceId,
|
|
1564
|
+
{
|
|
1565
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
1566
|
+
overallTimeoutMs: ctx.overallTimeoutMs,
|
|
1567
|
+
perResponderTimeoutMs: ctx.perResponderTimeoutMs,
|
|
1568
|
+
requireComplete: ctx.requireComplete
|
|
1569
|
+
}
|
|
1570
|
+
);
|
|
1571
|
+
return {
|
|
1572
|
+
...ctx,
|
|
1573
|
+
runtimeStatusReport: report,
|
|
1574
|
+
__inquiryMeta: inquiryMeta
|
|
1575
|
+
};
|
|
1576
|
+
} catch (error) {
|
|
1577
|
+
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
1578
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1579
|
+
CadenzaService.log(
|
|
1580
|
+
"Runtime status fallback inquiry failed.",
|
|
1581
|
+
{
|
|
1582
|
+
serviceName,
|
|
1583
|
+
serviceInstanceId,
|
|
1584
|
+
error: message
|
|
1585
|
+
},
|
|
1586
|
+
"warning"
|
|
1587
|
+
);
|
|
1588
|
+
emit("meta.service_registry.runtime_status_unreachable", {
|
|
1589
|
+
...ctx,
|
|
1590
|
+
serviceName,
|
|
1591
|
+
serviceInstanceId,
|
|
1592
|
+
serviceAddress: instance?.address ?? ctx.serviceAddress,
|
|
1593
|
+
servicePort: instance?.port ?? ctx.servicePort,
|
|
1594
|
+
__error: message,
|
|
1595
|
+
errored: true
|
|
1596
|
+
});
|
|
1597
|
+
return {
|
|
1598
|
+
...ctx,
|
|
1599
|
+
__error: message,
|
|
1600
|
+
errored: true
|
|
1601
|
+
};
|
|
1602
|
+
} finally {
|
|
1603
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
|
|
1604
|
+
}
|
|
1605
|
+
},
|
|
1606
|
+
"Runs runtime-status inquiry fallback for a dependee instance after missed heartbeats."
|
|
1607
|
+
).doOn("meta.service_registry.runtime_status_fallback_requested").emits("meta.service_registry.runtime_status_fallback_resolved").emitsOnFail("meta.service_registry.runtime_status_fallback_failed");
|
|
1608
|
+
this.collectReadinessTask = CadenzaService.createMetaTask(
|
|
1609
|
+
"Collect distributed readiness",
|
|
1610
|
+
async (ctx) => {
|
|
1611
|
+
const inquiryResult = await CadenzaService.inquire(
|
|
1612
|
+
META_READINESS_INTENT,
|
|
1613
|
+
{
|
|
1614
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
1615
|
+
includeDependencies: ctx.includeDependencies,
|
|
1616
|
+
refreshStaleDependencies: ctx.refreshStaleDependencies,
|
|
1617
|
+
targetServiceName: ctx.targetServiceName,
|
|
1618
|
+
targetServiceInstanceId: ctx.targetServiceInstanceId
|
|
1619
|
+
},
|
|
1620
|
+
ctx.inquiryOptions ?? ctx.__inquiryOptions ?? {}
|
|
1621
|
+
);
|
|
1622
|
+
return {
|
|
1623
|
+
...ctx,
|
|
1624
|
+
...inquiryResult
|
|
1625
|
+
};
|
|
1626
|
+
},
|
|
1627
|
+
"Collects distributed readiness reports from services."
|
|
1628
|
+
).doOn("meta.service_registry.readiness_requested").emits("meta.service_registry.readiness_collected").emitsOnFail("meta.service_registry.readiness_failed");
|
|
904
1629
|
this.collectTransportDiagnosticsTask = CadenzaService.createMetaTask(
|
|
905
1630
|
"Collect transport diagnostics",
|
|
906
1631
|
async (ctx) => {
|
|
@@ -1240,6 +1965,379 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1240
1965
|
localTaskName: task.name
|
|
1241
1966
|
};
|
|
1242
1967
|
}
|
|
1968
|
+
getInstance(serviceName, instanceId) {
|
|
1969
|
+
return this.instances.get(serviceName)?.find((instance) => instance.uuid === instanceId);
|
|
1970
|
+
}
|
|
1971
|
+
getLocalInstance() {
|
|
1972
|
+
if (!this.serviceName || !this.serviceInstanceId) {
|
|
1973
|
+
return void 0;
|
|
1974
|
+
}
|
|
1975
|
+
return this.getInstance(this.serviceName, this.serviceInstanceId);
|
|
1976
|
+
}
|
|
1977
|
+
registerDependee(serviceName, serviceInstanceId, options = {}) {
|
|
1978
|
+
if (!serviceName || !serviceInstanceId) {
|
|
1979
|
+
return;
|
|
1980
|
+
}
|
|
1981
|
+
if (!this.dependeesByService.has(serviceName)) {
|
|
1982
|
+
this.dependeesByService.set(serviceName, /* @__PURE__ */ new Set());
|
|
1983
|
+
}
|
|
1984
|
+
this.dependeesByService.get(serviceName).add(serviceInstanceId);
|
|
1985
|
+
this.dependeeByInstance.set(serviceInstanceId, serviceName);
|
|
1986
|
+
if (options.requiredForReadiness) {
|
|
1987
|
+
if (!this.readinessDependeesByService.has(serviceName)) {
|
|
1988
|
+
this.readinessDependeesByService.set(serviceName, /* @__PURE__ */ new Set());
|
|
1989
|
+
}
|
|
1990
|
+
this.readinessDependeesByService.get(serviceName).add(serviceInstanceId);
|
|
1991
|
+
this.readinessDependeeByInstance.set(serviceInstanceId, serviceName);
|
|
1992
|
+
}
|
|
1993
|
+
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
1994
|
+
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
1995
|
+
}
|
|
1996
|
+
unregisterDependee(serviceInstanceId, serviceName) {
|
|
1997
|
+
const dependeeServiceName = serviceName ?? this.dependeeByInstance.get(serviceInstanceId);
|
|
1998
|
+
if (dependeeServiceName) {
|
|
1999
|
+
this.dependeesByService.get(dependeeServiceName)?.delete(serviceInstanceId);
|
|
2000
|
+
if (!this.dependeesByService.get(dependeeServiceName)?.size) {
|
|
2001
|
+
this.dependeesByService.delete(dependeeServiceName);
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
this.dependeeByInstance.delete(serviceInstanceId);
|
|
2005
|
+
const readinessDependeeServiceName = serviceName ?? this.readinessDependeeByInstance.get(serviceInstanceId);
|
|
2006
|
+
if (readinessDependeeServiceName) {
|
|
2007
|
+
this.readinessDependeesByService.get(readinessDependeeServiceName)?.delete(serviceInstanceId);
|
|
2008
|
+
if (!this.readinessDependeesByService.get(readinessDependeeServiceName)?.size) {
|
|
2009
|
+
this.readinessDependeesByService.delete(readinessDependeeServiceName);
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
this.readinessDependeeByInstance.delete(serviceInstanceId);
|
|
2013
|
+
this.lastHeartbeatAtByInstance.delete(serviceInstanceId);
|
|
2014
|
+
this.missedHeartbeatsByInstance.delete(serviceInstanceId);
|
|
2015
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
|
|
2016
|
+
}
|
|
2017
|
+
getHeartbeatMisses(serviceInstanceId, now = Date.now()) {
|
|
2018
|
+
const observedMisses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
|
|
2019
|
+
const lastHeartbeatAt = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
|
|
2020
|
+
if (lastHeartbeatAt <= 0) {
|
|
2021
|
+
return Math.max(observedMisses, this.runtimeStatusMissThreshold);
|
|
2022
|
+
}
|
|
2023
|
+
const estimatedMisses = Math.max(
|
|
2024
|
+
0,
|
|
2025
|
+
Math.floor((now - lastHeartbeatAt) / this.runtimeStatusHeartbeatIntervalMs)
|
|
2026
|
+
);
|
|
2027
|
+
return Math.max(observedMisses, estimatedMisses);
|
|
2028
|
+
}
|
|
2029
|
+
shouldRequireReadinessFromCommunicationTypes(communicationTypes) {
|
|
2030
|
+
if (!Array.isArray(communicationTypes)) {
|
|
2031
|
+
return false;
|
|
2032
|
+
}
|
|
2033
|
+
return communicationTypes.some((type) => {
|
|
2034
|
+
const normalized = String(type).toLowerCase();
|
|
2035
|
+
return normalized === "delegation" || normalized === "inquiry";
|
|
2036
|
+
});
|
|
2037
|
+
}
|
|
2038
|
+
resolveRuntimeStatusSnapshot(numberOfRunningGraphs, isActive, isNonResponsive, isBlocked) {
|
|
2039
|
+
return resolveRuntimeStatus({
|
|
2040
|
+
numberOfRunningGraphs,
|
|
2041
|
+
isActive,
|
|
2042
|
+
isNonResponsive,
|
|
2043
|
+
isBlocked,
|
|
2044
|
+
degradedGraphThreshold: this.degradedGraphThreshold,
|
|
2045
|
+
overloadedGraphThreshold: this.overloadedGraphThreshold
|
|
2046
|
+
});
|
|
2047
|
+
}
|
|
2048
|
+
normalizeRuntimeStatusReport(ctx) {
|
|
2049
|
+
const serviceName = ctx.serviceName ?? ctx.__serviceName ?? ctx.serviceInstance?.serviceName;
|
|
2050
|
+
const serviceInstanceId = ctx.serviceInstanceId ?? ctx.__serviceInstanceId ?? ctx.serviceInstance?.uuid;
|
|
2051
|
+
if (!serviceName || !serviceInstanceId) {
|
|
2052
|
+
return null;
|
|
2053
|
+
}
|
|
2054
|
+
const servicePort = ctx.servicePort ?? ctx.port ?? ctx.serviceInstance?.port;
|
|
2055
|
+
const numberOfRunningGraphs = Math.max(
|
|
2056
|
+
0,
|
|
2057
|
+
Math.trunc(
|
|
2058
|
+
Number(ctx.numberOfRunningGraphs ?? ctx.__numberOfRunningGraphs ?? 0)
|
|
2059
|
+
)
|
|
2060
|
+
);
|
|
2061
|
+
const isActive = Boolean(ctx.isActive ?? ctx.__active ?? true);
|
|
2062
|
+
const isNonResponsive = Boolean(ctx.isNonResponsive ?? false);
|
|
2063
|
+
const isBlocked = Boolean(ctx.isBlocked ?? false);
|
|
2064
|
+
const resolved = this.resolveRuntimeStatusSnapshot(
|
|
2065
|
+
numberOfRunningGraphs,
|
|
2066
|
+
isActive,
|
|
2067
|
+
isNonResponsive,
|
|
2068
|
+
isBlocked
|
|
2069
|
+
);
|
|
2070
|
+
return {
|
|
2071
|
+
serviceName,
|
|
2072
|
+
serviceInstanceId,
|
|
2073
|
+
serviceAddress: ctx.serviceAddress ?? ctx.address ?? ctx.serviceInstance?.address,
|
|
2074
|
+
servicePort: typeof servicePort === "number" ? servicePort : void 0,
|
|
2075
|
+
exposed: typeof ctx.exposed === "boolean" ? ctx.exposed : typeof ctx.serviceInstance?.exposed === "boolean" ? ctx.serviceInstance.exposed : void 0,
|
|
2076
|
+
isFrontend: typeof ctx.isFrontend === "boolean" ? ctx.isFrontend : typeof ctx.serviceInstance?.isFrontend === "boolean" ? ctx.serviceInstance.isFrontend : void 0,
|
|
2077
|
+
reportedAt: ctx.reportedAt ?? (typeof ctx.__reportedAt === "string" ? ctx.__reportedAt : void 0) ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
2078
|
+
state: ctx.state === "healthy" || ctx.state === "degraded" || ctx.state === "overloaded" || ctx.state === "unavailable" ? ctx.state : resolved.state,
|
|
2079
|
+
acceptingWork: typeof ctx.acceptingWork === "boolean" ? ctx.acceptingWork : resolved.acceptingWork,
|
|
2080
|
+
numberOfRunningGraphs,
|
|
2081
|
+
isActive,
|
|
2082
|
+
isNonResponsive,
|
|
2083
|
+
isBlocked,
|
|
2084
|
+
health: ctx.health ?? ctx.__health ?? {}
|
|
2085
|
+
};
|
|
2086
|
+
}
|
|
2087
|
+
applyRuntimeStatusReport(report) {
|
|
2088
|
+
const instance = this.getInstance(report.serviceName, report.serviceInstanceId);
|
|
2089
|
+
if (!instance) {
|
|
2090
|
+
return false;
|
|
2091
|
+
}
|
|
2092
|
+
if (report.serviceAddress) {
|
|
2093
|
+
instance.address = report.serviceAddress;
|
|
2094
|
+
}
|
|
2095
|
+
if (typeof report.servicePort === "number") {
|
|
2096
|
+
instance.port = report.servicePort;
|
|
2097
|
+
}
|
|
2098
|
+
if (typeof report.exposed === "boolean") {
|
|
2099
|
+
instance.exposed = report.exposed;
|
|
2100
|
+
}
|
|
2101
|
+
if (typeof report.isFrontend === "boolean") {
|
|
2102
|
+
instance.isFrontend = report.isFrontend;
|
|
2103
|
+
}
|
|
2104
|
+
instance.numberOfRunningGraphs = report.numberOfRunningGraphs;
|
|
2105
|
+
instance.isActive = report.isActive;
|
|
2106
|
+
instance.isNonResponsive = report.isNonResponsive;
|
|
2107
|
+
instance.isBlocked = report.isBlocked;
|
|
2108
|
+
instance.runtimeState = report.state;
|
|
2109
|
+
instance.acceptingWork = report.acceptingWork;
|
|
2110
|
+
instance.reportedAt = report.reportedAt;
|
|
2111
|
+
instance.health = {
|
|
2112
|
+
...instance.health ?? {},
|
|
2113
|
+
...report.health ?? {},
|
|
2114
|
+
runtimeStatus: {
|
|
2115
|
+
state: report.state,
|
|
2116
|
+
acceptingWork: report.acceptingWork,
|
|
2117
|
+
reportedAt: report.reportedAt
|
|
2118
|
+
}
|
|
2119
|
+
};
|
|
2120
|
+
return true;
|
|
2121
|
+
}
|
|
2122
|
+
buildLocalRuntimeStatusReport(detailLevel = "minimal") {
|
|
2123
|
+
if (!this.serviceName || !this.serviceInstanceId) {
|
|
2124
|
+
return null;
|
|
2125
|
+
}
|
|
2126
|
+
const localInstance = this.getLocalInstance();
|
|
2127
|
+
if (!localInstance) {
|
|
2128
|
+
return null;
|
|
2129
|
+
}
|
|
2130
|
+
const numberOfRunningGraphs = this.activeRoutineExecutionIds.size || this.numberOfRunningGraphs || 0;
|
|
2131
|
+
this.numberOfRunningGraphs = numberOfRunningGraphs;
|
|
2132
|
+
const snapshot = this.resolveRuntimeStatusSnapshot(
|
|
2133
|
+
numberOfRunningGraphs,
|
|
2134
|
+
localInstance.isActive,
|
|
2135
|
+
localInstance.isNonResponsive,
|
|
2136
|
+
localInstance.isBlocked
|
|
2137
|
+
);
|
|
2138
|
+
const reportedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2139
|
+
const report = {
|
|
2140
|
+
serviceName: this.serviceName,
|
|
2141
|
+
serviceInstanceId: this.serviceInstanceId,
|
|
2142
|
+
serviceAddress: localInstance.address,
|
|
2143
|
+
servicePort: localInstance.port,
|
|
2144
|
+
exposed: localInstance.exposed,
|
|
2145
|
+
isFrontend: localInstance.isFrontend,
|
|
2146
|
+
reportedAt,
|
|
2147
|
+
state: snapshot.state,
|
|
2148
|
+
acceptingWork: snapshot.acceptingWork,
|
|
2149
|
+
numberOfRunningGraphs: snapshot.numberOfRunningGraphs,
|
|
2150
|
+
isActive: snapshot.isActive,
|
|
2151
|
+
isNonResponsive: snapshot.isNonResponsive,
|
|
2152
|
+
isBlocked: snapshot.isBlocked,
|
|
2153
|
+
health: {
|
|
2154
|
+
...localInstance.health ?? {},
|
|
2155
|
+
runtimeStatus: {
|
|
2156
|
+
state: snapshot.state,
|
|
2157
|
+
acceptingWork: snapshot.acceptingWork,
|
|
2158
|
+
reportedAt
|
|
2159
|
+
}
|
|
2160
|
+
}
|
|
2161
|
+
};
|
|
2162
|
+
this.applyRuntimeStatusReport(report);
|
|
2163
|
+
if (detailLevel !== "full") {
|
|
2164
|
+
delete report.health;
|
|
2165
|
+
}
|
|
2166
|
+
return report;
|
|
2167
|
+
}
|
|
2168
|
+
selectRuntimeStatusReportForTarget(inquiryResult, targetServiceName, targetServiceInstanceId) {
|
|
2169
|
+
const reports = Array.isArray(inquiryResult.runtimeStatusReports) ? inquiryResult.runtimeStatusReports : [];
|
|
2170
|
+
for (const candidate of reports) {
|
|
2171
|
+
const report = this.normalizeRuntimeStatusReport(candidate);
|
|
2172
|
+
if (!report) {
|
|
2173
|
+
continue;
|
|
2174
|
+
}
|
|
2175
|
+
if (report.serviceName === targetServiceName && report.serviceInstanceId === targetServiceInstanceId) {
|
|
2176
|
+
return report;
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
return null;
|
|
2180
|
+
}
|
|
2181
|
+
async resolveRuntimeStatusFallbackInquiry(serviceName, serviceInstanceId, options = {}) {
|
|
2182
|
+
const inquiryResult = await CadenzaService.inquire(
|
|
2183
|
+
META_RUNTIME_STATUS_INTENT,
|
|
2184
|
+
{
|
|
2185
|
+
targetServiceName: serviceName,
|
|
2186
|
+
targetServiceInstanceId: serviceInstanceId,
|
|
2187
|
+
detailLevel: options.detailLevel ?? "minimal"
|
|
2188
|
+
},
|
|
2189
|
+
{
|
|
2190
|
+
overallTimeoutMs: options.overallTimeoutMs ?? this.runtimeStatusFallbackTimeoutMs,
|
|
2191
|
+
perResponderTimeoutMs: options.perResponderTimeoutMs ?? Math.max(250, Math.floor(this.runtimeStatusFallbackTimeoutMs * 0.75)),
|
|
2192
|
+
requireComplete: options.requireComplete ?? false
|
|
2193
|
+
}
|
|
2194
|
+
);
|
|
2195
|
+
const report = this.selectRuntimeStatusReportForTarget(
|
|
2196
|
+
inquiryResult,
|
|
2197
|
+
serviceName,
|
|
2198
|
+
serviceInstanceId
|
|
2199
|
+
);
|
|
2200
|
+
if (!report) {
|
|
2201
|
+
throw new Error(
|
|
2202
|
+
`No runtime status report for ${serviceName}/${serviceInstanceId}`
|
|
2203
|
+
);
|
|
2204
|
+
}
|
|
2205
|
+
if (!this.applyRuntimeStatusReport(report)) {
|
|
2206
|
+
throw new Error(
|
|
2207
|
+
`No tracked instance for runtime fallback ${serviceName}/${serviceInstanceId}`
|
|
2208
|
+
);
|
|
2209
|
+
}
|
|
2210
|
+
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
2211
|
+
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
2212
|
+
return {
|
|
2213
|
+
report,
|
|
2214
|
+
inquiryMeta: inquiryResult.__inquiryMeta ?? {}
|
|
2215
|
+
};
|
|
2216
|
+
}
|
|
2217
|
+
evaluateDependencyReadinessDetail(serviceName, serviceInstanceId, now = Date.now()) {
|
|
2218
|
+
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
2219
|
+
const missedHeartbeats = this.getHeartbeatMisses(serviceInstanceId, now);
|
|
2220
|
+
const runtimeState = instance ? instance.runtimeState ?? this.resolveRuntimeStatusSnapshot(
|
|
2221
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
2222
|
+
instance.isActive,
|
|
2223
|
+
instance.isNonResponsive,
|
|
2224
|
+
instance.isBlocked
|
|
2225
|
+
).state : "unavailable";
|
|
2226
|
+
const acceptingWork = instance ? typeof instance.acceptingWork === "boolean" ? instance.acceptingWork : this.resolveRuntimeStatusSnapshot(
|
|
2227
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
2228
|
+
instance.isActive,
|
|
2229
|
+
instance.isNonResponsive,
|
|
2230
|
+
instance.isBlocked
|
|
2231
|
+
).acceptingWork : false;
|
|
2232
|
+
const evaluation = evaluateDependencyReadiness({
|
|
2233
|
+
exists: Boolean(instance),
|
|
2234
|
+
runtimeState,
|
|
2235
|
+
acceptingWork,
|
|
2236
|
+
missedHeartbeats,
|
|
2237
|
+
missThreshold: this.runtimeStatusMissThreshold
|
|
2238
|
+
});
|
|
2239
|
+
const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId);
|
|
2240
|
+
return {
|
|
2241
|
+
serviceName,
|
|
2242
|
+
serviceInstanceId,
|
|
2243
|
+
dependencyState: evaluation.state,
|
|
2244
|
+
runtimeState,
|
|
2245
|
+
acceptingWork,
|
|
2246
|
+
missedHeartbeats,
|
|
2247
|
+
stale: evaluation.stale,
|
|
2248
|
+
blocked: evaluation.blocked,
|
|
2249
|
+
reason: evaluation.reason,
|
|
2250
|
+
lastHeartbeatAt: lastHeartbeat ? new Date(lastHeartbeat).toISOString() : null,
|
|
2251
|
+
reportedAt: instance?.reportedAt ?? null
|
|
2252
|
+
};
|
|
2253
|
+
}
|
|
2254
|
+
async buildLocalReadinessReport(options = {}) {
|
|
2255
|
+
const localRuntime = this.buildLocalRuntimeStatusReport("minimal");
|
|
2256
|
+
if (!localRuntime) {
|
|
2257
|
+
return null;
|
|
2258
|
+
}
|
|
2259
|
+
const detailLevel = options.detailLevel ?? "minimal";
|
|
2260
|
+
const includeDependencies = options.includeDependencies ?? detailLevel === "full";
|
|
2261
|
+
const refreshStaleDependencies = options.refreshStaleDependencies ?? true;
|
|
2262
|
+
const dependencyPairs = Array.from(this.readinessDependeesByService.entries()).flatMap(
|
|
2263
|
+
([serviceName, instanceIds]) => Array.from(instanceIds).map((serviceInstanceId) => ({
|
|
2264
|
+
serviceName,
|
|
2265
|
+
serviceInstanceId
|
|
2266
|
+
}))
|
|
2267
|
+
).sort((left, right) => {
|
|
2268
|
+
if (left.serviceName !== right.serviceName) {
|
|
2269
|
+
return left.serviceName.localeCompare(right.serviceName);
|
|
2270
|
+
}
|
|
2271
|
+
return left.serviceInstanceId.localeCompare(right.serviceInstanceId);
|
|
2272
|
+
});
|
|
2273
|
+
if (refreshStaleDependencies) {
|
|
2274
|
+
for (const dependency of dependencyPairs) {
|
|
2275
|
+
const misses = this.getHeartbeatMisses(dependency.serviceInstanceId);
|
|
2276
|
+
if (misses < this.runtimeStatusMissThreshold) {
|
|
2277
|
+
continue;
|
|
2278
|
+
}
|
|
2279
|
+
if (this.runtimeStatusFallbackInFlightByInstance.has(
|
|
2280
|
+
dependency.serviceInstanceId
|
|
2281
|
+
)) {
|
|
2282
|
+
continue;
|
|
2283
|
+
}
|
|
2284
|
+
this.runtimeStatusFallbackInFlightByInstance.add(
|
|
2285
|
+
dependency.serviceInstanceId
|
|
2286
|
+
);
|
|
2287
|
+
try {
|
|
2288
|
+
await this.resolveRuntimeStatusFallbackInquiry(
|
|
2289
|
+
dependency.serviceName,
|
|
2290
|
+
dependency.serviceInstanceId
|
|
2291
|
+
);
|
|
2292
|
+
} catch (error) {
|
|
2293
|
+
CadenzaService.log(
|
|
2294
|
+
"Readiness dependency fallback failed.",
|
|
2295
|
+
{
|
|
2296
|
+
serviceName: dependency.serviceName,
|
|
2297
|
+
serviceInstanceId: dependency.serviceInstanceId,
|
|
2298
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2299
|
+
},
|
|
2300
|
+
"warning"
|
|
2301
|
+
);
|
|
2302
|
+
} finally {
|
|
2303
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(
|
|
2304
|
+
dependency.serviceInstanceId
|
|
2305
|
+
);
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
const now = Date.now();
|
|
2310
|
+
const dependencyDetails = dependencyPairs.map(
|
|
2311
|
+
(dependency) => this.evaluateDependencyReadinessDetail(
|
|
2312
|
+
dependency.serviceName,
|
|
2313
|
+
dependency.serviceInstanceId,
|
|
2314
|
+
now
|
|
2315
|
+
)
|
|
2316
|
+
);
|
|
2317
|
+
const dependencySummary = summarizeDependencyReadiness(
|
|
2318
|
+
dependencyDetails.map((detail) => ({
|
|
2319
|
+
state: detail.dependencyState,
|
|
2320
|
+
stale: detail.stale,
|
|
2321
|
+
blocked: detail.blocked,
|
|
2322
|
+
reason: detail.reason
|
|
2323
|
+
}))
|
|
2324
|
+
);
|
|
2325
|
+
const readinessState = resolveServiceReadinessState(
|
|
2326
|
+
localRuntime.state,
|
|
2327
|
+
localRuntime.acceptingWork,
|
|
2328
|
+
dependencySummary
|
|
2329
|
+
);
|
|
2330
|
+
return {
|
|
2331
|
+
serviceName: localRuntime.serviceName,
|
|
2332
|
+
serviceInstanceId: localRuntime.serviceInstanceId,
|
|
2333
|
+
reportedAt: new Date(now).toISOString(),
|
|
2334
|
+
readinessState,
|
|
2335
|
+
runtimeState: localRuntime.state,
|
|
2336
|
+
acceptingWork: localRuntime.acceptingWork,
|
|
2337
|
+
dependencySummary,
|
|
2338
|
+
...includeDependencies ? { dependencies: dependencyDetails } : {}
|
|
2339
|
+
};
|
|
2340
|
+
}
|
|
1243
2341
|
reset() {
|
|
1244
2342
|
this.instances.clear();
|
|
1245
2343
|
this.deputies.clear();
|
|
@@ -1247,6 +2345,17 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1247
2345
|
this.remoteIntents.clear();
|
|
1248
2346
|
this.remoteIntentDeputiesByKey.clear();
|
|
1249
2347
|
this.remoteIntentDeputiesByTask.clear();
|
|
2348
|
+
this.dependeesByService.clear();
|
|
2349
|
+
this.dependeeByInstance.clear();
|
|
2350
|
+
this.readinessDependeesByService.clear();
|
|
2351
|
+
this.readinessDependeeByInstance.clear();
|
|
2352
|
+
this.lastHeartbeatAtByInstance.clear();
|
|
2353
|
+
this.missedHeartbeatsByInstance.clear();
|
|
2354
|
+
this.runtimeStatusFallbackInFlightByInstance.clear();
|
|
2355
|
+
this.activeRoutineExecutionIds.clear();
|
|
2356
|
+
this.numberOfRunningGraphs = 0;
|
|
2357
|
+
this.runtimeStatusHeartbeatStarted = false;
|
|
2358
|
+
this.lastRuntimeStatusSnapshot = null;
|
|
1250
2359
|
}
|
|
1251
2360
|
};
|
|
1252
2361
|
|