@cadenza.io/service 2.6.1 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -298,6 +298,8 @@ var isBrowser = typeof window !== "undefined" && typeof window.document !== "und
298
298
  // src/utils/inquiry.ts
299
299
  var META_INTENT_PREFIX = "meta-";
300
300
  var META_RUNTIME_TRANSPORT_DIAGNOSTICS_INTENT = "meta-runtime-transport-diagnostics";
301
+ var META_RUNTIME_STATUS_INTENT = "meta-runtime-status";
302
+ var META_READINESS_INTENT = "meta-readiness";
301
303
  function isPlainObject(value) {
302
304
  return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
303
305
  }
@@ -361,8 +363,200 @@ function summarizeResponderStatuses(statuses) {
361
363
  return { responded, failed, timedOut, pending };
362
364
  }
363
365
 
366
+ // src/utils/readiness.ts
367
+ function evaluateDependencyReadiness(input) {
368
+ const missedHeartbeats = Math.max(
369
+ 0,
370
+ Math.trunc(Number(input.missedHeartbeats) || 0)
371
+ );
372
+ const stale = missedHeartbeats > 0;
373
+ const timeoutReached = missedHeartbeats >= Math.max(1, input.missThreshold);
374
+ if (!input.exists) {
375
+ return {
376
+ state: "unavailable",
377
+ stale: true,
378
+ blocked: true,
379
+ reason: "missing"
380
+ };
381
+ }
382
+ if (timeoutReached) {
383
+ return {
384
+ state: "unavailable",
385
+ stale: true,
386
+ blocked: true,
387
+ reason: "heartbeat-timeout"
388
+ };
389
+ }
390
+ if (input.runtimeState === "unavailable" || !input.acceptingWork) {
391
+ return {
392
+ state: "unavailable",
393
+ stale,
394
+ blocked: true,
395
+ reason: "runtime-unavailable"
396
+ };
397
+ }
398
+ if (stale) {
399
+ return {
400
+ state: "degraded",
401
+ stale: true,
402
+ blocked: false,
403
+ reason: "heartbeat-stale"
404
+ };
405
+ }
406
+ if (input.runtimeState === "overloaded") {
407
+ return {
408
+ state: "overloaded",
409
+ stale: false,
410
+ blocked: false,
411
+ reason: "runtime-overloaded"
412
+ };
413
+ }
414
+ if (input.runtimeState === "degraded") {
415
+ return {
416
+ state: "degraded",
417
+ stale: false,
418
+ blocked: false,
419
+ reason: "runtime-degraded"
420
+ };
421
+ }
422
+ return {
423
+ state: "ready",
424
+ stale: false,
425
+ blocked: false,
426
+ reason: "runtime-healthy"
427
+ };
428
+ }
429
+ function summarizeDependencyReadiness(evaluations) {
430
+ const summary = {
431
+ total: evaluations.length,
432
+ ready: 0,
433
+ degraded: 0,
434
+ overloaded: 0,
435
+ unavailable: 0,
436
+ stale: 0
437
+ };
438
+ for (const evaluation of evaluations) {
439
+ if (evaluation.state === "ready") summary.ready++;
440
+ if (evaluation.state === "degraded") summary.degraded++;
441
+ if (evaluation.state === "overloaded") summary.overloaded++;
442
+ if (evaluation.state === "unavailable") summary.unavailable++;
443
+ if (evaluation.stale) summary.stale++;
444
+ }
445
+ return summary;
446
+ }
447
+ function resolveServiceReadinessState(localRuntimeState, localAcceptingWork, dependencySummary) {
448
+ if (localRuntimeState === "unavailable" || !localAcceptingWork) {
449
+ return "blocked";
450
+ }
451
+ if (dependencySummary.unavailable > 0) {
452
+ return "blocked";
453
+ }
454
+ if (dependencySummary.degraded > 0 || dependencySummary.overloaded > 0 || dependencySummary.stale > 0) {
455
+ return "degraded";
456
+ }
457
+ return "ready";
458
+ }
459
+
460
+ // src/utils/runtimeStatus.ts
461
+ function resolveRuntimeStatus(input) {
462
+ const numberOfRunningGraphs = Math.max(
463
+ 0,
464
+ Math.trunc(Number(input.numberOfRunningGraphs) || 0)
465
+ );
466
+ const isActive = Boolean(input.isActive);
467
+ const isNonResponsive = Boolean(input.isNonResponsive);
468
+ const isBlocked = Boolean(input.isBlocked);
469
+ if (!isActive || isNonResponsive || isBlocked) {
470
+ return {
471
+ state: "unavailable",
472
+ acceptingWork: false,
473
+ numberOfRunningGraphs,
474
+ isActive,
475
+ isNonResponsive,
476
+ isBlocked
477
+ };
478
+ }
479
+ if (numberOfRunningGraphs >= input.overloadedGraphThreshold) {
480
+ return {
481
+ state: "overloaded",
482
+ acceptingWork: true,
483
+ numberOfRunningGraphs,
484
+ isActive,
485
+ isNonResponsive,
486
+ isBlocked
487
+ };
488
+ }
489
+ if (numberOfRunningGraphs >= input.degradedGraphThreshold) {
490
+ return {
491
+ state: "degraded",
492
+ acceptingWork: true,
493
+ numberOfRunningGraphs,
494
+ isActive,
495
+ isNonResponsive,
496
+ isBlocked
497
+ };
498
+ }
499
+ return {
500
+ state: "healthy",
501
+ acceptingWork: true,
502
+ numberOfRunningGraphs,
503
+ isActive,
504
+ isNonResponsive,
505
+ isBlocked
506
+ };
507
+ }
508
+ function runtimeStatusPriority(state) {
509
+ switch (state) {
510
+ case "healthy":
511
+ return 0;
512
+ case "degraded":
513
+ return 1;
514
+ case "overloaded":
515
+ return 2;
516
+ case "unavailable":
517
+ return 3;
518
+ default:
519
+ return 4;
520
+ }
521
+ }
522
+ function hasSignificantRuntimeStatusChange(previous, next) {
523
+ if (!previous) {
524
+ return true;
525
+ }
526
+ return previous.state !== next.state || previous.acceptingWork !== next.acceptingWork || previous.isActive !== next.isActive || previous.isNonResponsive !== next.isNonResponsive || previous.isBlocked !== next.isBlocked;
527
+ }
528
+
364
529
  // src/registry/ServiceRegistry.ts
365
530
  var META_SERVICE_REGISTRY_FULL_SYNC_INTENT = "meta-service-registry-full-sync";
531
+ var META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL = "meta.service_registry.runtime_status.heartbeat_tick";
532
+ var META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL = "meta.service_registry.runtime_status.monitor_tick";
533
+ var INTERNAL_RUNTIME_STATUS_TASK_NAMES = /* @__PURE__ */ new Set([
534
+ "Track local routine start",
535
+ "Track local routine end",
536
+ "Start runtime status sharing intervals",
537
+ "Broadcast runtime status",
538
+ "Monitor dependee heartbeat freshness",
539
+ "Resolve runtime status fallback inquiry",
540
+ "Respond runtime status inquiry",
541
+ "Respond readiness inquiry",
542
+ "Collect distributed readiness",
543
+ "Get status"
544
+ ]);
545
+ function readPositiveIntegerEnv(name, fallback) {
546
+ if (typeof process === "undefined") {
547
+ return fallback;
548
+ }
549
+ const raw = process.env?.[name];
550
+ const parsed = Number(raw);
551
+ if (!Number.isFinite(parsed)) {
552
+ return fallback;
553
+ }
554
+ const normalized = Math.trunc(parsed);
555
+ if (normalized <= 0) {
556
+ return fallback;
557
+ }
558
+ return normalized;
559
+ }
366
560
  var ServiceRegistry = class _ServiceRegistry {
367
561
  /**
368
562
  * Initializes a private constructor for managing service instances, remote signals,
@@ -380,6 +574,36 @@ var ServiceRegistry = class _ServiceRegistry {
380
574
  this.remoteIntents = /* @__PURE__ */ new Map();
381
575
  this.remoteIntentDeputiesByKey = /* @__PURE__ */ new Map();
382
576
  this.remoteIntentDeputiesByTask = /* @__PURE__ */ new Map();
577
+ this.dependeesByService = /* @__PURE__ */ new Map();
578
+ this.dependeeByInstance = /* @__PURE__ */ new Map();
579
+ this.readinessDependeesByService = /* @__PURE__ */ new Map();
580
+ this.readinessDependeeByInstance = /* @__PURE__ */ new Map();
581
+ this.lastHeartbeatAtByInstance = /* @__PURE__ */ new Map();
582
+ this.missedHeartbeatsByInstance = /* @__PURE__ */ new Map();
583
+ this.runtimeStatusFallbackInFlightByInstance = /* @__PURE__ */ new Set();
584
+ this.activeRoutineExecutionIds = /* @__PURE__ */ new Set();
585
+ this.runtimeStatusHeartbeatStarted = false;
586
+ this.lastRuntimeStatusSnapshot = null;
587
+ this.runtimeStatusHeartbeatIntervalMs = readPositiveIntegerEnv(
588
+ "CADENZA_RUNTIME_STATUS_HEARTBEAT_MS",
589
+ 3e4
590
+ );
591
+ this.runtimeStatusMissThreshold = readPositiveIntegerEnv(
592
+ "CADENZA_RUNTIME_STATUS_MISSED_HEARTBEATS",
593
+ 3
594
+ );
595
+ this.runtimeStatusFallbackTimeoutMs = readPositiveIntegerEnv(
596
+ "CADENZA_RUNTIME_STATUS_FALLBACK_TIMEOUT_MS",
597
+ 1500
598
+ );
599
+ this.degradedGraphThreshold = readPositiveIntegerEnv(
600
+ "CADENZA_RUNTIME_STATUS_DEGRADED_GRAPH_THRESHOLD",
601
+ 10
602
+ );
603
+ this.overloadedGraphThreshold = readPositiveIntegerEnv(
604
+ "CADENZA_RUNTIME_STATUS_OVERLOADED_GRAPH_THRESHOLD",
605
+ 20
606
+ );
383
607
  this.serviceName = null;
384
608
  this.serviceInstanceId = null;
385
609
  this.numberOfRunningGraphs = 0;
@@ -418,10 +642,137 @@ var ServiceRegistry = class _ServiceRegistry {
418
642
  }
419
643
  }
420
644
  });
645
+ CadenzaService.defineIntent({
646
+ name: META_RUNTIME_STATUS_INTENT,
647
+ description: "Gather lightweight runtime status reports from services in the distributed runtime.",
648
+ input: {
649
+ type: "object",
650
+ properties: {
651
+ detailLevel: {
652
+ type: "string",
653
+ constraints: {
654
+ oneOf: ["minimal", "full"]
655
+ }
656
+ },
657
+ targetServiceName: {
658
+ type: "string"
659
+ },
660
+ targetServiceInstanceId: {
661
+ type: "string"
662
+ }
663
+ }
664
+ },
665
+ output: {
666
+ type: "object",
667
+ properties: {
668
+ runtimeStatusReports: {
669
+ type: "array"
670
+ }
671
+ }
672
+ }
673
+ });
674
+ CadenzaService.createMetaTask(
675
+ "Respond runtime status inquiry",
676
+ (ctx) => {
677
+ const targetServiceName = ctx.targetServiceName;
678
+ const targetServiceInstanceId = ctx.targetServiceInstanceId;
679
+ const detailLevel = ctx.detailLevel === "full" ? "full" : "minimal";
680
+ const report = this.buildLocalRuntimeStatusReport(detailLevel);
681
+ if (!report) {
682
+ return {};
683
+ }
684
+ if (targetServiceName && targetServiceName !== report.serviceName) {
685
+ return {};
686
+ }
687
+ if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
688
+ return {};
689
+ }
690
+ return {
691
+ runtimeStatusReports: [report]
692
+ };
693
+ },
694
+ "Responds to runtime-status inquiries with local service instance status."
695
+ ).respondsTo(META_RUNTIME_STATUS_INTENT);
696
+ CadenzaService.defineIntent({
697
+ name: META_READINESS_INTENT,
698
+ description: "Gather service readiness reports derived from local runtime status and required dependees.",
699
+ input: {
700
+ type: "object",
701
+ properties: {
702
+ detailLevel: {
703
+ type: "string",
704
+ constraints: {
705
+ oneOf: ["minimal", "full"]
706
+ }
707
+ },
708
+ includeDependencies: {
709
+ type: "boolean"
710
+ },
711
+ refreshStaleDependencies: {
712
+ type: "boolean"
713
+ },
714
+ targetServiceName: {
715
+ type: "string"
716
+ },
717
+ targetServiceInstanceId: {
718
+ type: "string"
719
+ }
720
+ }
721
+ },
722
+ output: {
723
+ type: "object",
724
+ properties: {
725
+ readinessReports: {
726
+ type: "array"
727
+ }
728
+ }
729
+ }
730
+ });
731
+ CadenzaService.createMetaTask(
732
+ "Respond readiness inquiry",
733
+ async (ctx) => {
734
+ const targetServiceName = ctx.targetServiceName;
735
+ const targetServiceInstanceId = ctx.targetServiceInstanceId;
736
+ const report = await this.buildLocalReadinessReport({
737
+ detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
738
+ includeDependencies: ctx.includeDependencies,
739
+ refreshStaleDependencies: ctx.refreshStaleDependencies
740
+ });
741
+ if (!report) {
742
+ return {};
743
+ }
744
+ if (targetServiceName && targetServiceName !== report.serviceName) {
745
+ return {};
746
+ }
747
+ if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
748
+ return {};
749
+ }
750
+ return {
751
+ readinessReports: [report]
752
+ };
753
+ },
754
+ "Responds to distributed readiness inquiries using required dependee health."
755
+ ).respondsTo(META_READINESS_INTENT);
421
756
  this.handleInstanceUpdateTask = CadenzaService.createMetaTask(
422
757
  "Handle Instance Update",
423
758
  (ctx, emit) => {
424
- const { serviceInstance } = ctx;
759
+ const serviceInstance = ctx.serviceInstance ?? (ctx.__serviceInstanceId || ctx.serviceInstanceId ? {
760
+ uuid: ctx.__serviceInstanceId ?? ctx.serviceInstanceId,
761
+ serviceName: ctx.__serviceName ?? ctx.serviceName,
762
+ address: ctx.serviceAddress ?? "",
763
+ port: ctx.servicePort ?? 0,
764
+ exposed: !!ctx.exposed,
765
+ isFrontend: !!ctx.isFrontend,
766
+ isActive: typeof ctx.isActive === "boolean" ? ctx.isActive : typeof ctx.__active === "boolean" ? ctx.__active : true,
767
+ isNonResponsive: !!ctx.isNonResponsive,
768
+ isBlocked: !!ctx.isBlocked,
769
+ health: ctx.health ?? ctx.__health ?? {},
770
+ numberOfRunningGraphs: ctx.numberOfRunningGraphs ?? ctx.__numberOfRunningGraphs ?? 0,
771
+ isPrimary: false
772
+ } : void 0);
773
+ if (!serviceInstance?.uuid || !serviceInstance?.serviceName) {
774
+ return false;
775
+ }
425
776
  const {
426
777
  uuid: uuid4,
427
778
  serviceName,
@@ -443,6 +794,7 @@ var ServiceRegistry = class _ServiceRegistry {
443
794
  emit(`meta.socket_shutdown_requested:${address}_${port}`, {});
444
795
  emit(`meta.fetch.destroy_requested:${address}_${port}`, {});
445
796
  }
797
+ this.unregisterDependee(uuid4, serviceName);
446
798
  return;
447
799
  }
448
800
  if (!this.instances.has(serviceName))
@@ -454,6 +806,18 @@ var ServiceRegistry = class _ServiceRegistry {
454
806
  } else {
455
807
  instances.push(serviceInstance);
456
808
  }
809
+ const trackedInstance = existing ?? instances.find((instance) => instance.uuid === uuid4);
810
+ if (trackedInstance) {
811
+ const snapshot = this.resolveRuntimeStatusSnapshot(
812
+ trackedInstance.numberOfRunningGraphs ?? 0,
813
+ trackedInstance.isActive,
814
+ trackedInstance.isNonResponsive,
815
+ trackedInstance.isBlocked
816
+ );
817
+ trackedInstance.runtimeState = snapshot.state;
818
+ trackedInstance.acceptingWork = snapshot.acceptingWork;
819
+ trackedInstance.reportedAt = trackedInstance.reportedAt ?? (/* @__PURE__ */ new Date()).toISOString();
820
+ }
457
821
  if (this.serviceName === serviceName) {
458
822
  return false;
459
823
  }
@@ -493,13 +857,27 @@ var ServiceRegistry = class _ServiceRegistry {
493
857
  "global.meta.service_instance.inserted",
494
858
  "global.meta.service_instance.updated",
495
859
  "meta.service_instance.inserted",
496
- "meta.service_instance.updated",
497
- "meta.socket_client.status_received"
860
+ "meta.service_instance.updated"
498
861
  ).attachSignal(
499
862
  "meta.service_registry.dependee_registered",
500
863
  "meta.socket_shutdown_requested",
501
864
  "meta.fetch.destroy_requested"
502
865
  );
866
+ CadenzaService.createMetaTask(
867
+ "Track dependee registration",
868
+ (ctx) => {
869
+ if (!ctx.serviceName || !ctx.serviceInstanceId) {
870
+ return false;
871
+ }
872
+ this.registerDependee(ctx.serviceName, ctx.serviceInstanceId, {
873
+ requiredForReadiness: this.shouldRequireReadinessFromCommunicationTypes(
874
+ ctx.communicationTypes
875
+ )
876
+ });
877
+ return true;
878
+ },
879
+ "Tracks remote dependency instances for runtime heartbeat monitoring."
880
+ ).doOn("meta.service_registry.dependee_registered");
503
881
  CadenzaService.createMetaTask("Split service instances", function* (ctx) {
504
882
  if (!ctx.serviceInstances) {
505
883
  return;
@@ -599,6 +977,15 @@ var ServiceRegistry = class _ServiceRegistry {
599
977
  for (const instance of instances ?? []) {
600
978
  instance.isActive = false;
601
979
  instance.isNonResponsive = true;
980
+ const snapshot = this.resolveRuntimeStatusSnapshot(
981
+ instance.numberOfRunningGraphs ?? 0,
982
+ instance.isActive,
983
+ instance.isNonResponsive,
984
+ instance.isBlocked
985
+ );
986
+ instance.runtimeState = snapshot.state;
987
+ instance.acceptingWork = snapshot.acceptingWork;
988
+ instance.reportedAt = (/* @__PURE__ */ new Date()).toISOString();
602
989
  emit("global.meta.service_registry.service_not_responding", {
603
990
  data: {
604
991
  isActive: false,
@@ -616,7 +1003,8 @@ var ServiceRegistry = class _ServiceRegistry {
616
1003
  "meta.fetch.handshake_failed",
617
1004
  "meta.fetch.handshake_failed.*",
618
1005
  "meta.socket_client.disconnected",
619
- "meta.socket_client.disconnected.*"
1006
+ "meta.socket_client.disconnected.*",
1007
+ "meta.service_registry.runtime_status_unreachable"
620
1008
  ).attachSignal("global.meta.service_registry.service_not_responding");
621
1009
  this.handleServiceHandshakeTask = CadenzaService.createMetaTask(
622
1010
  "Handle service handshake",
@@ -631,6 +1019,15 @@ var ServiceRegistry = class _ServiceRegistry {
631
1019
  }
632
1020
  instance.isActive = true;
633
1021
  instance.isNonResponsive = false;
1022
+ const snapshot = this.resolveRuntimeStatusSnapshot(
1023
+ instance.numberOfRunningGraphs ?? 0,
1024
+ instance.isActive,
1025
+ instance.isNonResponsive,
1026
+ instance.isBlocked
1027
+ );
1028
+ instance.runtimeState = snapshot.state;
1029
+ instance.acceptingWork = snapshot.acceptingWork;
1030
+ instance.reportedAt = (/* @__PURE__ */ new Date()).toISOString();
634
1031
  emit("global.meta.service_registry.service_handshake", {
635
1032
  data: {
636
1033
  isActive: instance.isActive,
@@ -648,6 +1045,7 @@ var ServiceRegistry = class _ServiceRegistry {
648
1045
  if (indexToDelete >= 0) {
649
1046
  this.instances.get(serviceName)?.splice(indexToDelete, 1);
650
1047
  }
1048
+ this.unregisterDependee(i.uuid, serviceName);
651
1049
  emit("global.meta.service_registry.deleted", {
652
1050
  data: {
653
1051
  isActive: false,
@@ -669,14 +1067,46 @@ var ServiceRegistry = class _ServiceRegistry {
669
1067
  this.handleSocketStatusUpdateTask = CadenzaService.createMetaTask(
670
1068
  "Handle Socket Status Update",
671
1069
  (ctx) => {
672
- const instanceId = ctx.__serviceInstanceId;
673
- const serviceName = ctx.__serviceName;
674
- const instances = this.instances.get(serviceName);
675
- const instance = instances?.find((i) => i.uuid === instanceId);
676
- if (instance) {
677
- instance.health = ctx.health;
678
- instance.numberOfRunningGraphs = ctx.numberOfRunningGraphs;
1070
+ const report = this.normalizeRuntimeStatusReport(ctx);
1071
+ if (!report) {
1072
+ return false;
1073
+ }
1074
+ if (report.serviceName === this.serviceName && report.serviceInstanceId === this.serviceInstanceId) {
1075
+ return false;
1076
+ }
1077
+ let applied = this.applyRuntimeStatusReport(report);
1078
+ if (!applied && report.serviceAddress && typeof report.servicePort === "number") {
1079
+ if (!this.instances.has(report.serviceName)) {
1080
+ this.instances.set(report.serviceName, []);
1081
+ }
1082
+ this.instances.get(report.serviceName).push({
1083
+ uuid: report.serviceInstanceId,
1084
+ serviceName: report.serviceName,
1085
+ address: report.serviceAddress,
1086
+ port: report.servicePort,
1087
+ exposed: !!report.exposed,
1088
+ isFrontend: !!report.isFrontend,
1089
+ isActive: report.isActive,
1090
+ isNonResponsive: report.isNonResponsive,
1091
+ isBlocked: report.isBlocked,
1092
+ numberOfRunningGraphs: report.numberOfRunningGraphs,
1093
+ runtimeState: report.state,
1094
+ acceptingWork: report.acceptingWork,
1095
+ reportedAt: report.reportedAt,
1096
+ health: report.health ?? {},
1097
+ isPrimary: false
1098
+ });
1099
+ applied = true;
679
1100
  }
1101
+ if (!applied) {
1102
+ return false;
1103
+ }
1104
+ this.registerDependee(report.serviceName, report.serviceInstanceId);
1105
+ this.lastHeartbeatAtByInstance.set(report.serviceInstanceId, Date.now());
1106
+ this.missedHeartbeatsByInstance.set(report.serviceInstanceId, 0);
1107
+ this.runtimeStatusFallbackInFlightByInstance.delete(
1108
+ report.serviceInstanceId
1109
+ );
680
1110
  return true;
681
1111
  },
682
1112
  "Handles status update from socket broadcast"
@@ -806,7 +1236,25 @@ var ServiceRegistry = class _ServiceRegistry {
806
1236
  const { __serviceName, __triedInstances, __retries, __broadcast } = context;
807
1237
  let retries = __retries ?? 0;
808
1238
  let triedInstances = __triedInstances ?? [];
809
- const instances = this.instances.get(__serviceName)?.filter((i) => i.isActive && !i.isNonResponsive && !i.isBlocked).sort((a, b) => a.numberOfRunningGraphs - b.numberOfRunningGraphs);
1239
+ const instances = this.instances.get(__serviceName)?.filter((i) => i.isActive && !i.isNonResponsive && !i.isBlocked).sort((a, b) => {
1240
+ const leftStatus = this.resolveRuntimeStatusSnapshot(
1241
+ a.numberOfRunningGraphs ?? 0,
1242
+ a.isActive,
1243
+ a.isNonResponsive,
1244
+ a.isBlocked
1245
+ );
1246
+ const rightStatus = this.resolveRuntimeStatusSnapshot(
1247
+ b.numberOfRunningGraphs ?? 0,
1248
+ b.isActive,
1249
+ b.isNonResponsive,
1250
+ b.isBlocked
1251
+ );
1252
+ const priorityDelta = runtimeStatusPriority(leftStatus.state) - runtimeStatusPriority(rightStatus.state);
1253
+ if (priorityDelta !== 0) {
1254
+ return priorityDelta;
1255
+ }
1256
+ return (a.numberOfRunningGraphs ?? 0) - (b.numberOfRunningGraphs ?? 0);
1257
+ });
810
1258
  if (!instances || instances.length === 0 || retries > this.retryCount) {
811
1259
  context.errored = true;
812
1260
  context.__error = `No active instances for ${__serviceName}. Retries: ${retries}. ${this.instances.get(
@@ -892,15 +1340,292 @@ var ServiceRegistry = class _ServiceRegistry {
892
1340
  errored: true
893
1341
  };
894
1342
  }
895
- const self = this.instances.get(this.serviceName)?.find((i) => i.uuid === this.serviceInstanceId);
1343
+ const report = this.buildLocalRuntimeStatusReport("full");
1344
+ if (!report) {
1345
+ return {
1346
+ ...ctx,
1347
+ __status: "error",
1348
+ __error: "No local service instance available for status check",
1349
+ errored: true
1350
+ };
1351
+ }
896
1352
  return {
897
1353
  ...ctx,
898
1354
  __status: "ok",
899
- __numberOfRunningGraphs: self?.numberOfRunningGraphs ?? 0,
900
- __health: self?.health ?? {},
901
- __active: self?.isActive ?? false
1355
+ __serviceName: report.serviceName,
1356
+ __serviceInstanceId: report.serviceInstanceId,
1357
+ __numberOfRunningGraphs: report.numberOfRunningGraphs,
1358
+ __health: report.health ?? {},
1359
+ __active: report.isActive,
1360
+ reportedAt: report.reportedAt,
1361
+ serviceName: report.serviceName,
1362
+ serviceInstanceId: report.serviceInstanceId,
1363
+ numberOfRunningGraphs: report.numberOfRunningGraphs,
1364
+ health: report.health ?? {},
1365
+ isActive: report.isActive,
1366
+ isNonResponsive: report.isNonResponsive,
1367
+ isBlocked: report.isBlocked,
1368
+ state: report.state,
1369
+ acceptingWork: report.acceptingWork
902
1370
  };
903
- }).doOn("meta.socket.status_check_requested");
1371
+ }).doOn(
1372
+ "meta.socket.status_check_requested",
1373
+ "meta.rest.status_check_requested"
1374
+ );
1375
+ CadenzaService.createMetaTask(
1376
+ "Track local routine start",
1377
+ (ctx, emit) => {
1378
+ const sourceTaskName = String(ctx.__signalEmission?.taskName ?? "");
1379
+ if (INTERNAL_RUNTIME_STATUS_TASK_NAMES.has(sourceTaskName)) {
1380
+ return false;
1381
+ }
1382
+ const routineId = String(
1383
+ ctx.filter?.uuid ?? ctx.__routineExecId ?? ""
1384
+ );
1385
+ if (!routineId) {
1386
+ return false;
1387
+ }
1388
+ this.activeRoutineExecutionIds.add(routineId);
1389
+ this.numberOfRunningGraphs = this.activeRoutineExecutionIds.size;
1390
+ const localInstance = this.getLocalInstance();
1391
+ if (!localInstance) {
1392
+ return true;
1393
+ }
1394
+ const snapshot = this.resolveRuntimeStatusSnapshot(
1395
+ this.numberOfRunningGraphs,
1396
+ localInstance.isActive,
1397
+ localInstance.isNonResponsive,
1398
+ localInstance.isBlocked
1399
+ );
1400
+ if (hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
1401
+ emit("meta.service_registry.runtime_status_broadcast_requested", {
1402
+ reason: "runtime-state-change"
1403
+ });
1404
+ }
1405
+ return true;
1406
+ },
1407
+ "Tracks local routine starts for runtime load status."
1408
+ ).doOn("meta.node.started_routine_execution");
1409
+ CadenzaService.createMetaTask(
1410
+ "Track local routine end",
1411
+ (ctx, emit) => {
1412
+ const sourceTaskName = String(ctx.__signalEmission?.taskName ?? "");
1413
+ if (INTERNAL_RUNTIME_STATUS_TASK_NAMES.has(sourceTaskName)) {
1414
+ return false;
1415
+ }
1416
+ const routineId = String(
1417
+ ctx.filter?.uuid ?? ctx.__routineExecId ?? ""
1418
+ );
1419
+ if (!routineId) {
1420
+ return false;
1421
+ }
1422
+ this.activeRoutineExecutionIds.delete(routineId);
1423
+ this.numberOfRunningGraphs = this.activeRoutineExecutionIds.size;
1424
+ const localInstance = this.getLocalInstance();
1425
+ if (!localInstance) {
1426
+ return true;
1427
+ }
1428
+ const snapshot = this.resolveRuntimeStatusSnapshot(
1429
+ this.numberOfRunningGraphs,
1430
+ localInstance.isActive,
1431
+ localInstance.isNonResponsive,
1432
+ localInstance.isBlocked
1433
+ );
1434
+ if (hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
1435
+ emit("meta.service_registry.runtime_status_broadcast_requested", {
1436
+ reason: "runtime-state-change"
1437
+ });
1438
+ }
1439
+ return true;
1440
+ },
1441
+ "Tracks local routine completion for runtime load status."
1442
+ ).doOn("meta.node.ended_routine_execution");
1443
+ CadenzaService.createMetaTask(
1444
+ "Start runtime status sharing intervals",
1445
+ () => {
1446
+ if (this.runtimeStatusHeartbeatStarted) {
1447
+ return false;
1448
+ }
1449
+ this.runtimeStatusHeartbeatStarted = true;
1450
+ CadenzaService.interval(
1451
+ META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL,
1452
+ { reason: "heartbeat" },
1453
+ this.runtimeStatusHeartbeatIntervalMs,
1454
+ true
1455
+ );
1456
+ CadenzaService.interval(
1457
+ META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL,
1458
+ {},
1459
+ this.runtimeStatusHeartbeatIntervalMs
1460
+ );
1461
+ return true;
1462
+ },
1463
+ "Starts runtime status heartbeat and heartbeat-monitor loops once per service instance."
1464
+ ).doOn("meta.service_registry.instance_inserted");
1465
+ CadenzaService.createMetaTask(
1466
+ "Broadcast runtime status",
1467
+ (ctx, emit) => {
1468
+ const report = this.buildLocalRuntimeStatusReport(
1469
+ ctx.detailLevel === "full" ? "full" : "minimal"
1470
+ );
1471
+ if (!report) {
1472
+ return false;
1473
+ }
1474
+ const snapshot = this.resolveRuntimeStatusSnapshot(
1475
+ report.numberOfRunningGraphs,
1476
+ report.isActive,
1477
+ report.isNonResponsive,
1478
+ report.isBlocked
1479
+ );
1480
+ const force = ctx.reason === "heartbeat" || ctx.force === true || this.lastRuntimeStatusSnapshot === null;
1481
+ if (!force && !hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
1482
+ return false;
1483
+ }
1484
+ this.lastRuntimeStatusSnapshot = snapshot;
1485
+ emit("meta.service.updated", {
1486
+ __serviceName: report.serviceName,
1487
+ __serviceInstanceId: report.serviceInstanceId,
1488
+ __reportedAt: report.reportedAt,
1489
+ __numberOfRunningGraphs: report.numberOfRunningGraphs,
1490
+ __health: report.health ?? {},
1491
+ __active: report.isActive,
1492
+ serviceName: report.serviceName,
1493
+ serviceInstanceId: report.serviceInstanceId,
1494
+ serviceAddress: report.serviceAddress,
1495
+ servicePort: report.servicePort,
1496
+ exposed: report.exposed,
1497
+ isFrontend: report.isFrontend,
1498
+ reportedAt: report.reportedAt,
1499
+ numberOfRunningGraphs: report.numberOfRunningGraphs,
1500
+ health: report.health ?? {},
1501
+ isActive: report.isActive,
1502
+ isNonResponsive: report.isNonResponsive,
1503
+ isBlocked: report.isBlocked,
1504
+ state: report.state,
1505
+ acceptingWork: report.acceptingWork
1506
+ });
1507
+ return true;
1508
+ },
1509
+ "Broadcasts local runtime status to connected dependees."
1510
+ ).doOn(
1511
+ META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL,
1512
+ "meta.service_registry.runtime_status_broadcast_requested"
1513
+ );
1514
+ CadenzaService.createMetaTask(
1515
+ "Monitor dependee heartbeat freshness",
1516
+ (ctx, emit) => {
1517
+ if (!this.useSocket) {
1518
+ return false;
1519
+ }
1520
+ const now = Date.now();
1521
+ for (const [serviceName, instanceIds] of this.dependeesByService) {
1522
+ for (const serviceInstanceId of instanceIds) {
1523
+ const instance = this.getInstance(serviceName, serviceInstanceId);
1524
+ if (!instance || !instance.isActive || instance.isBlocked) {
1525
+ continue;
1526
+ }
1527
+ const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
1528
+ const misses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
1529
+ const heartbeatBudget = this.runtimeStatusHeartbeatIntervalMs * (misses + 1);
1530
+ if (lastHeartbeat > 0 && now - lastHeartbeat < heartbeatBudget) {
1531
+ continue;
1532
+ }
1533
+ const nextMisses = misses + 1;
1534
+ this.missedHeartbeatsByInstance.set(serviceInstanceId, nextMisses);
1535
+ if (nextMisses < this.runtimeStatusMissThreshold || this.runtimeStatusFallbackInFlightByInstance.has(serviceInstanceId)) {
1536
+ continue;
1537
+ }
1538
+ this.runtimeStatusFallbackInFlightByInstance.add(serviceInstanceId);
1539
+ emit("meta.service_registry.runtime_status_fallback_requested", {
1540
+ ...ctx,
1541
+ serviceName,
1542
+ serviceInstanceId,
1543
+ serviceAddress: instance.address,
1544
+ servicePort: instance.port
1545
+ });
1546
+ }
1547
+ }
1548
+ return true;
1549
+ },
1550
+ "Monitors dependee heartbeat freshness and requests inquiry fallback after repeated misses."
1551
+ ).doOn(META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL);
1552
+ CadenzaService.createMetaTask(
1553
+ "Resolve runtime status fallback inquiry",
1554
+ async (ctx, emit) => {
1555
+ const serviceName = ctx.serviceName;
1556
+ const serviceInstanceId = ctx.serviceInstanceId;
1557
+ if (!serviceName || !serviceInstanceId) {
1558
+ return false;
1559
+ }
1560
+ try {
1561
+ const { report, inquiryMeta } = await this.resolveRuntimeStatusFallbackInquiry(
1562
+ serviceName,
1563
+ serviceInstanceId,
1564
+ {
1565
+ detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
1566
+ overallTimeoutMs: ctx.overallTimeoutMs,
1567
+ perResponderTimeoutMs: ctx.perResponderTimeoutMs,
1568
+ requireComplete: ctx.requireComplete
1569
+ }
1570
+ );
1571
+ return {
1572
+ ...ctx,
1573
+ runtimeStatusReport: report,
1574
+ __inquiryMeta: inquiryMeta
1575
+ };
1576
+ } catch (error) {
1577
+ const instance = this.getInstance(serviceName, serviceInstanceId);
1578
+ const message = error instanceof Error ? error.message : String(error);
1579
+ CadenzaService.log(
1580
+ "Runtime status fallback inquiry failed.",
1581
+ {
1582
+ serviceName,
1583
+ serviceInstanceId,
1584
+ error: message
1585
+ },
1586
+ "warning"
1587
+ );
1588
+ emit("meta.service_registry.runtime_status_unreachable", {
1589
+ ...ctx,
1590
+ serviceName,
1591
+ serviceInstanceId,
1592
+ serviceAddress: instance?.address ?? ctx.serviceAddress,
1593
+ servicePort: instance?.port ?? ctx.servicePort,
1594
+ __error: message,
1595
+ errored: true
1596
+ });
1597
+ return {
1598
+ ...ctx,
1599
+ __error: message,
1600
+ errored: true
1601
+ };
1602
+ } finally {
1603
+ this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
1604
+ }
1605
+ },
1606
+ "Runs runtime-status inquiry fallback for a dependee instance after missed heartbeats."
1607
+ ).doOn("meta.service_registry.runtime_status_fallback_requested").emits("meta.service_registry.runtime_status_fallback_resolved").emitsOnFail("meta.service_registry.runtime_status_fallback_failed");
1608
+ this.collectReadinessTask = CadenzaService.createMetaTask(
1609
+ "Collect distributed readiness",
1610
+ async (ctx) => {
1611
+ const inquiryResult = await CadenzaService.inquire(
1612
+ META_READINESS_INTENT,
1613
+ {
1614
+ detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
1615
+ includeDependencies: ctx.includeDependencies,
1616
+ refreshStaleDependencies: ctx.refreshStaleDependencies,
1617
+ targetServiceName: ctx.targetServiceName,
1618
+ targetServiceInstanceId: ctx.targetServiceInstanceId
1619
+ },
1620
+ ctx.inquiryOptions ?? ctx.__inquiryOptions ?? {}
1621
+ );
1622
+ return {
1623
+ ...ctx,
1624
+ ...inquiryResult
1625
+ };
1626
+ },
1627
+ "Collects distributed readiness reports from services."
1628
+ ).doOn("meta.service_registry.readiness_requested").emits("meta.service_registry.readiness_collected").emitsOnFail("meta.service_registry.readiness_failed");
904
1629
  this.collectTransportDiagnosticsTask = CadenzaService.createMetaTask(
905
1630
  "Collect transport diagnostics",
906
1631
  async (ctx) => {
@@ -1240,6 +1965,379 @@ var ServiceRegistry = class _ServiceRegistry {
1240
1965
  localTaskName: task.name
1241
1966
  };
1242
1967
  }
1968
+ getInstance(serviceName, instanceId) {
1969
+ return this.instances.get(serviceName)?.find((instance) => instance.uuid === instanceId);
1970
+ }
1971
+ getLocalInstance() {
1972
+ if (!this.serviceName || !this.serviceInstanceId) {
1973
+ return void 0;
1974
+ }
1975
+ return this.getInstance(this.serviceName, this.serviceInstanceId);
1976
+ }
1977
+ registerDependee(serviceName, serviceInstanceId, options = {}) {
1978
+ if (!serviceName || !serviceInstanceId) {
1979
+ return;
1980
+ }
1981
+ if (!this.dependeesByService.has(serviceName)) {
1982
+ this.dependeesByService.set(serviceName, /* @__PURE__ */ new Set());
1983
+ }
1984
+ this.dependeesByService.get(serviceName).add(serviceInstanceId);
1985
+ this.dependeeByInstance.set(serviceInstanceId, serviceName);
1986
+ if (options.requiredForReadiness) {
1987
+ if (!this.readinessDependeesByService.has(serviceName)) {
1988
+ this.readinessDependeesByService.set(serviceName, /* @__PURE__ */ new Set());
1989
+ }
1990
+ this.readinessDependeesByService.get(serviceName).add(serviceInstanceId);
1991
+ this.readinessDependeeByInstance.set(serviceInstanceId, serviceName);
1992
+ }
1993
+ this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
1994
+ this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
1995
+ }
1996
+ unregisterDependee(serviceInstanceId, serviceName) {
1997
+ const dependeeServiceName = serviceName ?? this.dependeeByInstance.get(serviceInstanceId);
1998
+ if (dependeeServiceName) {
1999
+ this.dependeesByService.get(dependeeServiceName)?.delete(serviceInstanceId);
2000
+ if (!this.dependeesByService.get(dependeeServiceName)?.size) {
2001
+ this.dependeesByService.delete(dependeeServiceName);
2002
+ }
2003
+ }
2004
+ this.dependeeByInstance.delete(serviceInstanceId);
2005
+ const readinessDependeeServiceName = serviceName ?? this.readinessDependeeByInstance.get(serviceInstanceId);
2006
+ if (readinessDependeeServiceName) {
2007
+ this.readinessDependeesByService.get(readinessDependeeServiceName)?.delete(serviceInstanceId);
2008
+ if (!this.readinessDependeesByService.get(readinessDependeeServiceName)?.size) {
2009
+ this.readinessDependeesByService.delete(readinessDependeeServiceName);
2010
+ }
2011
+ }
2012
+ this.readinessDependeeByInstance.delete(serviceInstanceId);
2013
+ this.lastHeartbeatAtByInstance.delete(serviceInstanceId);
2014
+ this.missedHeartbeatsByInstance.delete(serviceInstanceId);
2015
+ this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
2016
+ }
2017
+ getHeartbeatMisses(serviceInstanceId, now = Date.now()) {
2018
+ const observedMisses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
2019
+ const lastHeartbeatAt = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
2020
+ if (lastHeartbeatAt <= 0) {
2021
+ return Math.max(observedMisses, this.runtimeStatusMissThreshold);
2022
+ }
2023
+ const estimatedMisses = Math.max(
2024
+ 0,
2025
+ Math.floor((now - lastHeartbeatAt) / this.runtimeStatusHeartbeatIntervalMs)
2026
+ );
2027
+ return Math.max(observedMisses, estimatedMisses);
2028
+ }
2029
+ shouldRequireReadinessFromCommunicationTypes(communicationTypes) {
2030
+ if (!Array.isArray(communicationTypes)) {
2031
+ return false;
2032
+ }
2033
+ return communicationTypes.some((type) => {
2034
+ const normalized = String(type).toLowerCase();
2035
+ return normalized === "delegation" || normalized === "inquiry";
2036
+ });
2037
+ }
2038
+ resolveRuntimeStatusSnapshot(numberOfRunningGraphs, isActive, isNonResponsive, isBlocked) {
2039
+ return resolveRuntimeStatus({
2040
+ numberOfRunningGraphs,
2041
+ isActive,
2042
+ isNonResponsive,
2043
+ isBlocked,
2044
+ degradedGraphThreshold: this.degradedGraphThreshold,
2045
+ overloadedGraphThreshold: this.overloadedGraphThreshold
2046
+ });
2047
+ }
2048
+ normalizeRuntimeStatusReport(ctx) {
2049
+ const serviceName = ctx.serviceName ?? ctx.__serviceName ?? ctx.serviceInstance?.serviceName;
2050
+ const serviceInstanceId = ctx.serviceInstanceId ?? ctx.__serviceInstanceId ?? ctx.serviceInstance?.uuid;
2051
+ if (!serviceName || !serviceInstanceId) {
2052
+ return null;
2053
+ }
2054
+ const servicePort = ctx.servicePort ?? ctx.port ?? ctx.serviceInstance?.port;
2055
+ const numberOfRunningGraphs = Math.max(
2056
+ 0,
2057
+ Math.trunc(
2058
+ Number(ctx.numberOfRunningGraphs ?? ctx.__numberOfRunningGraphs ?? 0)
2059
+ )
2060
+ );
2061
+ const isActive = Boolean(ctx.isActive ?? ctx.__active ?? true);
2062
+ const isNonResponsive = Boolean(ctx.isNonResponsive ?? false);
2063
+ const isBlocked = Boolean(ctx.isBlocked ?? false);
2064
+ const resolved = this.resolveRuntimeStatusSnapshot(
2065
+ numberOfRunningGraphs,
2066
+ isActive,
2067
+ isNonResponsive,
2068
+ isBlocked
2069
+ );
2070
+ return {
2071
+ serviceName,
2072
+ serviceInstanceId,
2073
+ serviceAddress: ctx.serviceAddress ?? ctx.address ?? ctx.serviceInstance?.address,
2074
+ servicePort: typeof servicePort === "number" ? servicePort : void 0,
2075
+ exposed: typeof ctx.exposed === "boolean" ? ctx.exposed : typeof ctx.serviceInstance?.exposed === "boolean" ? ctx.serviceInstance.exposed : void 0,
2076
+ isFrontend: typeof ctx.isFrontend === "boolean" ? ctx.isFrontend : typeof ctx.serviceInstance?.isFrontend === "boolean" ? ctx.serviceInstance.isFrontend : void 0,
2077
+ reportedAt: ctx.reportedAt ?? (typeof ctx.__reportedAt === "string" ? ctx.__reportedAt : void 0) ?? (/* @__PURE__ */ new Date()).toISOString(),
2078
+ state: ctx.state === "healthy" || ctx.state === "degraded" || ctx.state === "overloaded" || ctx.state === "unavailable" ? ctx.state : resolved.state,
2079
+ acceptingWork: typeof ctx.acceptingWork === "boolean" ? ctx.acceptingWork : resolved.acceptingWork,
2080
+ numberOfRunningGraphs,
2081
+ isActive,
2082
+ isNonResponsive,
2083
+ isBlocked,
2084
+ health: ctx.health ?? ctx.__health ?? {}
2085
+ };
2086
+ }
2087
+ applyRuntimeStatusReport(report) {
2088
+ const instance = this.getInstance(report.serviceName, report.serviceInstanceId);
2089
+ if (!instance) {
2090
+ return false;
2091
+ }
2092
+ if (report.serviceAddress) {
2093
+ instance.address = report.serviceAddress;
2094
+ }
2095
+ if (typeof report.servicePort === "number") {
2096
+ instance.port = report.servicePort;
2097
+ }
2098
+ if (typeof report.exposed === "boolean") {
2099
+ instance.exposed = report.exposed;
2100
+ }
2101
+ if (typeof report.isFrontend === "boolean") {
2102
+ instance.isFrontend = report.isFrontend;
2103
+ }
2104
+ instance.numberOfRunningGraphs = report.numberOfRunningGraphs;
2105
+ instance.isActive = report.isActive;
2106
+ instance.isNonResponsive = report.isNonResponsive;
2107
+ instance.isBlocked = report.isBlocked;
2108
+ instance.runtimeState = report.state;
2109
+ instance.acceptingWork = report.acceptingWork;
2110
+ instance.reportedAt = report.reportedAt;
2111
+ instance.health = {
2112
+ ...instance.health ?? {},
2113
+ ...report.health ?? {},
2114
+ runtimeStatus: {
2115
+ state: report.state,
2116
+ acceptingWork: report.acceptingWork,
2117
+ reportedAt: report.reportedAt
2118
+ }
2119
+ };
2120
+ return true;
2121
+ }
2122
+ buildLocalRuntimeStatusReport(detailLevel = "minimal") {
2123
+ if (!this.serviceName || !this.serviceInstanceId) {
2124
+ return null;
2125
+ }
2126
+ const localInstance = this.getLocalInstance();
2127
+ if (!localInstance) {
2128
+ return null;
2129
+ }
2130
+ const numberOfRunningGraphs = this.activeRoutineExecutionIds.size || this.numberOfRunningGraphs || 0;
2131
+ this.numberOfRunningGraphs = numberOfRunningGraphs;
2132
+ const snapshot = this.resolveRuntimeStatusSnapshot(
2133
+ numberOfRunningGraphs,
2134
+ localInstance.isActive,
2135
+ localInstance.isNonResponsive,
2136
+ localInstance.isBlocked
2137
+ );
2138
+ const reportedAt = (/* @__PURE__ */ new Date()).toISOString();
2139
+ const report = {
2140
+ serviceName: this.serviceName,
2141
+ serviceInstanceId: this.serviceInstanceId,
2142
+ serviceAddress: localInstance.address,
2143
+ servicePort: localInstance.port,
2144
+ exposed: localInstance.exposed,
2145
+ isFrontend: localInstance.isFrontend,
2146
+ reportedAt,
2147
+ state: snapshot.state,
2148
+ acceptingWork: snapshot.acceptingWork,
2149
+ numberOfRunningGraphs: snapshot.numberOfRunningGraphs,
2150
+ isActive: snapshot.isActive,
2151
+ isNonResponsive: snapshot.isNonResponsive,
2152
+ isBlocked: snapshot.isBlocked,
2153
+ health: {
2154
+ ...localInstance.health ?? {},
2155
+ runtimeStatus: {
2156
+ state: snapshot.state,
2157
+ acceptingWork: snapshot.acceptingWork,
2158
+ reportedAt
2159
+ }
2160
+ }
2161
+ };
2162
+ this.applyRuntimeStatusReport(report);
2163
+ if (detailLevel !== "full") {
2164
+ delete report.health;
2165
+ }
2166
+ return report;
2167
+ }
2168
+ selectRuntimeStatusReportForTarget(inquiryResult, targetServiceName, targetServiceInstanceId) {
2169
+ const reports = Array.isArray(inquiryResult.runtimeStatusReports) ? inquiryResult.runtimeStatusReports : [];
2170
+ for (const candidate of reports) {
2171
+ const report = this.normalizeRuntimeStatusReport(candidate);
2172
+ if (!report) {
2173
+ continue;
2174
+ }
2175
+ if (report.serviceName === targetServiceName && report.serviceInstanceId === targetServiceInstanceId) {
2176
+ return report;
2177
+ }
2178
+ }
2179
+ return null;
2180
+ }
2181
+ async resolveRuntimeStatusFallbackInquiry(serviceName, serviceInstanceId, options = {}) {
2182
+ const inquiryResult = await CadenzaService.inquire(
2183
+ META_RUNTIME_STATUS_INTENT,
2184
+ {
2185
+ targetServiceName: serviceName,
2186
+ targetServiceInstanceId: serviceInstanceId,
2187
+ detailLevel: options.detailLevel ?? "minimal"
2188
+ },
2189
+ {
2190
+ overallTimeoutMs: options.overallTimeoutMs ?? this.runtimeStatusFallbackTimeoutMs,
2191
+ perResponderTimeoutMs: options.perResponderTimeoutMs ?? Math.max(250, Math.floor(this.runtimeStatusFallbackTimeoutMs * 0.75)),
2192
+ requireComplete: options.requireComplete ?? false
2193
+ }
2194
+ );
2195
+ const report = this.selectRuntimeStatusReportForTarget(
2196
+ inquiryResult,
2197
+ serviceName,
2198
+ serviceInstanceId
2199
+ );
2200
+ if (!report) {
2201
+ throw new Error(
2202
+ `No runtime status report for ${serviceName}/${serviceInstanceId}`
2203
+ );
2204
+ }
2205
+ if (!this.applyRuntimeStatusReport(report)) {
2206
+ throw new Error(
2207
+ `No tracked instance for runtime fallback ${serviceName}/${serviceInstanceId}`
2208
+ );
2209
+ }
2210
+ this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
2211
+ this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
2212
+ return {
2213
+ report,
2214
+ inquiryMeta: inquiryResult.__inquiryMeta ?? {}
2215
+ };
2216
+ }
2217
+ evaluateDependencyReadinessDetail(serviceName, serviceInstanceId, now = Date.now()) {
2218
+ const instance = this.getInstance(serviceName, serviceInstanceId);
2219
+ const missedHeartbeats = this.getHeartbeatMisses(serviceInstanceId, now);
2220
+ const runtimeState = instance ? instance.runtimeState ?? this.resolveRuntimeStatusSnapshot(
2221
+ instance.numberOfRunningGraphs ?? 0,
2222
+ instance.isActive,
2223
+ instance.isNonResponsive,
2224
+ instance.isBlocked
2225
+ ).state : "unavailable";
2226
+ const acceptingWork = instance ? typeof instance.acceptingWork === "boolean" ? instance.acceptingWork : this.resolveRuntimeStatusSnapshot(
2227
+ instance.numberOfRunningGraphs ?? 0,
2228
+ instance.isActive,
2229
+ instance.isNonResponsive,
2230
+ instance.isBlocked
2231
+ ).acceptingWork : false;
2232
+ const evaluation = evaluateDependencyReadiness({
2233
+ exists: Boolean(instance),
2234
+ runtimeState,
2235
+ acceptingWork,
2236
+ missedHeartbeats,
2237
+ missThreshold: this.runtimeStatusMissThreshold
2238
+ });
2239
+ const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId);
2240
+ return {
2241
+ serviceName,
2242
+ serviceInstanceId,
2243
+ dependencyState: evaluation.state,
2244
+ runtimeState,
2245
+ acceptingWork,
2246
+ missedHeartbeats,
2247
+ stale: evaluation.stale,
2248
+ blocked: evaluation.blocked,
2249
+ reason: evaluation.reason,
2250
+ lastHeartbeatAt: lastHeartbeat ? new Date(lastHeartbeat).toISOString() : null,
2251
+ reportedAt: instance?.reportedAt ?? null
2252
+ };
2253
+ }
2254
+ async buildLocalReadinessReport(options = {}) {
2255
+ const localRuntime = this.buildLocalRuntimeStatusReport("minimal");
2256
+ if (!localRuntime) {
2257
+ return null;
2258
+ }
2259
+ const detailLevel = options.detailLevel ?? "minimal";
2260
+ const includeDependencies = options.includeDependencies ?? detailLevel === "full";
2261
+ const refreshStaleDependencies = options.refreshStaleDependencies ?? true;
2262
+ const dependencyPairs = Array.from(this.readinessDependeesByService.entries()).flatMap(
2263
+ ([serviceName, instanceIds]) => Array.from(instanceIds).map((serviceInstanceId) => ({
2264
+ serviceName,
2265
+ serviceInstanceId
2266
+ }))
2267
+ ).sort((left, right) => {
2268
+ if (left.serviceName !== right.serviceName) {
2269
+ return left.serviceName.localeCompare(right.serviceName);
2270
+ }
2271
+ return left.serviceInstanceId.localeCompare(right.serviceInstanceId);
2272
+ });
2273
+ if (refreshStaleDependencies) {
2274
+ for (const dependency of dependencyPairs) {
2275
+ const misses = this.getHeartbeatMisses(dependency.serviceInstanceId);
2276
+ if (misses < this.runtimeStatusMissThreshold) {
2277
+ continue;
2278
+ }
2279
+ if (this.runtimeStatusFallbackInFlightByInstance.has(
2280
+ dependency.serviceInstanceId
2281
+ )) {
2282
+ continue;
2283
+ }
2284
+ this.runtimeStatusFallbackInFlightByInstance.add(
2285
+ dependency.serviceInstanceId
2286
+ );
2287
+ try {
2288
+ await this.resolveRuntimeStatusFallbackInquiry(
2289
+ dependency.serviceName,
2290
+ dependency.serviceInstanceId
2291
+ );
2292
+ } catch (error) {
2293
+ CadenzaService.log(
2294
+ "Readiness dependency fallback failed.",
2295
+ {
2296
+ serviceName: dependency.serviceName,
2297
+ serviceInstanceId: dependency.serviceInstanceId,
2298
+ error: error instanceof Error ? error.message : String(error)
2299
+ },
2300
+ "warning"
2301
+ );
2302
+ } finally {
2303
+ this.runtimeStatusFallbackInFlightByInstance.delete(
2304
+ dependency.serviceInstanceId
2305
+ );
2306
+ }
2307
+ }
2308
+ }
2309
+ const now = Date.now();
2310
+ const dependencyDetails = dependencyPairs.map(
2311
+ (dependency) => this.evaluateDependencyReadinessDetail(
2312
+ dependency.serviceName,
2313
+ dependency.serviceInstanceId,
2314
+ now
2315
+ )
2316
+ );
2317
+ const dependencySummary = summarizeDependencyReadiness(
2318
+ dependencyDetails.map((detail) => ({
2319
+ state: detail.dependencyState,
2320
+ stale: detail.stale,
2321
+ blocked: detail.blocked,
2322
+ reason: detail.reason
2323
+ }))
2324
+ );
2325
+ const readinessState = resolveServiceReadinessState(
2326
+ localRuntime.state,
2327
+ localRuntime.acceptingWork,
2328
+ dependencySummary
2329
+ );
2330
+ return {
2331
+ serviceName: localRuntime.serviceName,
2332
+ serviceInstanceId: localRuntime.serviceInstanceId,
2333
+ reportedAt: new Date(now).toISOString(),
2334
+ readinessState,
2335
+ runtimeState: localRuntime.state,
2336
+ acceptingWork: localRuntime.acceptingWork,
2337
+ dependencySummary,
2338
+ ...includeDependencies ? { dependencies: dependencyDetails } : {}
2339
+ };
2340
+ }
1243
2341
  reset() {
1244
2342
  this.instances.clear();
1245
2343
  this.deputies.clear();
@@ -1247,6 +2345,17 @@ var ServiceRegistry = class _ServiceRegistry {
1247
2345
  this.remoteIntents.clear();
1248
2346
  this.remoteIntentDeputiesByKey.clear();
1249
2347
  this.remoteIntentDeputiesByTask.clear();
2348
+ this.dependeesByService.clear();
2349
+ this.dependeeByInstance.clear();
2350
+ this.readinessDependeesByService.clear();
2351
+ this.readinessDependeeByInstance.clear();
2352
+ this.lastHeartbeatAtByInstance.clear();
2353
+ this.missedHeartbeatsByInstance.clear();
2354
+ this.runtimeStatusFallbackInFlightByInstance.clear();
2355
+ this.activeRoutineExecutionIds.clear();
2356
+ this.numberOfRunningGraphs = 0;
2357
+ this.runtimeStatusHeartbeatStarted = false;
2358
+ this.lastRuntimeStatusSnapshot = null;
1250
2359
  }
1251
2360
  };
1252
2361