@cadenza.io/service 2.6.1 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -250,6 +250,8 @@ var isBrowser = typeof window !== "undefined" && typeof window.document !== "und
250
250
  // src/utils/inquiry.ts
251
251
  var META_INTENT_PREFIX = "meta-";
252
252
  var META_RUNTIME_TRANSPORT_DIAGNOSTICS_INTENT = "meta-runtime-transport-diagnostics";
253
+ var META_RUNTIME_STATUS_INTENT = "meta-runtime-status";
254
+ var META_READINESS_INTENT = "meta-readiness";
253
255
  function isPlainObject(value) {
254
256
  return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
255
257
  }
@@ -313,8 +315,200 @@ function summarizeResponderStatuses(statuses) {
313
315
  return { responded, failed, timedOut, pending };
314
316
  }
315
317
 
318
+ // src/utils/readiness.ts
319
+ function evaluateDependencyReadiness(input) {
320
+ const missedHeartbeats = Math.max(
321
+ 0,
322
+ Math.trunc(Number(input.missedHeartbeats) || 0)
323
+ );
324
+ const stale = missedHeartbeats > 0;
325
+ const timeoutReached = missedHeartbeats >= Math.max(1, input.missThreshold);
326
+ if (!input.exists) {
327
+ return {
328
+ state: "unavailable",
329
+ stale: true,
330
+ blocked: true,
331
+ reason: "missing"
332
+ };
333
+ }
334
+ if (timeoutReached) {
335
+ return {
336
+ state: "unavailable",
337
+ stale: true,
338
+ blocked: true,
339
+ reason: "heartbeat-timeout"
340
+ };
341
+ }
342
+ if (input.runtimeState === "unavailable" || !input.acceptingWork) {
343
+ return {
344
+ state: "unavailable",
345
+ stale,
346
+ blocked: true,
347
+ reason: "runtime-unavailable"
348
+ };
349
+ }
350
+ if (stale) {
351
+ return {
352
+ state: "degraded",
353
+ stale: true,
354
+ blocked: false,
355
+ reason: "heartbeat-stale"
356
+ };
357
+ }
358
+ if (input.runtimeState === "overloaded") {
359
+ return {
360
+ state: "overloaded",
361
+ stale: false,
362
+ blocked: false,
363
+ reason: "runtime-overloaded"
364
+ };
365
+ }
366
+ if (input.runtimeState === "degraded") {
367
+ return {
368
+ state: "degraded",
369
+ stale: false,
370
+ blocked: false,
371
+ reason: "runtime-degraded"
372
+ };
373
+ }
374
+ return {
375
+ state: "ready",
376
+ stale: false,
377
+ blocked: false,
378
+ reason: "runtime-healthy"
379
+ };
380
+ }
381
+ function summarizeDependencyReadiness(evaluations) {
382
+ const summary = {
383
+ total: evaluations.length,
384
+ ready: 0,
385
+ degraded: 0,
386
+ overloaded: 0,
387
+ unavailable: 0,
388
+ stale: 0
389
+ };
390
+ for (const evaluation of evaluations) {
391
+ if (evaluation.state === "ready") summary.ready++;
392
+ if (evaluation.state === "degraded") summary.degraded++;
393
+ if (evaluation.state === "overloaded") summary.overloaded++;
394
+ if (evaluation.state === "unavailable") summary.unavailable++;
395
+ if (evaluation.stale) summary.stale++;
396
+ }
397
+ return summary;
398
+ }
399
+ function resolveServiceReadinessState(localRuntimeState, localAcceptingWork, dependencySummary) {
400
+ if (localRuntimeState === "unavailable" || !localAcceptingWork) {
401
+ return "blocked";
402
+ }
403
+ if (dependencySummary.unavailable > 0) {
404
+ return "blocked";
405
+ }
406
+ if (dependencySummary.degraded > 0 || dependencySummary.overloaded > 0 || dependencySummary.stale > 0) {
407
+ return "degraded";
408
+ }
409
+ return "ready";
410
+ }
411
+
412
+ // src/utils/runtimeStatus.ts
413
+ function resolveRuntimeStatus(input) {
414
+ const numberOfRunningGraphs = Math.max(
415
+ 0,
416
+ Math.trunc(Number(input.numberOfRunningGraphs) || 0)
417
+ );
418
+ const isActive = Boolean(input.isActive);
419
+ const isNonResponsive = Boolean(input.isNonResponsive);
420
+ const isBlocked = Boolean(input.isBlocked);
421
+ if (!isActive || isNonResponsive || isBlocked) {
422
+ return {
423
+ state: "unavailable",
424
+ acceptingWork: false,
425
+ numberOfRunningGraphs,
426
+ isActive,
427
+ isNonResponsive,
428
+ isBlocked
429
+ };
430
+ }
431
+ if (numberOfRunningGraphs >= input.overloadedGraphThreshold) {
432
+ return {
433
+ state: "overloaded",
434
+ acceptingWork: true,
435
+ numberOfRunningGraphs,
436
+ isActive,
437
+ isNonResponsive,
438
+ isBlocked
439
+ };
440
+ }
441
+ if (numberOfRunningGraphs >= input.degradedGraphThreshold) {
442
+ return {
443
+ state: "degraded",
444
+ acceptingWork: true,
445
+ numberOfRunningGraphs,
446
+ isActive,
447
+ isNonResponsive,
448
+ isBlocked
449
+ };
450
+ }
451
+ return {
452
+ state: "healthy",
453
+ acceptingWork: true,
454
+ numberOfRunningGraphs,
455
+ isActive,
456
+ isNonResponsive,
457
+ isBlocked
458
+ };
459
+ }
460
+ function runtimeStatusPriority(state) {
461
+ switch (state) {
462
+ case "healthy":
463
+ return 0;
464
+ case "degraded":
465
+ return 1;
466
+ case "overloaded":
467
+ return 2;
468
+ case "unavailable":
469
+ return 3;
470
+ default:
471
+ return 4;
472
+ }
473
+ }
474
+ function hasSignificantRuntimeStatusChange(previous, next) {
475
+ if (!previous) {
476
+ return true;
477
+ }
478
+ return previous.state !== next.state || previous.acceptingWork !== next.acceptingWork || previous.isActive !== next.isActive || previous.isNonResponsive !== next.isNonResponsive || previous.isBlocked !== next.isBlocked;
479
+ }
480
+
316
481
  // src/registry/ServiceRegistry.ts
317
482
  var META_SERVICE_REGISTRY_FULL_SYNC_INTENT = "meta-service-registry-full-sync";
483
+ var META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL = "meta.service_registry.runtime_status.heartbeat_tick";
484
+ var META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL = "meta.service_registry.runtime_status.monitor_tick";
485
+ var INTERNAL_RUNTIME_STATUS_TASK_NAMES = /* @__PURE__ */ new Set([
486
+ "Track local routine start",
487
+ "Track local routine end",
488
+ "Start runtime status sharing intervals",
489
+ "Broadcast runtime status",
490
+ "Monitor dependee heartbeat freshness",
491
+ "Resolve runtime status fallback inquiry",
492
+ "Respond runtime status inquiry",
493
+ "Respond readiness inquiry",
494
+ "Collect distributed readiness",
495
+ "Get status"
496
+ ]);
497
+ function readPositiveIntegerEnv(name, fallback) {
498
+ if (typeof process === "undefined") {
499
+ return fallback;
500
+ }
501
+ const raw = process.env?.[name];
502
+ const parsed = Number(raw);
503
+ if (!Number.isFinite(parsed)) {
504
+ return fallback;
505
+ }
506
+ const normalized = Math.trunc(parsed);
507
+ if (normalized <= 0) {
508
+ return fallback;
509
+ }
510
+ return normalized;
511
+ }
318
512
  var ServiceRegistry = class _ServiceRegistry {
319
513
  /**
320
514
  * Initializes a private constructor for managing service instances, remote signals,
@@ -332,6 +526,36 @@ var ServiceRegistry = class _ServiceRegistry {
332
526
  this.remoteIntents = /* @__PURE__ */ new Map();
333
527
  this.remoteIntentDeputiesByKey = /* @__PURE__ */ new Map();
334
528
  this.remoteIntentDeputiesByTask = /* @__PURE__ */ new Map();
529
+ this.dependeesByService = /* @__PURE__ */ new Map();
530
+ this.dependeeByInstance = /* @__PURE__ */ new Map();
531
+ this.readinessDependeesByService = /* @__PURE__ */ new Map();
532
+ this.readinessDependeeByInstance = /* @__PURE__ */ new Map();
533
+ this.lastHeartbeatAtByInstance = /* @__PURE__ */ new Map();
534
+ this.missedHeartbeatsByInstance = /* @__PURE__ */ new Map();
535
+ this.runtimeStatusFallbackInFlightByInstance = /* @__PURE__ */ new Set();
536
+ this.activeRoutineExecutionIds = /* @__PURE__ */ new Set();
537
+ this.runtimeStatusHeartbeatStarted = false;
538
+ this.lastRuntimeStatusSnapshot = null;
539
+ this.runtimeStatusHeartbeatIntervalMs = readPositiveIntegerEnv(
540
+ "CADENZA_RUNTIME_STATUS_HEARTBEAT_MS",
541
+ 3e4
542
+ );
543
+ this.runtimeStatusMissThreshold = readPositiveIntegerEnv(
544
+ "CADENZA_RUNTIME_STATUS_MISSED_HEARTBEATS",
545
+ 3
546
+ );
547
+ this.runtimeStatusFallbackTimeoutMs = readPositiveIntegerEnv(
548
+ "CADENZA_RUNTIME_STATUS_FALLBACK_TIMEOUT_MS",
549
+ 1500
550
+ );
551
+ this.degradedGraphThreshold = readPositiveIntegerEnv(
552
+ "CADENZA_RUNTIME_STATUS_DEGRADED_GRAPH_THRESHOLD",
553
+ 10
554
+ );
555
+ this.overloadedGraphThreshold = readPositiveIntegerEnv(
556
+ "CADENZA_RUNTIME_STATUS_OVERLOADED_GRAPH_THRESHOLD",
557
+ 20
558
+ );
335
559
  this.serviceName = null;
336
560
  this.serviceInstanceId = null;
337
561
  this.numberOfRunningGraphs = 0;
@@ -370,10 +594,137 @@ var ServiceRegistry = class _ServiceRegistry {
370
594
  }
371
595
  }
372
596
  });
597
+ CadenzaService.defineIntent({
598
+ name: META_RUNTIME_STATUS_INTENT,
599
+ description: "Gather lightweight runtime status reports from services in the distributed runtime.",
600
+ input: {
601
+ type: "object",
602
+ properties: {
603
+ detailLevel: {
604
+ type: "string",
605
+ constraints: {
606
+ oneOf: ["minimal", "full"]
607
+ }
608
+ },
609
+ targetServiceName: {
610
+ type: "string"
611
+ },
612
+ targetServiceInstanceId: {
613
+ type: "string"
614
+ }
615
+ }
616
+ },
617
+ output: {
618
+ type: "object",
619
+ properties: {
620
+ runtimeStatusReports: {
621
+ type: "array"
622
+ }
623
+ }
624
+ }
625
+ });
626
+ CadenzaService.createMetaTask(
627
+ "Respond runtime status inquiry",
628
+ (ctx) => {
629
+ const targetServiceName = ctx.targetServiceName;
630
+ const targetServiceInstanceId = ctx.targetServiceInstanceId;
631
+ const detailLevel = ctx.detailLevel === "full" ? "full" : "minimal";
632
+ const report = this.buildLocalRuntimeStatusReport(detailLevel);
633
+ if (!report) {
634
+ return {};
635
+ }
636
+ if (targetServiceName && targetServiceName !== report.serviceName) {
637
+ return {};
638
+ }
639
+ if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
640
+ return {};
641
+ }
642
+ return {
643
+ runtimeStatusReports: [report]
644
+ };
645
+ },
646
+ "Responds to runtime-status inquiries with local service instance status."
647
+ ).respondsTo(META_RUNTIME_STATUS_INTENT);
648
+ CadenzaService.defineIntent({
649
+ name: META_READINESS_INTENT,
650
+ description: "Gather service readiness reports derived from local runtime status and required dependees.",
651
+ input: {
652
+ type: "object",
653
+ properties: {
654
+ detailLevel: {
655
+ type: "string",
656
+ constraints: {
657
+ oneOf: ["minimal", "full"]
658
+ }
659
+ },
660
+ includeDependencies: {
661
+ type: "boolean"
662
+ },
663
+ refreshStaleDependencies: {
664
+ type: "boolean"
665
+ },
666
+ targetServiceName: {
667
+ type: "string"
668
+ },
669
+ targetServiceInstanceId: {
670
+ type: "string"
671
+ }
672
+ }
673
+ },
674
+ output: {
675
+ type: "object",
676
+ properties: {
677
+ readinessReports: {
678
+ type: "array"
679
+ }
680
+ }
681
+ }
682
+ });
683
+ CadenzaService.createMetaTask(
684
+ "Respond readiness inquiry",
685
+ async (ctx) => {
686
+ const targetServiceName = ctx.targetServiceName;
687
+ const targetServiceInstanceId = ctx.targetServiceInstanceId;
688
+ const report = await this.buildLocalReadinessReport({
689
+ detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
690
+ includeDependencies: ctx.includeDependencies,
691
+ refreshStaleDependencies: ctx.refreshStaleDependencies
692
+ });
693
+ if (!report) {
694
+ return {};
695
+ }
696
+ if (targetServiceName && targetServiceName !== report.serviceName) {
697
+ return {};
698
+ }
699
+ if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
700
+ return {};
701
+ }
702
+ return {
703
+ readinessReports: [report]
704
+ };
705
+ },
706
+ "Responds to distributed readiness inquiries using required dependee health."
707
+ ).respondsTo(META_READINESS_INTENT);
373
708
  this.handleInstanceUpdateTask = CadenzaService.createMetaTask(
374
709
  "Handle Instance Update",
375
710
  (ctx, emit) => {
376
- const { serviceInstance } = ctx;
711
+ const serviceInstance = ctx.serviceInstance ?? (ctx.__serviceInstanceId || ctx.serviceInstanceId ? {
712
+ uuid: ctx.__serviceInstanceId ?? ctx.serviceInstanceId,
713
+ serviceName: ctx.__serviceName ?? ctx.serviceName,
714
+ address: ctx.serviceAddress ?? "",
715
+ port: ctx.servicePort ?? 0,
716
+ exposed: !!ctx.exposed,
717
+ isFrontend: !!ctx.isFrontend,
718
+ isActive: typeof ctx.isActive === "boolean" ? ctx.isActive : typeof ctx.__active === "boolean" ? ctx.__active : true,
719
+ isNonResponsive: !!ctx.isNonResponsive,
720
+ isBlocked: !!ctx.isBlocked,
721
+ health: ctx.health ?? ctx.__health ?? {},
722
+ numberOfRunningGraphs: ctx.numberOfRunningGraphs ?? ctx.__numberOfRunningGraphs ?? 0,
723
+ isPrimary: false
724
+ } : void 0);
725
+ if (!serviceInstance?.uuid || !serviceInstance?.serviceName) {
726
+ return false;
727
+ }
377
728
  const {
378
729
  uuid: uuid4,
379
730
  serviceName,
@@ -395,6 +746,7 @@ var ServiceRegistry = class _ServiceRegistry {
395
746
  emit(`meta.socket_shutdown_requested:${address}_${port}`, {});
396
747
  emit(`meta.fetch.destroy_requested:${address}_${port}`, {});
397
748
  }
749
+ this.unregisterDependee(uuid4, serviceName);
398
750
  return;
399
751
  }
400
752
  if (!this.instances.has(serviceName))
@@ -406,6 +758,18 @@ var ServiceRegistry = class _ServiceRegistry {
406
758
  } else {
407
759
  instances.push(serviceInstance);
408
760
  }
761
+ const trackedInstance = existing ?? instances.find((instance) => instance.uuid === uuid4);
762
+ if (trackedInstance) {
763
+ const snapshot = this.resolveRuntimeStatusSnapshot(
764
+ trackedInstance.numberOfRunningGraphs ?? 0,
765
+ trackedInstance.isActive,
766
+ trackedInstance.isNonResponsive,
767
+ trackedInstance.isBlocked
768
+ );
769
+ trackedInstance.runtimeState = snapshot.state;
770
+ trackedInstance.acceptingWork = snapshot.acceptingWork;
771
+ trackedInstance.reportedAt = trackedInstance.reportedAt ?? (/* @__PURE__ */ new Date()).toISOString();
772
+ }
409
773
  if (this.serviceName === serviceName) {
410
774
  return false;
411
775
  }
@@ -445,13 +809,27 @@ var ServiceRegistry = class _ServiceRegistry {
445
809
  "global.meta.service_instance.inserted",
446
810
  "global.meta.service_instance.updated",
447
811
  "meta.service_instance.inserted",
448
- "meta.service_instance.updated",
449
- "meta.socket_client.status_received"
812
+ "meta.service_instance.updated"
450
813
  ).attachSignal(
451
814
  "meta.service_registry.dependee_registered",
452
815
  "meta.socket_shutdown_requested",
453
816
  "meta.fetch.destroy_requested"
454
817
  );
818
+ CadenzaService.createMetaTask(
819
+ "Track dependee registration",
820
+ (ctx) => {
821
+ if (!ctx.serviceName || !ctx.serviceInstanceId) {
822
+ return false;
823
+ }
824
+ this.registerDependee(ctx.serviceName, ctx.serviceInstanceId, {
825
+ requiredForReadiness: this.shouldRequireReadinessFromCommunicationTypes(
826
+ ctx.communicationTypes
827
+ )
828
+ });
829
+ return true;
830
+ },
831
+ "Tracks remote dependency instances for runtime heartbeat monitoring."
832
+ ).doOn("meta.service_registry.dependee_registered");
455
833
  CadenzaService.createMetaTask("Split service instances", function* (ctx) {
456
834
  if (!ctx.serviceInstances) {
457
835
  return;
@@ -551,6 +929,15 @@ var ServiceRegistry = class _ServiceRegistry {
551
929
  for (const instance of instances ?? []) {
552
930
  instance.isActive = false;
553
931
  instance.isNonResponsive = true;
932
+ const snapshot = this.resolveRuntimeStatusSnapshot(
933
+ instance.numberOfRunningGraphs ?? 0,
934
+ instance.isActive,
935
+ instance.isNonResponsive,
936
+ instance.isBlocked
937
+ );
938
+ instance.runtimeState = snapshot.state;
939
+ instance.acceptingWork = snapshot.acceptingWork;
940
+ instance.reportedAt = (/* @__PURE__ */ new Date()).toISOString();
554
941
  emit("global.meta.service_registry.service_not_responding", {
555
942
  data: {
556
943
  isActive: false,
@@ -568,7 +955,8 @@ var ServiceRegistry = class _ServiceRegistry {
568
955
  "meta.fetch.handshake_failed",
569
956
  "meta.fetch.handshake_failed.*",
570
957
  "meta.socket_client.disconnected",
571
- "meta.socket_client.disconnected.*"
958
+ "meta.socket_client.disconnected.*",
959
+ "meta.service_registry.runtime_status_unreachable"
572
960
  ).attachSignal("global.meta.service_registry.service_not_responding");
573
961
  this.handleServiceHandshakeTask = CadenzaService.createMetaTask(
574
962
  "Handle service handshake",
@@ -583,6 +971,15 @@ var ServiceRegistry = class _ServiceRegistry {
583
971
  }
584
972
  instance.isActive = true;
585
973
  instance.isNonResponsive = false;
974
+ const snapshot = this.resolveRuntimeStatusSnapshot(
975
+ instance.numberOfRunningGraphs ?? 0,
976
+ instance.isActive,
977
+ instance.isNonResponsive,
978
+ instance.isBlocked
979
+ );
980
+ instance.runtimeState = snapshot.state;
981
+ instance.acceptingWork = snapshot.acceptingWork;
982
+ instance.reportedAt = (/* @__PURE__ */ new Date()).toISOString();
586
983
  emit("global.meta.service_registry.service_handshake", {
587
984
  data: {
588
985
  isActive: instance.isActive,
@@ -600,6 +997,7 @@ var ServiceRegistry = class _ServiceRegistry {
600
997
  if (indexToDelete >= 0) {
601
998
  this.instances.get(serviceName)?.splice(indexToDelete, 1);
602
999
  }
1000
+ this.unregisterDependee(i.uuid, serviceName);
603
1001
  emit("global.meta.service_registry.deleted", {
604
1002
  data: {
605
1003
  isActive: false,
@@ -621,14 +1019,46 @@ var ServiceRegistry = class _ServiceRegistry {
621
1019
  this.handleSocketStatusUpdateTask = CadenzaService.createMetaTask(
622
1020
  "Handle Socket Status Update",
623
1021
  (ctx) => {
624
- const instanceId = ctx.__serviceInstanceId;
625
- const serviceName = ctx.__serviceName;
626
- const instances = this.instances.get(serviceName);
627
- const instance = instances?.find((i) => i.uuid === instanceId);
628
- if (instance) {
629
- instance.health = ctx.health;
630
- instance.numberOfRunningGraphs = ctx.numberOfRunningGraphs;
1022
+ const report = this.normalizeRuntimeStatusReport(ctx);
1023
+ if (!report) {
1024
+ return false;
1025
+ }
1026
+ if (report.serviceName === this.serviceName && report.serviceInstanceId === this.serviceInstanceId) {
1027
+ return false;
1028
+ }
1029
+ let applied = this.applyRuntimeStatusReport(report);
1030
+ if (!applied && report.serviceAddress && typeof report.servicePort === "number") {
1031
+ if (!this.instances.has(report.serviceName)) {
1032
+ this.instances.set(report.serviceName, []);
1033
+ }
1034
+ this.instances.get(report.serviceName).push({
1035
+ uuid: report.serviceInstanceId,
1036
+ serviceName: report.serviceName,
1037
+ address: report.serviceAddress,
1038
+ port: report.servicePort,
1039
+ exposed: !!report.exposed,
1040
+ isFrontend: !!report.isFrontend,
1041
+ isActive: report.isActive,
1042
+ isNonResponsive: report.isNonResponsive,
1043
+ isBlocked: report.isBlocked,
1044
+ numberOfRunningGraphs: report.numberOfRunningGraphs,
1045
+ runtimeState: report.state,
1046
+ acceptingWork: report.acceptingWork,
1047
+ reportedAt: report.reportedAt,
1048
+ health: report.health ?? {},
1049
+ isPrimary: false
1050
+ });
1051
+ applied = true;
631
1052
  }
1053
+ if (!applied) {
1054
+ return false;
1055
+ }
1056
+ this.registerDependee(report.serviceName, report.serviceInstanceId);
1057
+ this.lastHeartbeatAtByInstance.set(report.serviceInstanceId, Date.now());
1058
+ this.missedHeartbeatsByInstance.set(report.serviceInstanceId, 0);
1059
+ this.runtimeStatusFallbackInFlightByInstance.delete(
1060
+ report.serviceInstanceId
1061
+ );
632
1062
  return true;
633
1063
  },
634
1064
  "Handles status update from socket broadcast"
@@ -758,7 +1188,25 @@ var ServiceRegistry = class _ServiceRegistry {
758
1188
  const { __serviceName, __triedInstances, __retries, __broadcast } = context;
759
1189
  let retries = __retries ?? 0;
760
1190
  let triedInstances = __triedInstances ?? [];
761
- const instances = this.instances.get(__serviceName)?.filter((i) => i.isActive && !i.isNonResponsive && !i.isBlocked).sort((a, b) => a.numberOfRunningGraphs - b.numberOfRunningGraphs);
1191
+ const instances = this.instances.get(__serviceName)?.filter((i) => i.isActive && !i.isNonResponsive && !i.isBlocked).sort((a, b) => {
1192
+ const leftStatus = this.resolveRuntimeStatusSnapshot(
1193
+ a.numberOfRunningGraphs ?? 0,
1194
+ a.isActive,
1195
+ a.isNonResponsive,
1196
+ a.isBlocked
1197
+ );
1198
+ const rightStatus = this.resolveRuntimeStatusSnapshot(
1199
+ b.numberOfRunningGraphs ?? 0,
1200
+ b.isActive,
1201
+ b.isNonResponsive,
1202
+ b.isBlocked
1203
+ );
1204
+ const priorityDelta = runtimeStatusPriority(leftStatus.state) - runtimeStatusPriority(rightStatus.state);
1205
+ if (priorityDelta !== 0) {
1206
+ return priorityDelta;
1207
+ }
1208
+ return (a.numberOfRunningGraphs ?? 0) - (b.numberOfRunningGraphs ?? 0);
1209
+ });
762
1210
  if (!instances || instances.length === 0 || retries > this.retryCount) {
763
1211
  context.errored = true;
764
1212
  context.__error = `No active instances for ${__serviceName}. Retries: ${retries}. ${this.instances.get(
@@ -844,15 +1292,292 @@ var ServiceRegistry = class _ServiceRegistry {
844
1292
  errored: true
845
1293
  };
846
1294
  }
847
- const self = this.instances.get(this.serviceName)?.find((i) => i.uuid === this.serviceInstanceId);
1295
+ const report = this.buildLocalRuntimeStatusReport("full");
1296
+ if (!report) {
1297
+ return {
1298
+ ...ctx,
1299
+ __status: "error",
1300
+ __error: "No local service instance available for status check",
1301
+ errored: true
1302
+ };
1303
+ }
848
1304
  return {
849
1305
  ...ctx,
850
1306
  __status: "ok",
851
- __numberOfRunningGraphs: self?.numberOfRunningGraphs ?? 0,
852
- __health: self?.health ?? {},
853
- __active: self?.isActive ?? false
1307
+ __serviceName: report.serviceName,
1308
+ __serviceInstanceId: report.serviceInstanceId,
1309
+ __numberOfRunningGraphs: report.numberOfRunningGraphs,
1310
+ __health: report.health ?? {},
1311
+ __active: report.isActive,
1312
+ reportedAt: report.reportedAt,
1313
+ serviceName: report.serviceName,
1314
+ serviceInstanceId: report.serviceInstanceId,
1315
+ numberOfRunningGraphs: report.numberOfRunningGraphs,
1316
+ health: report.health ?? {},
1317
+ isActive: report.isActive,
1318
+ isNonResponsive: report.isNonResponsive,
1319
+ isBlocked: report.isBlocked,
1320
+ state: report.state,
1321
+ acceptingWork: report.acceptingWork
854
1322
  };
855
- }).doOn("meta.socket.status_check_requested");
1323
+ }).doOn(
1324
+ "meta.socket.status_check_requested",
1325
+ "meta.rest.status_check_requested"
1326
+ );
1327
+ CadenzaService.createMetaTask(
1328
+ "Track local routine start",
1329
+ (ctx, emit) => {
1330
+ const sourceTaskName = String(ctx.__signalEmission?.taskName ?? "");
1331
+ if (INTERNAL_RUNTIME_STATUS_TASK_NAMES.has(sourceTaskName)) {
1332
+ return false;
1333
+ }
1334
+ const routineId = String(
1335
+ ctx.filter?.uuid ?? ctx.__routineExecId ?? ""
1336
+ );
1337
+ if (!routineId) {
1338
+ return false;
1339
+ }
1340
+ this.activeRoutineExecutionIds.add(routineId);
1341
+ this.numberOfRunningGraphs = this.activeRoutineExecutionIds.size;
1342
+ const localInstance = this.getLocalInstance();
1343
+ if (!localInstance) {
1344
+ return true;
1345
+ }
1346
+ const snapshot = this.resolveRuntimeStatusSnapshot(
1347
+ this.numberOfRunningGraphs,
1348
+ localInstance.isActive,
1349
+ localInstance.isNonResponsive,
1350
+ localInstance.isBlocked
1351
+ );
1352
+ if (hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
1353
+ emit("meta.service_registry.runtime_status_broadcast_requested", {
1354
+ reason: "runtime-state-change"
1355
+ });
1356
+ }
1357
+ return true;
1358
+ },
1359
+ "Tracks local routine starts for runtime load status."
1360
+ ).doOn("meta.node.started_routine_execution");
1361
+ CadenzaService.createMetaTask(
1362
+ "Track local routine end",
1363
+ (ctx, emit) => {
1364
+ const sourceTaskName = String(ctx.__signalEmission?.taskName ?? "");
1365
+ if (INTERNAL_RUNTIME_STATUS_TASK_NAMES.has(sourceTaskName)) {
1366
+ return false;
1367
+ }
1368
+ const routineId = String(
1369
+ ctx.filter?.uuid ?? ctx.__routineExecId ?? ""
1370
+ );
1371
+ if (!routineId) {
1372
+ return false;
1373
+ }
1374
+ this.activeRoutineExecutionIds.delete(routineId);
1375
+ this.numberOfRunningGraphs = this.activeRoutineExecutionIds.size;
1376
+ const localInstance = this.getLocalInstance();
1377
+ if (!localInstance) {
1378
+ return true;
1379
+ }
1380
+ const snapshot = this.resolveRuntimeStatusSnapshot(
1381
+ this.numberOfRunningGraphs,
1382
+ localInstance.isActive,
1383
+ localInstance.isNonResponsive,
1384
+ localInstance.isBlocked
1385
+ );
1386
+ if (hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
1387
+ emit("meta.service_registry.runtime_status_broadcast_requested", {
1388
+ reason: "runtime-state-change"
1389
+ });
1390
+ }
1391
+ return true;
1392
+ },
1393
+ "Tracks local routine completion for runtime load status."
1394
+ ).doOn("meta.node.ended_routine_execution");
1395
+ CadenzaService.createMetaTask(
1396
+ "Start runtime status sharing intervals",
1397
+ () => {
1398
+ if (this.runtimeStatusHeartbeatStarted) {
1399
+ return false;
1400
+ }
1401
+ this.runtimeStatusHeartbeatStarted = true;
1402
+ CadenzaService.interval(
1403
+ META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL,
1404
+ { reason: "heartbeat" },
1405
+ this.runtimeStatusHeartbeatIntervalMs,
1406
+ true
1407
+ );
1408
+ CadenzaService.interval(
1409
+ META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL,
1410
+ {},
1411
+ this.runtimeStatusHeartbeatIntervalMs
1412
+ );
1413
+ return true;
1414
+ },
1415
+ "Starts runtime status heartbeat and heartbeat-monitor loops once per service instance."
1416
+ ).doOn("meta.service_registry.instance_inserted");
1417
+ CadenzaService.createMetaTask(
1418
+ "Broadcast runtime status",
1419
+ (ctx, emit) => {
1420
+ const report = this.buildLocalRuntimeStatusReport(
1421
+ ctx.detailLevel === "full" ? "full" : "minimal"
1422
+ );
1423
+ if (!report) {
1424
+ return false;
1425
+ }
1426
+ const snapshot = this.resolveRuntimeStatusSnapshot(
1427
+ report.numberOfRunningGraphs,
1428
+ report.isActive,
1429
+ report.isNonResponsive,
1430
+ report.isBlocked
1431
+ );
1432
+ const force = ctx.reason === "heartbeat" || ctx.force === true || this.lastRuntimeStatusSnapshot === null;
1433
+ if (!force && !hasSignificantRuntimeStatusChange(this.lastRuntimeStatusSnapshot, snapshot)) {
1434
+ return false;
1435
+ }
1436
+ this.lastRuntimeStatusSnapshot = snapshot;
1437
+ emit("meta.service.updated", {
1438
+ __serviceName: report.serviceName,
1439
+ __serviceInstanceId: report.serviceInstanceId,
1440
+ __reportedAt: report.reportedAt,
1441
+ __numberOfRunningGraphs: report.numberOfRunningGraphs,
1442
+ __health: report.health ?? {},
1443
+ __active: report.isActive,
1444
+ serviceName: report.serviceName,
1445
+ serviceInstanceId: report.serviceInstanceId,
1446
+ serviceAddress: report.serviceAddress,
1447
+ servicePort: report.servicePort,
1448
+ exposed: report.exposed,
1449
+ isFrontend: report.isFrontend,
1450
+ reportedAt: report.reportedAt,
1451
+ numberOfRunningGraphs: report.numberOfRunningGraphs,
1452
+ health: report.health ?? {},
1453
+ isActive: report.isActive,
1454
+ isNonResponsive: report.isNonResponsive,
1455
+ isBlocked: report.isBlocked,
1456
+ state: report.state,
1457
+ acceptingWork: report.acceptingWork
1458
+ });
1459
+ return true;
1460
+ },
1461
+ "Broadcasts local runtime status to connected dependees."
1462
+ ).doOn(
1463
+ META_RUNTIME_STATUS_HEARTBEAT_TICK_SIGNAL,
1464
+ "meta.service_registry.runtime_status_broadcast_requested"
1465
+ );
1466
+ CadenzaService.createMetaTask(
1467
+ "Monitor dependee heartbeat freshness",
1468
+ (ctx, emit) => {
1469
+ if (!this.useSocket) {
1470
+ return false;
1471
+ }
1472
+ const now = Date.now();
1473
+ for (const [serviceName, instanceIds] of this.dependeesByService) {
1474
+ for (const serviceInstanceId of instanceIds) {
1475
+ const instance = this.getInstance(serviceName, serviceInstanceId);
1476
+ if (!instance || !instance.isActive || instance.isBlocked) {
1477
+ continue;
1478
+ }
1479
+ const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
1480
+ const misses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
1481
+ const heartbeatBudget = this.runtimeStatusHeartbeatIntervalMs * (misses + 1);
1482
+ if (lastHeartbeat > 0 && now - lastHeartbeat < heartbeatBudget) {
1483
+ continue;
1484
+ }
1485
+ const nextMisses = misses + 1;
1486
+ this.missedHeartbeatsByInstance.set(serviceInstanceId, nextMisses);
1487
+ if (nextMisses < this.runtimeStatusMissThreshold || this.runtimeStatusFallbackInFlightByInstance.has(serviceInstanceId)) {
1488
+ continue;
1489
+ }
1490
+ this.runtimeStatusFallbackInFlightByInstance.add(serviceInstanceId);
1491
+ emit("meta.service_registry.runtime_status_fallback_requested", {
1492
+ ...ctx,
1493
+ serviceName,
1494
+ serviceInstanceId,
1495
+ serviceAddress: instance.address,
1496
+ servicePort: instance.port
1497
+ });
1498
+ }
1499
+ }
1500
+ return true;
1501
+ },
1502
+ "Monitors dependee heartbeat freshness and requests inquiry fallback after repeated misses."
1503
+ ).doOn(META_RUNTIME_STATUS_MONITOR_TICK_SIGNAL);
1504
+ CadenzaService.createMetaTask(
1505
+ "Resolve runtime status fallback inquiry",
1506
+ async (ctx, emit) => {
1507
+ const serviceName = ctx.serviceName;
1508
+ const serviceInstanceId = ctx.serviceInstanceId;
1509
+ if (!serviceName || !serviceInstanceId) {
1510
+ return false;
1511
+ }
1512
+ try {
1513
+ const { report, inquiryMeta } = await this.resolveRuntimeStatusFallbackInquiry(
1514
+ serviceName,
1515
+ serviceInstanceId,
1516
+ {
1517
+ detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
1518
+ overallTimeoutMs: ctx.overallTimeoutMs,
1519
+ perResponderTimeoutMs: ctx.perResponderTimeoutMs,
1520
+ requireComplete: ctx.requireComplete
1521
+ }
1522
+ );
1523
+ return {
1524
+ ...ctx,
1525
+ runtimeStatusReport: report,
1526
+ __inquiryMeta: inquiryMeta
1527
+ };
1528
+ } catch (error) {
1529
+ const instance = this.getInstance(serviceName, serviceInstanceId);
1530
+ const message = error instanceof Error ? error.message : String(error);
1531
+ CadenzaService.log(
1532
+ "Runtime status fallback inquiry failed.",
1533
+ {
1534
+ serviceName,
1535
+ serviceInstanceId,
1536
+ error: message
1537
+ },
1538
+ "warning"
1539
+ );
1540
+ emit("meta.service_registry.runtime_status_unreachable", {
1541
+ ...ctx,
1542
+ serviceName,
1543
+ serviceInstanceId,
1544
+ serviceAddress: instance?.address ?? ctx.serviceAddress,
1545
+ servicePort: instance?.port ?? ctx.servicePort,
1546
+ __error: message,
1547
+ errored: true
1548
+ });
1549
+ return {
1550
+ ...ctx,
1551
+ __error: message,
1552
+ errored: true
1553
+ };
1554
+ } finally {
1555
+ this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
1556
+ }
1557
+ },
1558
+ "Runs runtime-status inquiry fallback for a dependee instance after missed heartbeats."
1559
+ ).doOn("meta.service_registry.runtime_status_fallback_requested").emits("meta.service_registry.runtime_status_fallback_resolved").emitsOnFail("meta.service_registry.runtime_status_fallback_failed");
1560
+ this.collectReadinessTask = CadenzaService.createMetaTask(
1561
+ "Collect distributed readiness",
1562
+ async (ctx) => {
1563
+ const inquiryResult = await CadenzaService.inquire(
1564
+ META_READINESS_INTENT,
1565
+ {
1566
+ detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
1567
+ includeDependencies: ctx.includeDependencies,
1568
+ refreshStaleDependencies: ctx.refreshStaleDependencies,
1569
+ targetServiceName: ctx.targetServiceName,
1570
+ targetServiceInstanceId: ctx.targetServiceInstanceId
1571
+ },
1572
+ ctx.inquiryOptions ?? ctx.__inquiryOptions ?? {}
1573
+ );
1574
+ return {
1575
+ ...ctx,
1576
+ ...inquiryResult
1577
+ };
1578
+ },
1579
+ "Collects distributed readiness reports from services."
1580
+ ).doOn("meta.service_registry.readiness_requested").emits("meta.service_registry.readiness_collected").emitsOnFail("meta.service_registry.readiness_failed");
856
1581
  this.collectTransportDiagnosticsTask = CadenzaService.createMetaTask(
857
1582
  "Collect transport diagnostics",
858
1583
  async (ctx) => {
@@ -1192,6 +1917,379 @@ var ServiceRegistry = class _ServiceRegistry {
1192
1917
  localTaskName: task.name
1193
1918
  };
1194
1919
  }
1920
+ getInstance(serviceName, instanceId) {
1921
+ return this.instances.get(serviceName)?.find((instance) => instance.uuid === instanceId);
1922
+ }
1923
+ getLocalInstance() {
1924
+ if (!this.serviceName || !this.serviceInstanceId) {
1925
+ return void 0;
1926
+ }
1927
+ return this.getInstance(this.serviceName, this.serviceInstanceId);
1928
+ }
1929
+ registerDependee(serviceName, serviceInstanceId, options = {}) {
1930
+ if (!serviceName || !serviceInstanceId) {
1931
+ return;
1932
+ }
1933
+ if (!this.dependeesByService.has(serviceName)) {
1934
+ this.dependeesByService.set(serviceName, /* @__PURE__ */ new Set());
1935
+ }
1936
+ this.dependeesByService.get(serviceName).add(serviceInstanceId);
1937
+ this.dependeeByInstance.set(serviceInstanceId, serviceName);
1938
+ if (options.requiredForReadiness) {
1939
+ if (!this.readinessDependeesByService.has(serviceName)) {
1940
+ this.readinessDependeesByService.set(serviceName, /* @__PURE__ */ new Set());
1941
+ }
1942
+ this.readinessDependeesByService.get(serviceName).add(serviceInstanceId);
1943
+ this.readinessDependeeByInstance.set(serviceInstanceId, serviceName);
1944
+ }
1945
+ this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
1946
+ this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
1947
+ }
1948
+ unregisterDependee(serviceInstanceId, serviceName) {
1949
+ const dependeeServiceName = serviceName ?? this.dependeeByInstance.get(serviceInstanceId);
1950
+ if (dependeeServiceName) {
1951
+ this.dependeesByService.get(dependeeServiceName)?.delete(serviceInstanceId);
1952
+ if (!this.dependeesByService.get(dependeeServiceName)?.size) {
1953
+ this.dependeesByService.delete(dependeeServiceName);
1954
+ }
1955
+ }
1956
+ this.dependeeByInstance.delete(serviceInstanceId);
1957
+ const readinessDependeeServiceName = serviceName ?? this.readinessDependeeByInstance.get(serviceInstanceId);
1958
+ if (readinessDependeeServiceName) {
1959
+ this.readinessDependeesByService.get(readinessDependeeServiceName)?.delete(serviceInstanceId);
1960
+ if (!this.readinessDependeesByService.get(readinessDependeeServiceName)?.size) {
1961
+ this.readinessDependeesByService.delete(readinessDependeeServiceName);
1962
+ }
1963
+ }
1964
+ this.readinessDependeeByInstance.delete(serviceInstanceId);
1965
+ this.lastHeartbeatAtByInstance.delete(serviceInstanceId);
1966
+ this.missedHeartbeatsByInstance.delete(serviceInstanceId);
1967
+ this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
1968
+ }
1969
+ getHeartbeatMisses(serviceInstanceId, now = Date.now()) {
1970
+ const observedMisses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
1971
+ const lastHeartbeatAt = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
1972
+ if (lastHeartbeatAt <= 0) {
1973
+ return Math.max(observedMisses, this.runtimeStatusMissThreshold);
1974
+ }
1975
+ const estimatedMisses = Math.max(
1976
+ 0,
1977
+ Math.floor((now - lastHeartbeatAt) / this.runtimeStatusHeartbeatIntervalMs)
1978
+ );
1979
+ return Math.max(observedMisses, estimatedMisses);
1980
+ }
1981
+ shouldRequireReadinessFromCommunicationTypes(communicationTypes) {
1982
+ if (!Array.isArray(communicationTypes)) {
1983
+ return false;
1984
+ }
1985
+ return communicationTypes.some((type) => {
1986
+ const normalized = String(type).toLowerCase();
1987
+ return normalized === "delegation" || normalized === "inquiry";
1988
+ });
1989
+ }
1990
+ resolveRuntimeStatusSnapshot(numberOfRunningGraphs, isActive, isNonResponsive, isBlocked) {
1991
+ return resolveRuntimeStatus({
1992
+ numberOfRunningGraphs,
1993
+ isActive,
1994
+ isNonResponsive,
1995
+ isBlocked,
1996
+ degradedGraphThreshold: this.degradedGraphThreshold,
1997
+ overloadedGraphThreshold: this.overloadedGraphThreshold
1998
+ });
1999
+ }
2000
+ normalizeRuntimeStatusReport(ctx) {
2001
+ const serviceName = ctx.serviceName ?? ctx.__serviceName ?? ctx.serviceInstance?.serviceName;
2002
+ const serviceInstanceId = ctx.serviceInstanceId ?? ctx.__serviceInstanceId ?? ctx.serviceInstance?.uuid;
2003
+ if (!serviceName || !serviceInstanceId) {
2004
+ return null;
2005
+ }
2006
+ const servicePort = ctx.servicePort ?? ctx.port ?? ctx.serviceInstance?.port;
2007
+ const numberOfRunningGraphs = Math.max(
2008
+ 0,
2009
+ Math.trunc(
2010
+ Number(ctx.numberOfRunningGraphs ?? ctx.__numberOfRunningGraphs ?? 0)
2011
+ )
2012
+ );
2013
+ const isActive = Boolean(ctx.isActive ?? ctx.__active ?? true);
2014
+ const isNonResponsive = Boolean(ctx.isNonResponsive ?? false);
2015
+ const isBlocked = Boolean(ctx.isBlocked ?? false);
2016
+ const resolved = this.resolveRuntimeStatusSnapshot(
2017
+ numberOfRunningGraphs,
2018
+ isActive,
2019
+ isNonResponsive,
2020
+ isBlocked
2021
+ );
2022
+ return {
2023
+ serviceName,
2024
+ serviceInstanceId,
2025
+ serviceAddress: ctx.serviceAddress ?? ctx.address ?? ctx.serviceInstance?.address,
2026
+ servicePort: typeof servicePort === "number" ? servicePort : void 0,
2027
+ exposed: typeof ctx.exposed === "boolean" ? ctx.exposed : typeof ctx.serviceInstance?.exposed === "boolean" ? ctx.serviceInstance.exposed : void 0,
2028
+ isFrontend: typeof ctx.isFrontend === "boolean" ? ctx.isFrontend : typeof ctx.serviceInstance?.isFrontend === "boolean" ? ctx.serviceInstance.isFrontend : void 0,
2029
+ reportedAt: ctx.reportedAt ?? (typeof ctx.__reportedAt === "string" ? ctx.__reportedAt : void 0) ?? (/* @__PURE__ */ new Date()).toISOString(),
2030
+ state: ctx.state === "healthy" || ctx.state === "degraded" || ctx.state === "overloaded" || ctx.state === "unavailable" ? ctx.state : resolved.state,
2031
+ acceptingWork: typeof ctx.acceptingWork === "boolean" ? ctx.acceptingWork : resolved.acceptingWork,
2032
+ numberOfRunningGraphs,
2033
+ isActive,
2034
+ isNonResponsive,
2035
+ isBlocked,
2036
+ health: ctx.health ?? ctx.__health ?? {}
2037
+ };
2038
+ }
2039
+ applyRuntimeStatusReport(report) {
2040
+ const instance = this.getInstance(report.serviceName, report.serviceInstanceId);
2041
+ if (!instance) {
2042
+ return false;
2043
+ }
2044
+ if (report.serviceAddress) {
2045
+ instance.address = report.serviceAddress;
2046
+ }
2047
+ if (typeof report.servicePort === "number") {
2048
+ instance.port = report.servicePort;
2049
+ }
2050
+ if (typeof report.exposed === "boolean") {
2051
+ instance.exposed = report.exposed;
2052
+ }
2053
+ if (typeof report.isFrontend === "boolean") {
2054
+ instance.isFrontend = report.isFrontend;
2055
+ }
2056
+ instance.numberOfRunningGraphs = report.numberOfRunningGraphs;
2057
+ instance.isActive = report.isActive;
2058
+ instance.isNonResponsive = report.isNonResponsive;
2059
+ instance.isBlocked = report.isBlocked;
2060
+ instance.runtimeState = report.state;
2061
+ instance.acceptingWork = report.acceptingWork;
2062
+ instance.reportedAt = report.reportedAt;
2063
+ instance.health = {
2064
+ ...instance.health ?? {},
2065
+ ...report.health ?? {},
2066
+ runtimeStatus: {
2067
+ state: report.state,
2068
+ acceptingWork: report.acceptingWork,
2069
+ reportedAt: report.reportedAt
2070
+ }
2071
+ };
2072
+ return true;
2073
+ }
2074
+ buildLocalRuntimeStatusReport(detailLevel = "minimal") {
2075
+ if (!this.serviceName || !this.serviceInstanceId) {
2076
+ return null;
2077
+ }
2078
+ const localInstance = this.getLocalInstance();
2079
+ if (!localInstance) {
2080
+ return null;
2081
+ }
2082
+ const numberOfRunningGraphs = this.activeRoutineExecutionIds.size || this.numberOfRunningGraphs || 0;
2083
+ this.numberOfRunningGraphs = numberOfRunningGraphs;
2084
+ const snapshot = this.resolveRuntimeStatusSnapshot(
2085
+ numberOfRunningGraphs,
2086
+ localInstance.isActive,
2087
+ localInstance.isNonResponsive,
2088
+ localInstance.isBlocked
2089
+ );
2090
+ const reportedAt = (/* @__PURE__ */ new Date()).toISOString();
2091
+ const report = {
2092
+ serviceName: this.serviceName,
2093
+ serviceInstanceId: this.serviceInstanceId,
2094
+ serviceAddress: localInstance.address,
2095
+ servicePort: localInstance.port,
2096
+ exposed: localInstance.exposed,
2097
+ isFrontend: localInstance.isFrontend,
2098
+ reportedAt,
2099
+ state: snapshot.state,
2100
+ acceptingWork: snapshot.acceptingWork,
2101
+ numberOfRunningGraphs: snapshot.numberOfRunningGraphs,
2102
+ isActive: snapshot.isActive,
2103
+ isNonResponsive: snapshot.isNonResponsive,
2104
+ isBlocked: snapshot.isBlocked,
2105
+ health: {
2106
+ ...localInstance.health ?? {},
2107
+ runtimeStatus: {
2108
+ state: snapshot.state,
2109
+ acceptingWork: snapshot.acceptingWork,
2110
+ reportedAt
2111
+ }
2112
+ }
2113
+ };
2114
+ this.applyRuntimeStatusReport(report);
2115
+ if (detailLevel !== "full") {
2116
+ delete report.health;
2117
+ }
2118
+ return report;
2119
+ }
2120
+ selectRuntimeStatusReportForTarget(inquiryResult, targetServiceName, targetServiceInstanceId) {
2121
+ const reports = Array.isArray(inquiryResult.runtimeStatusReports) ? inquiryResult.runtimeStatusReports : [];
2122
+ for (const candidate of reports) {
2123
+ const report = this.normalizeRuntimeStatusReport(candidate);
2124
+ if (!report) {
2125
+ continue;
2126
+ }
2127
+ if (report.serviceName === targetServiceName && report.serviceInstanceId === targetServiceInstanceId) {
2128
+ return report;
2129
+ }
2130
+ }
2131
+ return null;
2132
+ }
2133
+ async resolveRuntimeStatusFallbackInquiry(serviceName, serviceInstanceId, options = {}) {
2134
+ const inquiryResult = await CadenzaService.inquire(
2135
+ META_RUNTIME_STATUS_INTENT,
2136
+ {
2137
+ targetServiceName: serviceName,
2138
+ targetServiceInstanceId: serviceInstanceId,
2139
+ detailLevel: options.detailLevel ?? "minimal"
2140
+ },
2141
+ {
2142
+ overallTimeoutMs: options.overallTimeoutMs ?? this.runtimeStatusFallbackTimeoutMs,
2143
+ perResponderTimeoutMs: options.perResponderTimeoutMs ?? Math.max(250, Math.floor(this.runtimeStatusFallbackTimeoutMs * 0.75)),
2144
+ requireComplete: options.requireComplete ?? false
2145
+ }
2146
+ );
2147
+ const report = this.selectRuntimeStatusReportForTarget(
2148
+ inquiryResult,
2149
+ serviceName,
2150
+ serviceInstanceId
2151
+ );
2152
+ if (!report) {
2153
+ throw new Error(
2154
+ `No runtime status report for ${serviceName}/${serviceInstanceId}`
2155
+ );
2156
+ }
2157
+ if (!this.applyRuntimeStatusReport(report)) {
2158
+ throw new Error(
2159
+ `No tracked instance for runtime fallback ${serviceName}/${serviceInstanceId}`
2160
+ );
2161
+ }
2162
+ this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
2163
+ this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
2164
+ return {
2165
+ report,
2166
+ inquiryMeta: inquiryResult.__inquiryMeta ?? {}
2167
+ };
2168
+ }
2169
+ evaluateDependencyReadinessDetail(serviceName, serviceInstanceId, now = Date.now()) {
2170
+ const instance = this.getInstance(serviceName, serviceInstanceId);
2171
+ const missedHeartbeats = this.getHeartbeatMisses(serviceInstanceId, now);
2172
+ const runtimeState = instance ? instance.runtimeState ?? this.resolveRuntimeStatusSnapshot(
2173
+ instance.numberOfRunningGraphs ?? 0,
2174
+ instance.isActive,
2175
+ instance.isNonResponsive,
2176
+ instance.isBlocked
2177
+ ).state : "unavailable";
2178
+ const acceptingWork = instance ? typeof instance.acceptingWork === "boolean" ? instance.acceptingWork : this.resolveRuntimeStatusSnapshot(
2179
+ instance.numberOfRunningGraphs ?? 0,
2180
+ instance.isActive,
2181
+ instance.isNonResponsive,
2182
+ instance.isBlocked
2183
+ ).acceptingWork : false;
2184
+ const evaluation = evaluateDependencyReadiness({
2185
+ exists: Boolean(instance),
2186
+ runtimeState,
2187
+ acceptingWork,
2188
+ missedHeartbeats,
2189
+ missThreshold: this.runtimeStatusMissThreshold
2190
+ });
2191
+ const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId);
2192
+ return {
2193
+ serviceName,
2194
+ serviceInstanceId,
2195
+ dependencyState: evaluation.state,
2196
+ runtimeState,
2197
+ acceptingWork,
2198
+ missedHeartbeats,
2199
+ stale: evaluation.stale,
2200
+ blocked: evaluation.blocked,
2201
+ reason: evaluation.reason,
2202
+ lastHeartbeatAt: lastHeartbeat ? new Date(lastHeartbeat).toISOString() : null,
2203
+ reportedAt: instance?.reportedAt ?? null
2204
+ };
2205
+ }
2206
+ async buildLocalReadinessReport(options = {}) {
2207
+ const localRuntime = this.buildLocalRuntimeStatusReport("minimal");
2208
+ if (!localRuntime) {
2209
+ return null;
2210
+ }
2211
+ const detailLevel = options.detailLevel ?? "minimal";
2212
+ const includeDependencies = options.includeDependencies ?? detailLevel === "full";
2213
+ const refreshStaleDependencies = options.refreshStaleDependencies ?? true;
2214
+ const dependencyPairs = Array.from(this.readinessDependeesByService.entries()).flatMap(
2215
+ ([serviceName, instanceIds]) => Array.from(instanceIds).map((serviceInstanceId) => ({
2216
+ serviceName,
2217
+ serviceInstanceId
2218
+ }))
2219
+ ).sort((left, right) => {
2220
+ if (left.serviceName !== right.serviceName) {
2221
+ return left.serviceName.localeCompare(right.serviceName);
2222
+ }
2223
+ return left.serviceInstanceId.localeCompare(right.serviceInstanceId);
2224
+ });
2225
+ if (refreshStaleDependencies) {
2226
+ for (const dependency of dependencyPairs) {
2227
+ const misses = this.getHeartbeatMisses(dependency.serviceInstanceId);
2228
+ if (misses < this.runtimeStatusMissThreshold) {
2229
+ continue;
2230
+ }
2231
+ if (this.runtimeStatusFallbackInFlightByInstance.has(
2232
+ dependency.serviceInstanceId
2233
+ )) {
2234
+ continue;
2235
+ }
2236
+ this.runtimeStatusFallbackInFlightByInstance.add(
2237
+ dependency.serviceInstanceId
2238
+ );
2239
+ try {
2240
+ await this.resolveRuntimeStatusFallbackInquiry(
2241
+ dependency.serviceName,
2242
+ dependency.serviceInstanceId
2243
+ );
2244
+ } catch (error) {
2245
+ CadenzaService.log(
2246
+ "Readiness dependency fallback failed.",
2247
+ {
2248
+ serviceName: dependency.serviceName,
2249
+ serviceInstanceId: dependency.serviceInstanceId,
2250
+ error: error instanceof Error ? error.message : String(error)
2251
+ },
2252
+ "warning"
2253
+ );
2254
+ } finally {
2255
+ this.runtimeStatusFallbackInFlightByInstance.delete(
2256
+ dependency.serviceInstanceId
2257
+ );
2258
+ }
2259
+ }
2260
+ }
2261
+ const now = Date.now();
2262
+ const dependencyDetails = dependencyPairs.map(
2263
+ (dependency) => this.evaluateDependencyReadinessDetail(
2264
+ dependency.serviceName,
2265
+ dependency.serviceInstanceId,
2266
+ now
2267
+ )
2268
+ );
2269
+ const dependencySummary = summarizeDependencyReadiness(
2270
+ dependencyDetails.map((detail) => ({
2271
+ state: detail.dependencyState,
2272
+ stale: detail.stale,
2273
+ blocked: detail.blocked,
2274
+ reason: detail.reason
2275
+ }))
2276
+ );
2277
+ const readinessState = resolveServiceReadinessState(
2278
+ localRuntime.state,
2279
+ localRuntime.acceptingWork,
2280
+ dependencySummary
2281
+ );
2282
+ return {
2283
+ serviceName: localRuntime.serviceName,
2284
+ serviceInstanceId: localRuntime.serviceInstanceId,
2285
+ reportedAt: new Date(now).toISOString(),
2286
+ readinessState,
2287
+ runtimeState: localRuntime.state,
2288
+ acceptingWork: localRuntime.acceptingWork,
2289
+ dependencySummary,
2290
+ ...includeDependencies ? { dependencies: dependencyDetails } : {}
2291
+ };
2292
+ }
1195
2293
  reset() {
1196
2294
  this.instances.clear();
1197
2295
  this.deputies.clear();
@@ -1199,6 +2297,17 @@ var ServiceRegistry = class _ServiceRegistry {
1199
2297
  this.remoteIntents.clear();
1200
2298
  this.remoteIntentDeputiesByKey.clear();
1201
2299
  this.remoteIntentDeputiesByTask.clear();
2300
+ this.dependeesByService.clear();
2301
+ this.dependeeByInstance.clear();
2302
+ this.readinessDependeesByService.clear();
2303
+ this.readinessDependeeByInstance.clear();
2304
+ this.lastHeartbeatAtByInstance.clear();
2305
+ this.missedHeartbeatsByInstance.clear();
2306
+ this.runtimeStatusFallbackInFlightByInstance.clear();
2307
+ this.activeRoutineExecutionIds.clear();
2308
+ this.numberOfRunningGraphs = 0;
2309
+ this.runtimeStatusHeartbeatStarted = false;
2310
+ this.lastRuntimeStatusSnapshot = null;
1202
2311
  }
1203
2312
  };
1204
2313