@cadenza.io/service 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +69 -29
- package/dist/index.d.ts +69 -29
- package/dist/index.js +1610 -643
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1613 -644
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -30,6 +30,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/index.ts
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
|
+
Actor: () => import_core4.Actor,
|
|
33
34
|
DatabaseTask: () => DatabaseTask,
|
|
34
35
|
DebounceTask: () => import_core4.DebounceTask,
|
|
35
36
|
DeputyTask: () => DeputyTask,
|
|
@@ -299,6 +300,7 @@ var isBrowser = typeof window !== "undefined" && typeof window.document !== "und
|
|
|
299
300
|
var META_INTENT_PREFIX = "meta-";
|
|
300
301
|
var META_RUNTIME_TRANSPORT_DIAGNOSTICS_INTENT = "meta-runtime-transport-diagnostics";
|
|
301
302
|
var META_RUNTIME_STATUS_INTENT = "meta-runtime-status";
|
|
303
|
+
var META_READINESS_INTENT = "meta-readiness";
|
|
302
304
|
function isPlainObject(value) {
|
|
303
305
|
return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
|
|
304
306
|
}
|
|
@@ -362,6 +364,100 @@ function summarizeResponderStatuses(statuses) {
|
|
|
362
364
|
return { responded, failed, timedOut, pending };
|
|
363
365
|
}
|
|
364
366
|
|
|
367
|
+
// src/utils/readiness.ts
|
|
368
|
+
function evaluateDependencyReadiness(input) {
|
|
369
|
+
const missedHeartbeats = Math.max(
|
|
370
|
+
0,
|
|
371
|
+
Math.trunc(Number(input.missedHeartbeats) || 0)
|
|
372
|
+
);
|
|
373
|
+
const stale = missedHeartbeats > 0;
|
|
374
|
+
const timeoutReached = missedHeartbeats >= Math.max(1, input.missThreshold);
|
|
375
|
+
if (!input.exists) {
|
|
376
|
+
return {
|
|
377
|
+
state: "unavailable",
|
|
378
|
+
stale: true,
|
|
379
|
+
blocked: true,
|
|
380
|
+
reason: "missing"
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
if (timeoutReached) {
|
|
384
|
+
return {
|
|
385
|
+
state: "unavailable",
|
|
386
|
+
stale: true,
|
|
387
|
+
blocked: true,
|
|
388
|
+
reason: "heartbeat-timeout"
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
if (input.runtimeState === "unavailable" || !input.acceptingWork) {
|
|
392
|
+
return {
|
|
393
|
+
state: "unavailable",
|
|
394
|
+
stale,
|
|
395
|
+
blocked: true,
|
|
396
|
+
reason: "runtime-unavailable"
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
if (stale) {
|
|
400
|
+
return {
|
|
401
|
+
state: "degraded",
|
|
402
|
+
stale: true,
|
|
403
|
+
blocked: false,
|
|
404
|
+
reason: "heartbeat-stale"
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
if (input.runtimeState === "overloaded") {
|
|
408
|
+
return {
|
|
409
|
+
state: "overloaded",
|
|
410
|
+
stale: false,
|
|
411
|
+
blocked: false,
|
|
412
|
+
reason: "runtime-overloaded"
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
if (input.runtimeState === "degraded") {
|
|
416
|
+
return {
|
|
417
|
+
state: "degraded",
|
|
418
|
+
stale: false,
|
|
419
|
+
blocked: false,
|
|
420
|
+
reason: "runtime-degraded"
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
return {
|
|
424
|
+
state: "ready",
|
|
425
|
+
stale: false,
|
|
426
|
+
blocked: false,
|
|
427
|
+
reason: "runtime-healthy"
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
function summarizeDependencyReadiness(evaluations) {
|
|
431
|
+
const summary = {
|
|
432
|
+
total: evaluations.length,
|
|
433
|
+
ready: 0,
|
|
434
|
+
degraded: 0,
|
|
435
|
+
overloaded: 0,
|
|
436
|
+
unavailable: 0,
|
|
437
|
+
stale: 0
|
|
438
|
+
};
|
|
439
|
+
for (const evaluation of evaluations) {
|
|
440
|
+
if (evaluation.state === "ready") summary.ready++;
|
|
441
|
+
if (evaluation.state === "degraded") summary.degraded++;
|
|
442
|
+
if (evaluation.state === "overloaded") summary.overloaded++;
|
|
443
|
+
if (evaluation.state === "unavailable") summary.unavailable++;
|
|
444
|
+
if (evaluation.stale) summary.stale++;
|
|
445
|
+
}
|
|
446
|
+
return summary;
|
|
447
|
+
}
|
|
448
|
+
function resolveServiceReadinessState(localRuntimeState, localAcceptingWork, dependencySummary) {
|
|
449
|
+
if (localRuntimeState === "unavailable" || !localAcceptingWork) {
|
|
450
|
+
return "blocked";
|
|
451
|
+
}
|
|
452
|
+
if (dependencySummary.unavailable > 0) {
|
|
453
|
+
return "blocked";
|
|
454
|
+
}
|
|
455
|
+
if (dependencySummary.degraded > 0 || dependencySummary.overloaded > 0 || dependencySummary.stale > 0) {
|
|
456
|
+
return "degraded";
|
|
457
|
+
}
|
|
458
|
+
return "ready";
|
|
459
|
+
}
|
|
460
|
+
|
|
365
461
|
// src/utils/runtimeStatus.ts
|
|
366
462
|
function resolveRuntimeStatus(input) {
|
|
367
463
|
const numberOfRunningGraphs = Math.max(
|
|
@@ -443,6 +539,8 @@ var INTERNAL_RUNTIME_STATUS_TASK_NAMES = /* @__PURE__ */ new Set([
|
|
|
443
539
|
"Monitor dependee heartbeat freshness",
|
|
444
540
|
"Resolve runtime status fallback inquiry",
|
|
445
541
|
"Respond runtime status inquiry",
|
|
542
|
+
"Respond readiness inquiry",
|
|
543
|
+
"Collect distributed readiness",
|
|
446
544
|
"Get status"
|
|
447
545
|
]);
|
|
448
546
|
function readPositiveIntegerEnv(name, fallback) {
|
|
@@ -479,6 +577,8 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
479
577
|
this.remoteIntentDeputiesByTask = /* @__PURE__ */ new Map();
|
|
480
578
|
this.dependeesByService = /* @__PURE__ */ new Map();
|
|
481
579
|
this.dependeeByInstance = /* @__PURE__ */ new Map();
|
|
580
|
+
this.readinessDependeesByService = /* @__PURE__ */ new Map();
|
|
581
|
+
this.readinessDependeeByInstance = /* @__PURE__ */ new Map();
|
|
482
582
|
this.lastHeartbeatAtByInstance = /* @__PURE__ */ new Map();
|
|
483
583
|
this.missedHeartbeatsByInstance = /* @__PURE__ */ new Map();
|
|
484
584
|
this.runtimeStatusFallbackInFlightByInstance = /* @__PURE__ */ new Set();
|
|
@@ -594,6 +694,66 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
594
694
|
},
|
|
595
695
|
"Responds to runtime-status inquiries with local service instance status."
|
|
596
696
|
).respondsTo(META_RUNTIME_STATUS_INTENT);
|
|
697
|
+
CadenzaService.defineIntent({
|
|
698
|
+
name: META_READINESS_INTENT,
|
|
699
|
+
description: "Gather service readiness reports derived from local runtime status and required dependees.",
|
|
700
|
+
input: {
|
|
701
|
+
type: "object",
|
|
702
|
+
properties: {
|
|
703
|
+
detailLevel: {
|
|
704
|
+
type: "string",
|
|
705
|
+
constraints: {
|
|
706
|
+
oneOf: ["minimal", "full"]
|
|
707
|
+
}
|
|
708
|
+
},
|
|
709
|
+
includeDependencies: {
|
|
710
|
+
type: "boolean"
|
|
711
|
+
},
|
|
712
|
+
refreshStaleDependencies: {
|
|
713
|
+
type: "boolean"
|
|
714
|
+
},
|
|
715
|
+
targetServiceName: {
|
|
716
|
+
type: "string"
|
|
717
|
+
},
|
|
718
|
+
targetServiceInstanceId: {
|
|
719
|
+
type: "string"
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
},
|
|
723
|
+
output: {
|
|
724
|
+
type: "object",
|
|
725
|
+
properties: {
|
|
726
|
+
readinessReports: {
|
|
727
|
+
type: "array"
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
});
|
|
732
|
+
CadenzaService.createMetaTask(
|
|
733
|
+
"Respond readiness inquiry",
|
|
734
|
+
async (ctx) => {
|
|
735
|
+
const targetServiceName = ctx.targetServiceName;
|
|
736
|
+
const targetServiceInstanceId = ctx.targetServiceInstanceId;
|
|
737
|
+
const report = await this.buildLocalReadinessReport({
|
|
738
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
739
|
+
includeDependencies: ctx.includeDependencies,
|
|
740
|
+
refreshStaleDependencies: ctx.refreshStaleDependencies
|
|
741
|
+
});
|
|
742
|
+
if (!report) {
|
|
743
|
+
return {};
|
|
744
|
+
}
|
|
745
|
+
if (targetServiceName && targetServiceName !== report.serviceName) {
|
|
746
|
+
return {};
|
|
747
|
+
}
|
|
748
|
+
if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
|
|
749
|
+
return {};
|
|
750
|
+
}
|
|
751
|
+
return {
|
|
752
|
+
readinessReports: [report]
|
|
753
|
+
};
|
|
754
|
+
},
|
|
755
|
+
"Responds to distributed readiness inquiries using required dependee health."
|
|
756
|
+
).respondsTo(META_READINESS_INTENT);
|
|
597
757
|
this.handleInstanceUpdateTask = CadenzaService.createMetaTask(
|
|
598
758
|
"Handle Instance Update",
|
|
599
759
|
(ctx, emit) => {
|
|
@@ -710,7 +870,11 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
710
870
|
if (!ctx.serviceName || !ctx.serviceInstanceId) {
|
|
711
871
|
return false;
|
|
712
872
|
}
|
|
713
|
-
this.registerDependee(ctx.serviceName, ctx.serviceInstanceId
|
|
873
|
+
this.registerDependee(ctx.serviceName, ctx.serviceInstanceId, {
|
|
874
|
+
requiredForReadiness: this.shouldRequireReadinessFromCommunicationTypes(
|
|
875
|
+
ctx.communicationTypes
|
|
876
|
+
)
|
|
877
|
+
});
|
|
714
878
|
return true;
|
|
715
879
|
},
|
|
716
880
|
"Tracks remote dependency instances for runtime heartbeat monitoring."
|
|
@@ -1395,36 +1559,20 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1395
1559
|
return false;
|
|
1396
1560
|
}
|
|
1397
1561
|
try {
|
|
1398
|
-
const
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
targetServiceName: serviceName,
|
|
1402
|
-
targetServiceInstanceId: serviceInstanceId,
|
|
1403
|
-
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal"
|
|
1404
|
-
},
|
|
1562
|
+
const { report, inquiryMeta } = await this.resolveRuntimeStatusFallbackInquiry(
|
|
1563
|
+
serviceName,
|
|
1564
|
+
serviceInstanceId,
|
|
1405
1565
|
{
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1566
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
1567
|
+
overallTimeoutMs: ctx.overallTimeoutMs,
|
|
1568
|
+
perResponderTimeoutMs: ctx.perResponderTimeoutMs,
|
|
1569
|
+
requireComplete: ctx.requireComplete
|
|
1409
1570
|
}
|
|
1410
1571
|
);
|
|
1411
|
-
const report = this.selectRuntimeStatusReportForTarget(
|
|
1412
|
-
inquiryResult,
|
|
1413
|
-
serviceName,
|
|
1414
|
-
serviceInstanceId
|
|
1415
|
-
);
|
|
1416
|
-
if (!report) {
|
|
1417
|
-
throw new Error(
|
|
1418
|
-
`No runtime status report for ${serviceName}/${serviceInstanceId}`
|
|
1419
|
-
);
|
|
1420
|
-
}
|
|
1421
|
-
this.applyRuntimeStatusReport(report);
|
|
1422
|
-
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
1423
|
-
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
1424
1572
|
return {
|
|
1425
1573
|
...ctx,
|
|
1426
1574
|
runtimeStatusReport: report,
|
|
1427
|
-
__inquiryMeta:
|
|
1575
|
+
__inquiryMeta: inquiryMeta
|
|
1428
1576
|
};
|
|
1429
1577
|
} catch (error) {
|
|
1430
1578
|
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
@@ -1458,6 +1606,27 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1458
1606
|
},
|
|
1459
1607
|
"Runs runtime-status inquiry fallback for a dependee instance after missed heartbeats."
|
|
1460
1608
|
).doOn("meta.service_registry.runtime_status_fallback_requested").emits("meta.service_registry.runtime_status_fallback_resolved").emitsOnFail("meta.service_registry.runtime_status_fallback_failed");
|
|
1609
|
+
this.collectReadinessTask = CadenzaService.createMetaTask(
|
|
1610
|
+
"Collect distributed readiness",
|
|
1611
|
+
async (ctx) => {
|
|
1612
|
+
const inquiryResult = await CadenzaService.inquire(
|
|
1613
|
+
META_READINESS_INTENT,
|
|
1614
|
+
{
|
|
1615
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
1616
|
+
includeDependencies: ctx.includeDependencies,
|
|
1617
|
+
refreshStaleDependencies: ctx.refreshStaleDependencies,
|
|
1618
|
+
targetServiceName: ctx.targetServiceName,
|
|
1619
|
+
targetServiceInstanceId: ctx.targetServiceInstanceId
|
|
1620
|
+
},
|
|
1621
|
+
ctx.inquiryOptions ?? ctx.__inquiryOptions ?? {}
|
|
1622
|
+
);
|
|
1623
|
+
return {
|
|
1624
|
+
...ctx,
|
|
1625
|
+
...inquiryResult
|
|
1626
|
+
};
|
|
1627
|
+
},
|
|
1628
|
+
"Collects distributed readiness reports from services."
|
|
1629
|
+
).doOn("meta.service_registry.readiness_requested").emits("meta.service_registry.readiness_collected").emitsOnFail("meta.service_registry.readiness_failed");
|
|
1461
1630
|
this.collectTransportDiagnosticsTask = CadenzaService.createMetaTask(
|
|
1462
1631
|
"Collect transport diagnostics",
|
|
1463
1632
|
async (ctx) => {
|
|
@@ -1806,7 +1975,7 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1806
1975
|
}
|
|
1807
1976
|
return this.getInstance(this.serviceName, this.serviceInstanceId);
|
|
1808
1977
|
}
|
|
1809
|
-
registerDependee(serviceName, serviceInstanceId) {
|
|
1978
|
+
registerDependee(serviceName, serviceInstanceId, options = {}) {
|
|
1810
1979
|
if (!serviceName || !serviceInstanceId) {
|
|
1811
1980
|
return;
|
|
1812
1981
|
}
|
|
@@ -1815,6 +1984,13 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1815
1984
|
}
|
|
1816
1985
|
this.dependeesByService.get(serviceName).add(serviceInstanceId);
|
|
1817
1986
|
this.dependeeByInstance.set(serviceInstanceId, serviceName);
|
|
1987
|
+
if (options.requiredForReadiness) {
|
|
1988
|
+
if (!this.readinessDependeesByService.has(serviceName)) {
|
|
1989
|
+
this.readinessDependeesByService.set(serviceName, /* @__PURE__ */ new Set());
|
|
1990
|
+
}
|
|
1991
|
+
this.readinessDependeesByService.get(serviceName).add(serviceInstanceId);
|
|
1992
|
+
this.readinessDependeeByInstance.set(serviceInstanceId, serviceName);
|
|
1993
|
+
}
|
|
1818
1994
|
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
1819
1995
|
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
1820
1996
|
}
|
|
@@ -1827,10 +2003,39 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1827
2003
|
}
|
|
1828
2004
|
}
|
|
1829
2005
|
this.dependeeByInstance.delete(serviceInstanceId);
|
|
2006
|
+
const readinessDependeeServiceName = serviceName ?? this.readinessDependeeByInstance.get(serviceInstanceId);
|
|
2007
|
+
if (readinessDependeeServiceName) {
|
|
2008
|
+
this.readinessDependeesByService.get(readinessDependeeServiceName)?.delete(serviceInstanceId);
|
|
2009
|
+
if (!this.readinessDependeesByService.get(readinessDependeeServiceName)?.size) {
|
|
2010
|
+
this.readinessDependeesByService.delete(readinessDependeeServiceName);
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
this.readinessDependeeByInstance.delete(serviceInstanceId);
|
|
1830
2014
|
this.lastHeartbeatAtByInstance.delete(serviceInstanceId);
|
|
1831
2015
|
this.missedHeartbeatsByInstance.delete(serviceInstanceId);
|
|
1832
2016
|
this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
|
|
1833
2017
|
}
|
|
2018
|
+
getHeartbeatMisses(serviceInstanceId, now = Date.now()) {
|
|
2019
|
+
const observedMisses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
|
|
2020
|
+
const lastHeartbeatAt = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
|
|
2021
|
+
if (lastHeartbeatAt <= 0) {
|
|
2022
|
+
return Math.max(observedMisses, this.runtimeStatusMissThreshold);
|
|
2023
|
+
}
|
|
2024
|
+
const estimatedMisses = Math.max(
|
|
2025
|
+
0,
|
|
2026
|
+
Math.floor((now - lastHeartbeatAt) / this.runtimeStatusHeartbeatIntervalMs)
|
|
2027
|
+
);
|
|
2028
|
+
return Math.max(observedMisses, estimatedMisses);
|
|
2029
|
+
}
|
|
2030
|
+
shouldRequireReadinessFromCommunicationTypes(communicationTypes) {
|
|
2031
|
+
if (!Array.isArray(communicationTypes)) {
|
|
2032
|
+
return false;
|
|
2033
|
+
}
|
|
2034
|
+
return communicationTypes.some((type) => {
|
|
2035
|
+
const normalized = String(type).toLowerCase();
|
|
2036
|
+
return normalized === "delegation" || normalized === "inquiry";
|
|
2037
|
+
});
|
|
2038
|
+
}
|
|
1834
2039
|
resolveRuntimeStatusSnapshot(numberOfRunningGraphs, isActive, isNonResponsive, isBlocked) {
|
|
1835
2040
|
return resolveRuntimeStatus({
|
|
1836
2041
|
numberOfRunningGraphs,
|
|
@@ -1974,6 +2179,166 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1974
2179
|
}
|
|
1975
2180
|
return null;
|
|
1976
2181
|
}
|
|
2182
|
+
async resolveRuntimeStatusFallbackInquiry(serviceName, serviceInstanceId, options = {}) {
|
|
2183
|
+
const inquiryResult = await CadenzaService.inquire(
|
|
2184
|
+
META_RUNTIME_STATUS_INTENT,
|
|
2185
|
+
{
|
|
2186
|
+
targetServiceName: serviceName,
|
|
2187
|
+
targetServiceInstanceId: serviceInstanceId,
|
|
2188
|
+
detailLevel: options.detailLevel ?? "minimal"
|
|
2189
|
+
},
|
|
2190
|
+
{
|
|
2191
|
+
overallTimeoutMs: options.overallTimeoutMs ?? this.runtimeStatusFallbackTimeoutMs,
|
|
2192
|
+
perResponderTimeoutMs: options.perResponderTimeoutMs ?? Math.max(250, Math.floor(this.runtimeStatusFallbackTimeoutMs * 0.75)),
|
|
2193
|
+
requireComplete: options.requireComplete ?? false
|
|
2194
|
+
}
|
|
2195
|
+
);
|
|
2196
|
+
const report = this.selectRuntimeStatusReportForTarget(
|
|
2197
|
+
inquiryResult,
|
|
2198
|
+
serviceName,
|
|
2199
|
+
serviceInstanceId
|
|
2200
|
+
);
|
|
2201
|
+
if (!report) {
|
|
2202
|
+
throw new Error(
|
|
2203
|
+
`No runtime status report for ${serviceName}/${serviceInstanceId}`
|
|
2204
|
+
);
|
|
2205
|
+
}
|
|
2206
|
+
if (!this.applyRuntimeStatusReport(report)) {
|
|
2207
|
+
throw new Error(
|
|
2208
|
+
`No tracked instance for runtime fallback ${serviceName}/${serviceInstanceId}`
|
|
2209
|
+
);
|
|
2210
|
+
}
|
|
2211
|
+
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
2212
|
+
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
2213
|
+
return {
|
|
2214
|
+
report,
|
|
2215
|
+
inquiryMeta: inquiryResult.__inquiryMeta ?? {}
|
|
2216
|
+
};
|
|
2217
|
+
}
|
|
2218
|
+
evaluateDependencyReadinessDetail(serviceName, serviceInstanceId, now = Date.now()) {
|
|
2219
|
+
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
2220
|
+
const missedHeartbeats = this.getHeartbeatMisses(serviceInstanceId, now);
|
|
2221
|
+
const runtimeState = instance ? instance.runtimeState ?? this.resolveRuntimeStatusSnapshot(
|
|
2222
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
2223
|
+
instance.isActive,
|
|
2224
|
+
instance.isNonResponsive,
|
|
2225
|
+
instance.isBlocked
|
|
2226
|
+
).state : "unavailable";
|
|
2227
|
+
const acceptingWork = instance ? typeof instance.acceptingWork === "boolean" ? instance.acceptingWork : this.resolveRuntimeStatusSnapshot(
|
|
2228
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
2229
|
+
instance.isActive,
|
|
2230
|
+
instance.isNonResponsive,
|
|
2231
|
+
instance.isBlocked
|
|
2232
|
+
).acceptingWork : false;
|
|
2233
|
+
const evaluation = evaluateDependencyReadiness({
|
|
2234
|
+
exists: Boolean(instance),
|
|
2235
|
+
runtimeState,
|
|
2236
|
+
acceptingWork,
|
|
2237
|
+
missedHeartbeats,
|
|
2238
|
+
missThreshold: this.runtimeStatusMissThreshold
|
|
2239
|
+
});
|
|
2240
|
+
const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId);
|
|
2241
|
+
return {
|
|
2242
|
+
serviceName,
|
|
2243
|
+
serviceInstanceId,
|
|
2244
|
+
dependencyState: evaluation.state,
|
|
2245
|
+
runtimeState,
|
|
2246
|
+
acceptingWork,
|
|
2247
|
+
missedHeartbeats,
|
|
2248
|
+
stale: evaluation.stale,
|
|
2249
|
+
blocked: evaluation.blocked,
|
|
2250
|
+
reason: evaluation.reason,
|
|
2251
|
+
lastHeartbeatAt: lastHeartbeat ? new Date(lastHeartbeat).toISOString() : null,
|
|
2252
|
+
reportedAt: instance?.reportedAt ?? null
|
|
2253
|
+
};
|
|
2254
|
+
}
|
|
2255
|
+
async buildLocalReadinessReport(options = {}) {
|
|
2256
|
+
const localRuntime = this.buildLocalRuntimeStatusReport("minimal");
|
|
2257
|
+
if (!localRuntime) {
|
|
2258
|
+
return null;
|
|
2259
|
+
}
|
|
2260
|
+
const detailLevel = options.detailLevel ?? "minimal";
|
|
2261
|
+
const includeDependencies = options.includeDependencies ?? detailLevel === "full";
|
|
2262
|
+
const refreshStaleDependencies = options.refreshStaleDependencies ?? true;
|
|
2263
|
+
const dependencyPairs = Array.from(this.readinessDependeesByService.entries()).flatMap(
|
|
2264
|
+
([serviceName, instanceIds]) => Array.from(instanceIds).map((serviceInstanceId) => ({
|
|
2265
|
+
serviceName,
|
|
2266
|
+
serviceInstanceId
|
|
2267
|
+
}))
|
|
2268
|
+
).sort((left, right) => {
|
|
2269
|
+
if (left.serviceName !== right.serviceName) {
|
|
2270
|
+
return left.serviceName.localeCompare(right.serviceName);
|
|
2271
|
+
}
|
|
2272
|
+
return left.serviceInstanceId.localeCompare(right.serviceInstanceId);
|
|
2273
|
+
});
|
|
2274
|
+
if (refreshStaleDependencies) {
|
|
2275
|
+
for (const dependency of dependencyPairs) {
|
|
2276
|
+
const misses = this.getHeartbeatMisses(dependency.serviceInstanceId);
|
|
2277
|
+
if (misses < this.runtimeStatusMissThreshold) {
|
|
2278
|
+
continue;
|
|
2279
|
+
}
|
|
2280
|
+
if (this.runtimeStatusFallbackInFlightByInstance.has(
|
|
2281
|
+
dependency.serviceInstanceId
|
|
2282
|
+
)) {
|
|
2283
|
+
continue;
|
|
2284
|
+
}
|
|
2285
|
+
this.runtimeStatusFallbackInFlightByInstance.add(
|
|
2286
|
+
dependency.serviceInstanceId
|
|
2287
|
+
);
|
|
2288
|
+
try {
|
|
2289
|
+
await this.resolveRuntimeStatusFallbackInquiry(
|
|
2290
|
+
dependency.serviceName,
|
|
2291
|
+
dependency.serviceInstanceId
|
|
2292
|
+
);
|
|
2293
|
+
} catch (error) {
|
|
2294
|
+
CadenzaService.log(
|
|
2295
|
+
"Readiness dependency fallback failed.",
|
|
2296
|
+
{
|
|
2297
|
+
serviceName: dependency.serviceName,
|
|
2298
|
+
serviceInstanceId: dependency.serviceInstanceId,
|
|
2299
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2300
|
+
},
|
|
2301
|
+
"warning"
|
|
2302
|
+
);
|
|
2303
|
+
} finally {
|
|
2304
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(
|
|
2305
|
+
dependency.serviceInstanceId
|
|
2306
|
+
);
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
}
|
|
2310
|
+
const now = Date.now();
|
|
2311
|
+
const dependencyDetails = dependencyPairs.map(
|
|
2312
|
+
(dependency) => this.evaluateDependencyReadinessDetail(
|
|
2313
|
+
dependency.serviceName,
|
|
2314
|
+
dependency.serviceInstanceId,
|
|
2315
|
+
now
|
|
2316
|
+
)
|
|
2317
|
+
);
|
|
2318
|
+
const dependencySummary = summarizeDependencyReadiness(
|
|
2319
|
+
dependencyDetails.map((detail) => ({
|
|
2320
|
+
state: detail.dependencyState,
|
|
2321
|
+
stale: detail.stale,
|
|
2322
|
+
blocked: detail.blocked,
|
|
2323
|
+
reason: detail.reason
|
|
2324
|
+
}))
|
|
2325
|
+
);
|
|
2326
|
+
const readinessState = resolveServiceReadinessState(
|
|
2327
|
+
localRuntime.state,
|
|
2328
|
+
localRuntime.acceptingWork,
|
|
2329
|
+
dependencySummary
|
|
2330
|
+
);
|
|
2331
|
+
return {
|
|
2332
|
+
serviceName: localRuntime.serviceName,
|
|
2333
|
+
serviceInstanceId: localRuntime.serviceInstanceId,
|
|
2334
|
+
reportedAt: new Date(now).toISOString(),
|
|
2335
|
+
readinessState,
|
|
2336
|
+
runtimeState: localRuntime.state,
|
|
2337
|
+
acceptingWork: localRuntime.acceptingWork,
|
|
2338
|
+
dependencySummary,
|
|
2339
|
+
...includeDependencies ? { dependencies: dependencyDetails } : {}
|
|
2340
|
+
};
|
|
2341
|
+
}
|
|
1977
2342
|
reset() {
|
|
1978
2343
|
this.instances.clear();
|
|
1979
2344
|
this.deputies.clear();
|
|
@@ -1983,6 +2348,8 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1983
2348
|
this.remoteIntentDeputiesByTask.clear();
|
|
1984
2349
|
this.dependeesByService.clear();
|
|
1985
2350
|
this.dependeeByInstance.clear();
|
|
2351
|
+
this.readinessDependeesByService.clear();
|
|
2352
|
+
this.readinessDependeeByInstance.clear();
|
|
1986
2353
|
this.lastHeartbeatAtByInstance.clear();
|
|
1987
2354
|
this.missedHeartbeatsByInstance.clear();
|
|
1988
2355
|
this.runtimeStatusFallbackInFlightByInstance.clear();
|
|
@@ -2108,6 +2475,8 @@ var RestController = class _RestController {
|
|
|
2108
2475
|
constructor() {
|
|
2109
2476
|
this.fetchClientDiagnostics = /* @__PURE__ */ new Map();
|
|
2110
2477
|
this.diagnosticsErrorHistoryLimit = 100;
|
|
2478
|
+
this.diagnosticsMaxClientEntries = 500;
|
|
2479
|
+
this.destroyedDiagnosticsTtlMs = 15 * 6e4;
|
|
2111
2480
|
/**
|
|
2112
2481
|
* Fetches data from the given URL with a specified timeout. This function performs
|
|
2113
2482
|
* a fetch request with the ability to cancel the request if it exceeds the provided timeout duration.
|
|
@@ -2761,6 +3130,28 @@ var RestController = class _RestController {
|
|
|
2761
3130
|
if (!this._instance) this._instance = new _RestController();
|
|
2762
3131
|
return this._instance;
|
|
2763
3132
|
}
|
|
3133
|
+
pruneFetchClientDiagnostics(now = Date.now()) {
|
|
3134
|
+
for (const [fetchId, state] of this.fetchClientDiagnostics.entries()) {
|
|
3135
|
+
if (state.destroyed && now - state.updatedAt > this.destroyedDiagnosticsTtlMs) {
|
|
3136
|
+
this.fetchClientDiagnostics.delete(fetchId);
|
|
3137
|
+
}
|
|
3138
|
+
}
|
|
3139
|
+
if (this.fetchClientDiagnostics.size <= this.diagnosticsMaxClientEntries) {
|
|
3140
|
+
return;
|
|
3141
|
+
}
|
|
3142
|
+
const entriesByEvictionPriority = Array.from(
|
|
3143
|
+
this.fetchClientDiagnostics.entries()
|
|
3144
|
+
).sort((left, right) => {
|
|
3145
|
+
if (left[1].destroyed !== right[1].destroyed) {
|
|
3146
|
+
return left[1].destroyed ? -1 : 1;
|
|
3147
|
+
}
|
|
3148
|
+
return left[1].updatedAt - right[1].updatedAt;
|
|
3149
|
+
});
|
|
3150
|
+
while (this.fetchClientDiagnostics.size > this.diagnosticsMaxClientEntries && entriesByEvictionPriority.length > 0) {
|
|
3151
|
+
const [fetchId] = entriesByEvictionPriority.shift();
|
|
3152
|
+
this.fetchClientDiagnostics.delete(fetchId);
|
|
3153
|
+
}
|
|
3154
|
+
}
|
|
2764
3155
|
resolveTransportDiagnosticsOptions(ctx) {
|
|
2765
3156
|
const detailLevel = ctx.detailLevel === "full" ? "full" : "summary";
|
|
2766
3157
|
const includeErrorHistory = Boolean(ctx.includeErrorHistory);
|
|
@@ -2773,6 +3164,8 @@ var RestController = class _RestController {
|
|
|
2773
3164
|
};
|
|
2774
3165
|
}
|
|
2775
3166
|
ensureFetchClientDiagnostics(fetchId, serviceName, url) {
|
|
3167
|
+
const now = Date.now();
|
|
3168
|
+
this.pruneFetchClientDiagnostics(now);
|
|
2776
3169
|
let state = this.fetchClientDiagnostics.get(fetchId);
|
|
2777
3170
|
if (!state) {
|
|
2778
3171
|
state = {
|
|
@@ -2792,13 +3185,14 @@ var RestController = class _RestController {
|
|
|
2792
3185
|
signalFailures: 0,
|
|
2793
3186
|
statusChecks: 0,
|
|
2794
3187
|
statusFailures: 0,
|
|
2795
|
-
updatedAt:
|
|
3188
|
+
updatedAt: now
|
|
2796
3189
|
};
|
|
2797
3190
|
this.fetchClientDiagnostics.set(fetchId, state);
|
|
2798
3191
|
} else {
|
|
2799
3192
|
state.serviceName = serviceName;
|
|
2800
3193
|
state.url = url;
|
|
2801
3194
|
}
|
|
3195
|
+
this.pruneFetchClientDiagnostics(now);
|
|
2802
3196
|
return state;
|
|
2803
3197
|
}
|
|
2804
3198
|
getErrorMessage(error) {
|
|
@@ -2830,6 +3224,7 @@ var RestController = class _RestController {
|
|
|
2830
3224
|
}
|
|
2831
3225
|
}
|
|
2832
3226
|
collectFetchTransportDiagnostics(ctx) {
|
|
3227
|
+
this.pruneFetchClientDiagnostics();
|
|
2833
3228
|
const { detailLevel, includeErrorHistory, errorHistoryLimit } = this.resolveTransportDiagnosticsOptions(ctx);
|
|
2834
3229
|
const serviceName = CadenzaService.serviceRegistry.serviceName ?? "UnknownService";
|
|
2835
3230
|
const states = Array.from(this.fetchClientDiagnostics.values()).sort(
|
|
@@ -2946,55 +3341,225 @@ var waitForSocketConnection = async (socket, timeoutMs, createError) => {
|
|
|
2946
3341
|
|
|
2947
3342
|
// src/network/SocketController.ts
|
|
2948
3343
|
var SocketController = class _SocketController {
|
|
2949
|
-
/**
|
|
2950
|
-
* Constructs the `SocketServer`, setting up a WebSocket server with specific configurations,
|
|
2951
|
-
* including connection state recovery, rate limiting, CORS handling, and custom event handling.
|
|
2952
|
-
* This class sets up the communication infrastructure for scalable, resilient, and secure WebSocket-based interactions
|
|
2953
|
-
* using metadata-driven task execution with `Cadenza`.
|
|
2954
|
-
*
|
|
2955
|
-
* It provides support for:
|
|
2956
|
-
* - Origin-based access control for connections.
|
|
2957
|
-
* - Optional payload sanitization.
|
|
2958
|
-
* - Configurable rate limiting and behavior on limit breaches (soft/hard disconnects).
|
|
2959
|
-
* - Event handlers for connection, handshake, delegation, signaling, status checks, and disconnection.
|
|
2960
|
-
*
|
|
2961
|
-
* The server can handle both internal and external interactions depending on the provided configurations,
|
|
2962
|
-
* and integrates directly with Cadenza's task workflow engine.
|
|
2963
|
-
*
|
|
2964
|
-
* Initializes the `SocketServer` to be ready for WebSocket communication.
|
|
2965
|
-
*/
|
|
2966
3344
|
constructor() {
|
|
2967
|
-
this.socketClientDiagnostics = /* @__PURE__ */ new Map();
|
|
2968
3345
|
this.diagnosticsErrorHistoryLimit = 100;
|
|
3346
|
+
this.diagnosticsMaxClientEntries = 500;
|
|
3347
|
+
this.destroyedDiagnosticsTtlMs = 15 * 6e4;
|
|
3348
|
+
this.socketServerDefaultKey = "socket-server-default";
|
|
3349
|
+
this.socketServerActor = CadenzaService.createActor(
|
|
3350
|
+
{
|
|
3351
|
+
name: "SocketServerActor",
|
|
3352
|
+
description: "Holds durable socket server session state and runtime socket server handle",
|
|
3353
|
+
defaultKey: this.socketServerDefaultKey,
|
|
3354
|
+
keyResolver: (input) => this.resolveSocketServerKey(input),
|
|
3355
|
+
loadPolicy: "lazy",
|
|
3356
|
+
writeContract: "overwrite",
|
|
3357
|
+
initState: this.createInitialSocketServerSessionState(
|
|
3358
|
+
this.socketServerDefaultKey
|
|
3359
|
+
)
|
|
3360
|
+
},
|
|
3361
|
+
{ isMeta: true }
|
|
3362
|
+
);
|
|
3363
|
+
this.socketClientActor = CadenzaService.createActor(
|
|
3364
|
+
{
|
|
3365
|
+
name: "SocketClientActor",
|
|
3366
|
+
description: "Holds durable socket client session state and runtime socket connection handles",
|
|
3367
|
+
defaultKey: "socket-client-default",
|
|
3368
|
+
keyResolver: (input) => this.resolveSocketClientFetchId(input),
|
|
3369
|
+
loadPolicy: "lazy",
|
|
3370
|
+
writeContract: "overwrite",
|
|
3371
|
+
initState: this.createInitialSocketClientSessionState()
|
|
3372
|
+
},
|
|
3373
|
+
{ isMeta: true }
|
|
3374
|
+
);
|
|
3375
|
+
this.socketClientDiagnosticsActor = CadenzaService.createActor(
|
|
3376
|
+
{
|
|
3377
|
+
name: "SocketClientDiagnosticsActor",
|
|
3378
|
+
description: "Tracks socket client diagnostics snapshots per fetchId for transport observability",
|
|
3379
|
+
defaultKey: "socket-client-diagnostics",
|
|
3380
|
+
loadPolicy: "eager",
|
|
3381
|
+
writeContract: "overwrite",
|
|
3382
|
+
initState: {
|
|
3383
|
+
entries: {}
|
|
3384
|
+
}
|
|
3385
|
+
},
|
|
3386
|
+
{ isMeta: true }
|
|
3387
|
+
);
|
|
3388
|
+
this.registerDiagnosticsTasks();
|
|
3389
|
+
this.registerSocketServerTasks();
|
|
3390
|
+
this.registerSocketClientTasks();
|
|
2969
3391
|
CadenzaService.createMetaTask(
|
|
2970
3392
|
"Collect socket transport diagnostics",
|
|
2971
|
-
|
|
3393
|
+
this.socketClientDiagnosticsActor.task(
|
|
3394
|
+
({ state, input }) => this.collectSocketTransportDiagnostics(input, state.entries),
|
|
3395
|
+
{ mode: "read" }
|
|
3396
|
+
),
|
|
2972
3397
|
"Responds to distributed transport diagnostics inquiries with socket client data."
|
|
2973
3398
|
).respondsTo(META_RUNTIME_TRANSPORT_DIAGNOSTICS_INTENT);
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
3399
|
+
}
|
|
3400
|
+
static get instance() {
|
|
3401
|
+
if (!this._instance) this._instance = new _SocketController();
|
|
3402
|
+
return this._instance;
|
|
3403
|
+
}
|
|
3404
|
+
registerDiagnosticsTasks() {
|
|
3405
|
+
CadenzaService.createThrottledMetaTask(
|
|
3406
|
+
"SocketClientDiagnosticsActor.Upsert",
|
|
3407
|
+
this.socketClientDiagnosticsActor.task(
|
|
3408
|
+
({ state, input, setState }) => {
|
|
3409
|
+
const fetchId = String(input.fetchId ?? "").trim();
|
|
3410
|
+
if (!fetchId) {
|
|
2979
3411
|
return;
|
|
2980
3412
|
}
|
|
2981
|
-
const
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
3413
|
+
const now = Date.now();
|
|
3414
|
+
const entries = { ...state.entries };
|
|
3415
|
+
const existing = entries[fetchId];
|
|
3416
|
+
const base = existing ? {
|
|
3417
|
+
...existing,
|
|
3418
|
+
errorHistory: [...existing.errorHistory]
|
|
3419
|
+
} : {
|
|
3420
|
+
fetchId,
|
|
3421
|
+
serviceName: String(input.serviceName ?? ""),
|
|
3422
|
+
url: String(input.url ?? ""),
|
|
3423
|
+
socketId: null,
|
|
3424
|
+
connected: false,
|
|
3425
|
+
handshake: false,
|
|
3426
|
+
reconnectAttempts: 0,
|
|
3427
|
+
connectErrors: 0,
|
|
3428
|
+
reconnectErrors: 0,
|
|
3429
|
+
socketErrors: 0,
|
|
3430
|
+
pendingDelegations: 0,
|
|
3431
|
+
pendingTimers: 0,
|
|
3432
|
+
destroyed: false,
|
|
3433
|
+
lastHandshakeAt: null,
|
|
3434
|
+
lastHandshakeError: null,
|
|
3435
|
+
lastDisconnectAt: null,
|
|
3436
|
+
lastError: null,
|
|
3437
|
+
lastErrorAt: 0,
|
|
3438
|
+
errorHistory: [],
|
|
3439
|
+
updatedAt: now
|
|
3440
|
+
};
|
|
3441
|
+
if (input.serviceName !== void 0) {
|
|
3442
|
+
base.serviceName = String(input.serviceName);
|
|
3443
|
+
}
|
|
3444
|
+
if (input.url !== void 0) {
|
|
3445
|
+
base.url = String(input.url);
|
|
3446
|
+
}
|
|
3447
|
+
const patch = input.patch && typeof input.patch === "object" ? input.patch : {};
|
|
3448
|
+
Object.assign(base, patch);
|
|
3449
|
+
base.fetchId = fetchId;
|
|
3450
|
+
base.updatedAt = now;
|
|
3451
|
+
const errorMessage = input.error !== void 0 ? this.getErrorMessage(input.error) : void 0;
|
|
3452
|
+
if (errorMessage) {
|
|
3453
|
+
base.lastError = errorMessage;
|
|
3454
|
+
base.lastErrorAt = now;
|
|
3455
|
+
base.errorHistory.push({
|
|
3456
|
+
at: new Date(now).toISOString(),
|
|
3457
|
+
message: errorMessage
|
|
3458
|
+
});
|
|
3459
|
+
if (base.errorHistory.length > this.diagnosticsErrorHistoryLimit) {
|
|
3460
|
+
base.errorHistory.splice(
|
|
3461
|
+
0,
|
|
3462
|
+
base.errorHistory.length - this.diagnosticsErrorHistoryLimit
|
|
3463
|
+
);
|
|
2991
3464
|
}
|
|
3465
|
+
}
|
|
3466
|
+
entries[fetchId] = base;
|
|
3467
|
+
this.pruneDiagnosticsEntries(entries, now);
|
|
3468
|
+
setState({ entries });
|
|
3469
|
+
},
|
|
3470
|
+
{ mode: "write" }
|
|
3471
|
+
),
|
|
3472
|
+
(context) => String(context?.fetchId ?? "default"),
|
|
3473
|
+
"Upserts socket client diagnostics in actor state."
|
|
3474
|
+
).doOn("meta.socket_client.diagnostics_upsert_requested");
|
|
3475
|
+
}
|
|
3476
|
+
registerSocketServerTasks() {
|
|
3477
|
+
CadenzaService.createThrottledMetaTask(
|
|
3478
|
+
"SocketServerActor.PatchSession",
|
|
3479
|
+
this.socketServerActor.task(
|
|
3480
|
+
({ state, input, setState }) => {
|
|
3481
|
+
const patch = input.patch && typeof input.patch === "object" ? input.patch : {};
|
|
3482
|
+
setState({
|
|
3483
|
+
...state,
|
|
3484
|
+
...patch,
|
|
3485
|
+
updatedAt: Date.now()
|
|
3486
|
+
});
|
|
3487
|
+
},
|
|
3488
|
+
{ mode: "write" }
|
|
3489
|
+
),
|
|
3490
|
+
(context) => String(context?.serverKey ?? this.socketServerDefaultKey),
|
|
3491
|
+
"Applies partial durable session updates for socket server actor."
|
|
3492
|
+
).doOn("meta.socket_server.session_patch_requested");
|
|
3493
|
+
CadenzaService.createMetaTask(
|
|
3494
|
+
"SocketServerActor.ClearRuntime",
|
|
3495
|
+
this.socketServerActor.task(
|
|
3496
|
+
({ setRuntimeState }) => {
|
|
3497
|
+
setRuntimeState(null);
|
|
3498
|
+
},
|
|
3499
|
+
{ mode: "write" }
|
|
3500
|
+
),
|
|
3501
|
+
"Clears socket server runtime handle after shutdown."
|
|
3502
|
+
).doOn("meta.socket_server.runtime_clear_requested");
|
|
3503
|
+
const setupSocketServerTask = CadenzaService.createMetaTask(
|
|
3504
|
+
"Setup SocketServer",
|
|
3505
|
+
this.socketServerActor.task(
|
|
3506
|
+
({ state, runtimeState, input, actor, setState, setRuntimeState, emit }) => {
|
|
3507
|
+
const serverKey = this.resolveSocketServerKey(input) ?? actor.key ?? this.socketServerDefaultKey;
|
|
3508
|
+
const shouldUseSocket = Boolean(input.__useSocket);
|
|
3509
|
+
if (!shouldUseSocket) {
|
|
3510
|
+
this.destroySocketServerRuntimeHandle(runtimeState);
|
|
3511
|
+
setRuntimeState(null);
|
|
3512
|
+
setState({
|
|
3513
|
+
...state,
|
|
3514
|
+
serverKey,
|
|
3515
|
+
useSocket: false,
|
|
3516
|
+
status: "inactive",
|
|
3517
|
+
connectionCount: 0,
|
|
3518
|
+
lastShutdownAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3519
|
+
updatedAt: Date.now()
|
|
3520
|
+
});
|
|
3521
|
+
return;
|
|
3522
|
+
}
|
|
3523
|
+
let runtimeHandle = runtimeState;
|
|
3524
|
+
if (!runtimeHandle) {
|
|
3525
|
+
runtimeHandle = this.createSocketServerRuntimeHandleFromContext(input);
|
|
3526
|
+
setRuntimeState(runtimeHandle);
|
|
3527
|
+
}
|
|
3528
|
+
const profile = String(input.__securityProfile ?? state.securityProfile ?? "medium");
|
|
3529
|
+
const networkType = String(input.__networkType ?? state.networkType ?? "internal");
|
|
3530
|
+
const schedulePatch = (patch) => {
|
|
3531
|
+
CadenzaService.emit("meta.socket_server.session_patch_requested", {
|
|
3532
|
+
serverKey,
|
|
3533
|
+
patch
|
|
3534
|
+
});
|
|
3535
|
+
};
|
|
3536
|
+
if (runtimeHandle.initialized) {
|
|
3537
|
+
schedulePatch({
|
|
3538
|
+
status: "active",
|
|
3539
|
+
useSocket: true,
|
|
3540
|
+
securityProfile: profile,
|
|
3541
|
+
networkType,
|
|
3542
|
+
connectionCount: runtimeHandle.connectedSocketIds.size,
|
|
3543
|
+
lastStartedAt: state.lastStartedAt ?? (/* @__PURE__ */ new Date()).toISOString()
|
|
3544
|
+
});
|
|
3545
|
+
return;
|
|
3546
|
+
}
|
|
3547
|
+
const server = runtimeHandle.server;
|
|
3548
|
+
runtimeHandle.initialized = true;
|
|
3549
|
+
setState({
|
|
3550
|
+
...state,
|
|
3551
|
+
serverKey,
|
|
3552
|
+
useSocket: true,
|
|
3553
|
+
status: "active",
|
|
3554
|
+
securityProfile: profile,
|
|
3555
|
+
networkType,
|
|
3556
|
+
connectionCount: runtimeHandle.connectedSocketIds.size,
|
|
3557
|
+
lastStartedAt: state.lastStartedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
3558
|
+
updatedAt: Date.now()
|
|
2992
3559
|
});
|
|
2993
|
-
const profile = ctx.__securityProfile ?? "medium";
|
|
2994
3560
|
server.use((socket, next) => {
|
|
2995
3561
|
const origin = socket?.handshake?.headers?.origin;
|
|
2996
3562
|
const allowedOrigins = ["*"];
|
|
2997
|
-
const networkType = ctx.__networkType ?? "internal";
|
|
2998
3563
|
let effectiveOrigin = origin || "unknown";
|
|
2999
3564
|
if (networkType === "internal") effectiveOrigin = "internal";
|
|
3000
3565
|
if (profile !== "low" && !allowedOrigins.includes(effectiveOrigin) && !allowedOrigins.includes("*")) {
|
|
@@ -3005,7 +3570,9 @@ var SocketController = class _SocketController {
|
|
|
3005
3570
|
medium: { points: 1e4, duration: 10 },
|
|
3006
3571
|
high: { points: 1e3, duration: 60, blockDuration: 300 }
|
|
3007
3572
|
};
|
|
3008
|
-
const limiter = new import_rate_limiter_flexible2.RateLimiterMemory(
|
|
3573
|
+
const limiter = new import_rate_limiter_flexible2.RateLimiterMemory(
|
|
3574
|
+
limiterOptions[profile] ?? limiterOptions.medium
|
|
3575
|
+
);
|
|
3009
3576
|
const clientKey = socket?.handshake?.address || "unknown";
|
|
3010
3577
|
socket.use((packet, packetNext) => {
|
|
3011
3578
|
limiter.consume(clientKey).then(() => packetNext()).catch((rej) => {
|
|
@@ -3040,116 +3607,111 @@ var SocketController = class _SocketController {
|
|
|
3040
3607
|
});
|
|
3041
3608
|
next();
|
|
3042
3609
|
});
|
|
3043
|
-
if (!server) {
|
|
3044
|
-
CadenzaService.log("Socket setup error: No server", {}, "error");
|
|
3045
|
-
return { ...ctx, __error: "No server", errored: true };
|
|
3046
|
-
}
|
|
3047
3610
|
server.on("connection", (ws) => {
|
|
3611
|
+
runtimeHandle.connectedSocketIds.add(ws.id);
|
|
3612
|
+
schedulePatch({
|
|
3613
|
+
connectionCount: runtimeHandle.connectedSocketIds.size,
|
|
3614
|
+
lastConnectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3615
|
+
status: "active"
|
|
3616
|
+
});
|
|
3048
3617
|
try {
|
|
3049
|
-
ws.on(
|
|
3050
|
-
"
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
}
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
if (ctx3.__routineExecId) {
|
|
3070
|
-
emit(
|
|
3071
|
-
`meta.socket_client.transmitted:${ctx3.__routineExecId}`,
|
|
3072
|
-
{}
|
|
3073
|
-
);
|
|
3074
|
-
}
|
|
3618
|
+
ws.on("handshake", (ctx, callback) => {
|
|
3619
|
+
CadenzaService.log("SocketServer: New connection", {
|
|
3620
|
+
...ctx,
|
|
3621
|
+
socketId: ws.id
|
|
3622
|
+
});
|
|
3623
|
+
callback({
|
|
3624
|
+
status: "success",
|
|
3625
|
+
serviceName: CadenzaService.serviceRegistry.serviceName
|
|
3626
|
+
});
|
|
3627
|
+
if (ctx.isFrontend) {
|
|
3628
|
+
const fetchId = `browser:${ctx.serviceInstanceId}`;
|
|
3629
|
+
CadenzaService.createMetaTask(
|
|
3630
|
+
`Transmit signal to ${fetchId}`,
|
|
3631
|
+
(c, emitter) => {
|
|
3632
|
+
if (c.__signalName === void 0) {
|
|
3633
|
+
return;
|
|
3634
|
+
}
|
|
3635
|
+
ws.emit("signal", c);
|
|
3636
|
+
if (c.__routineExecId) {
|
|
3637
|
+
emitter(`meta.socket_client.transmitted:${c.__routineExecId}`, {});
|
|
3075
3638
|
}
|
|
3076
|
-
).doOn(
|
|
3077
|
-
`meta.service_registry.selected_instance_for_socket:${fetchId}`
|
|
3078
|
-
).attachSignal("meta.socket_client.transmitted");
|
|
3079
|
-
}
|
|
3080
|
-
CadenzaService.emit("meta.socket.handshake", ctx2);
|
|
3081
|
-
}
|
|
3082
|
-
);
|
|
3083
|
-
ws.on(
|
|
3084
|
-
"delegation",
|
|
3085
|
-
(ctx2, callback) => {
|
|
3086
|
-
const deputyExecId = ctx2.__metadata.__deputyExecId;
|
|
3087
|
-
CadenzaService.createEphemeralMetaTask(
|
|
3088
|
-
"Resolve delegation",
|
|
3089
|
-
(ctx3) => {
|
|
3090
|
-
callback(ctx3);
|
|
3091
|
-
},
|
|
3092
|
-
"Resolves a delegation request using the provided callback from the client (.emitWithAck())",
|
|
3093
|
-
{ register: false }
|
|
3094
|
-
).doOn(`meta.node.graph_completed:${deputyExecId}`).emits(`meta.socket.delegation_resolved:${deputyExecId}`);
|
|
3095
|
-
CadenzaService.createEphemeralMetaTask(
|
|
3096
|
-
"Delegation progress update",
|
|
3097
|
-
(ctx3) => {
|
|
3098
|
-
if (ctx3.__progress !== void 0)
|
|
3099
|
-
ws.emit("delegation_progress", ctx3);
|
|
3100
3639
|
},
|
|
3101
|
-
"
|
|
3102
|
-
|
|
3103
|
-
once: false,
|
|
3104
|
-
destroyCondition: (ctx3) => ctx3.data.progress === 1 || ctx3.data?.progress === void 0,
|
|
3105
|
-
register: false
|
|
3106
|
-
}
|
|
3107
|
-
).doOn(
|
|
3108
|
-
`meta.node.routine_execution_progress:${deputyExecId}`,
|
|
3109
|
-
`meta.node.graph_completed:${deputyExecId}`
|
|
3110
|
-
).emitsOnFail(`meta.socket.progress_failed:${deputyExecId}`);
|
|
3111
|
-
CadenzaService.emit("meta.socket.delegation_requested", {
|
|
3112
|
-
...ctx2,
|
|
3113
|
-
__name: ctx2.__remoteRoutineName
|
|
3114
|
-
});
|
|
3640
|
+
"Transmit frontend bound signal through active websocket."
|
|
3641
|
+
).doOn(`meta.service_registry.selected_instance_for_socket:${fetchId}`).attachSignal("meta.socket_client.transmitted");
|
|
3115
3642
|
}
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3643
|
+
CadenzaService.emit("meta.socket.handshake", ctx);
|
|
3644
|
+
});
|
|
3645
|
+
ws.on("delegation", (ctx, callback) => {
|
|
3646
|
+
const deputyExecId = ctx.__metadata.__deputyExecId;
|
|
3647
|
+
CadenzaService.createEphemeralMetaTask(
|
|
3648
|
+
"Resolve delegation",
|
|
3649
|
+
(delegationCtx) => {
|
|
3650
|
+
callback(delegationCtx);
|
|
3651
|
+
},
|
|
3652
|
+
"Resolves a delegation request using client callback.",
|
|
3653
|
+
{ register: false }
|
|
3654
|
+
).doOn(`meta.node.graph_completed:${deputyExecId}`).emits(`meta.socket.delegation_resolved:${deputyExecId}`);
|
|
3655
|
+
CadenzaService.createEphemeralMetaTask(
|
|
3656
|
+
"Delegation progress update",
|
|
3657
|
+
(progressCtx) => {
|
|
3658
|
+
if (progressCtx.__progress !== void 0) {
|
|
3659
|
+
ws.emit("delegation_progress", progressCtx);
|
|
3660
|
+
}
|
|
3661
|
+
},
|
|
3662
|
+
"Updates delegation progress to client.",
|
|
3663
|
+
{
|
|
3664
|
+
once: false,
|
|
3665
|
+
destroyCondition: (progressCtx) => progressCtx.data.progress === 1 || progressCtx.data?.progress === void 0,
|
|
3666
|
+
register: false
|
|
3137
3667
|
}
|
|
3668
|
+
).doOn(
|
|
3669
|
+
`meta.node.routine_execution_progress:${deputyExecId}`,
|
|
3670
|
+
`meta.node.graph_completed:${deputyExecId}`
|
|
3671
|
+
).emitsOnFail(`meta.socket.progress_failed:${deputyExecId}`);
|
|
3672
|
+
CadenzaService.emit("meta.socket.delegation_requested", {
|
|
3673
|
+
...ctx,
|
|
3674
|
+
__name: ctx.__remoteRoutineName
|
|
3675
|
+
});
|
|
3676
|
+
});
|
|
3677
|
+
ws.on("signal", (ctx, callback) => {
|
|
3678
|
+
if (CadenzaService.signalBroker.listObservedSignals().includes(ctx.__signalName)) {
|
|
3679
|
+
callback({
|
|
3680
|
+
__status: "success",
|
|
3681
|
+
__signalName: ctx.__signalName
|
|
3682
|
+
});
|
|
3683
|
+
CadenzaService.emit(ctx.__signalName, ctx);
|
|
3684
|
+
} else {
|
|
3685
|
+
CadenzaService.log(
|
|
3686
|
+
`No such signal ${ctx.__signalName} on ${ctx.__serviceName}`,
|
|
3687
|
+
"warning"
|
|
3688
|
+
);
|
|
3689
|
+
callback({
|
|
3690
|
+
...ctx,
|
|
3691
|
+
__status: "error",
|
|
3692
|
+
__error: `No such signal: ${ctx.__signalName}`,
|
|
3693
|
+
errored: true
|
|
3694
|
+
});
|
|
3138
3695
|
}
|
|
3139
|
-
);
|
|
3696
|
+
});
|
|
3140
3697
|
ws.on(
|
|
3141
3698
|
"status_check",
|
|
3142
|
-
(
|
|
3699
|
+
(ctx, callback) => {
|
|
3143
3700
|
CadenzaService.createEphemeralMetaTask(
|
|
3144
3701
|
"Resolve status check",
|
|
3145
3702
|
callback,
|
|
3146
3703
|
"Resolves a status check request",
|
|
3147
3704
|
{ register: false }
|
|
3148
3705
|
).doAfter(CadenzaService.serviceRegistry.getStatusTask);
|
|
3149
|
-
CadenzaService.emit("meta.socket.status_check_requested",
|
|
3706
|
+
CadenzaService.emit("meta.socket.status_check_requested", ctx);
|
|
3150
3707
|
}
|
|
3151
3708
|
);
|
|
3152
3709
|
ws.on("disconnect", () => {
|
|
3710
|
+
runtimeHandle.connectedSocketIds.delete(ws.id);
|
|
3711
|
+
schedulePatch({
|
|
3712
|
+
connectionCount: runtimeHandle.connectedSocketIds.size,
|
|
3713
|
+
lastDisconnectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3714
|
+
});
|
|
3153
3715
|
CadenzaService.log(
|
|
3154
3716
|
"Socket client disconnected",
|
|
3155
3717
|
{ socketId: ws.id },
|
|
@@ -3159,514 +3721,888 @@ var SocketController = class _SocketController {
|
|
|
3159
3721
|
__wsId: ws.id
|
|
3160
3722
|
});
|
|
3161
3723
|
});
|
|
3162
|
-
} catch (
|
|
3724
|
+
} catch (error) {
|
|
3163
3725
|
CadenzaService.log(
|
|
3164
3726
|
"SocketServer: Error in socket event",
|
|
3165
|
-
{ error
|
|
3727
|
+
{ error },
|
|
3166
3728
|
"error"
|
|
3167
3729
|
);
|
|
3168
3730
|
}
|
|
3169
3731
|
CadenzaService.emit("meta.socket.connected", { __wsId: ws.id });
|
|
3170
3732
|
});
|
|
3171
|
-
CadenzaService.createMetaTask(
|
|
3172
|
-
|
|
3173
|
-
(
|
|
3733
|
+
runtimeHandle.broadcastStatusTask = CadenzaService.createMetaTask(
|
|
3734
|
+
`Broadcast status ${serverKey}`,
|
|
3735
|
+
(ctx) => server.emit("status_update", ctx),
|
|
3174
3736
|
"Broadcasts the status of the server to all clients"
|
|
3175
3737
|
).doOn("meta.service.updated");
|
|
3176
|
-
CadenzaService.createMetaTask(
|
|
3177
|
-
|
|
3178
|
-
() =>
|
|
3738
|
+
runtimeHandle.shutdownTask = CadenzaService.createMetaTask(
|
|
3739
|
+
`Shutdown SocketServer ${serverKey}`,
|
|
3740
|
+
async () => {
|
|
3741
|
+
this.destroySocketServerRuntimeHandle(runtimeHandle);
|
|
3742
|
+
CadenzaService.emit("meta.socket_server.runtime_clear_requested", {
|
|
3743
|
+
serverKey
|
|
3744
|
+
});
|
|
3745
|
+
CadenzaService.emit("meta.socket_server.session_patch_requested", {
|
|
3746
|
+
serverKey,
|
|
3747
|
+
patch: {
|
|
3748
|
+
useSocket: false,
|
|
3749
|
+
status: "shutdown",
|
|
3750
|
+
connectionCount: 0,
|
|
3751
|
+
lastShutdownAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3752
|
+
}
|
|
3753
|
+
});
|
|
3754
|
+
},
|
|
3179
3755
|
"Shuts down the socket server"
|
|
3180
3756
|
).doOn("meta.socket_server_shutdown_requested").emits("meta.socket.shutdown");
|
|
3181
|
-
return
|
|
3182
|
-
}
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3757
|
+
return true;
|
|
3758
|
+
},
|
|
3759
|
+
{ mode: "write" }
|
|
3760
|
+
),
|
|
3761
|
+
"Initializes socket server runtime through actor state."
|
|
3762
|
+
);
|
|
3763
|
+
setupSocketServerTask.doOn("global.meta.rest.network_configured");
|
|
3764
|
+
}
|
|
3765
|
+
registerSocketClientTasks() {
|
|
3766
|
+
CadenzaService.createThrottledMetaTask(
|
|
3767
|
+
"SocketClientActor.ApplySessionOperation",
|
|
3768
|
+
this.socketClientActor.task(
|
|
3769
|
+
({ state, input, setState }) => {
|
|
3770
|
+
const operation = String(
|
|
3771
|
+
input.operation ?? "transmit"
|
|
3772
|
+
);
|
|
3773
|
+
const patch = input.patch && typeof input.patch === "object" ? input.patch : {};
|
|
3774
|
+
let next = {
|
|
3775
|
+
...state,
|
|
3776
|
+
...patch,
|
|
3777
|
+
communicationTypes: patch.communicationTypes !== void 0 ? this.normalizeCommunicationTypes(patch.communicationTypes) : state.communicationTypes,
|
|
3778
|
+
updatedAt: Date.now()
|
|
3779
|
+
};
|
|
3780
|
+
if (input.serviceName !== void 0) {
|
|
3781
|
+
next.serviceName = String(input.serviceName);
|
|
3782
|
+
}
|
|
3783
|
+
if (input.serviceAddress !== void 0) {
|
|
3784
|
+
next.serviceAddress = String(input.serviceAddress);
|
|
3785
|
+
}
|
|
3786
|
+
if (input.serviceInstanceId !== void 0) {
|
|
3787
|
+
next.serviceInstanceId = String(input.serviceInstanceId);
|
|
3788
|
+
}
|
|
3789
|
+
if (input.protocol !== void 0) {
|
|
3790
|
+
next.protocol = String(input.protocol);
|
|
3791
|
+
}
|
|
3792
|
+
if (input.url !== void 0) {
|
|
3793
|
+
next.url = String(input.url);
|
|
3794
|
+
}
|
|
3795
|
+
if (input.servicePort !== void 0) {
|
|
3796
|
+
next.servicePort = Number(input.servicePort);
|
|
3797
|
+
}
|
|
3798
|
+
if (input.fetchId !== void 0) {
|
|
3799
|
+
next.fetchId = String(input.fetchId);
|
|
3800
|
+
}
|
|
3801
|
+
if (operation === "connect") {
|
|
3802
|
+
next.destroyed = false;
|
|
3803
|
+
} else if (operation === "handshake") {
|
|
3804
|
+
next.destroyed = false;
|
|
3805
|
+
next.connected = patch.connected ?? true;
|
|
3806
|
+
next.handshake = patch.handshake ?? true;
|
|
3807
|
+
} else if (operation === "shutdown") {
|
|
3808
|
+
next.connected = false;
|
|
3809
|
+
next.handshake = false;
|
|
3810
|
+
next.destroyed = true;
|
|
3811
|
+
next.pendingDelegations = 0;
|
|
3812
|
+
next.pendingTimers = 0;
|
|
3813
|
+
}
|
|
3814
|
+
setState(next);
|
|
3815
|
+
return next;
|
|
3816
|
+
},
|
|
3817
|
+
{ mode: "write" }
|
|
3818
|
+
),
|
|
3819
|
+
(context) => String(this.resolveSocketClientFetchId(context ?? {}) ?? "default"),
|
|
3820
|
+
"Applies socket client session operation patch in actor durable state."
|
|
3821
|
+
).doOn("meta.socket_client.session_operation_requested");
|
|
3822
|
+
CadenzaService.createMetaTask(
|
|
3823
|
+
"SocketClientActor.ClearRuntime",
|
|
3824
|
+
this.socketClientActor.task(
|
|
3825
|
+
({ setRuntimeState }) => {
|
|
3826
|
+
setRuntimeState(null);
|
|
3827
|
+
},
|
|
3828
|
+
{ mode: "write" }
|
|
3829
|
+
),
|
|
3830
|
+
"Clears socket client runtime handle."
|
|
3831
|
+
).doOn("meta.socket_client.runtime_clear_requested");
|
|
3186
3832
|
CadenzaService.createMetaTask(
|
|
3187
3833
|
"Connect to socket server",
|
|
3188
|
-
(
|
|
3189
|
-
|
|
3190
|
-
serviceInstanceId
|
|
3191
|
-
communicationTypes
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
fetchId,
|
|
3203
|
-
serviceName,
|
|
3204
|
-
URL
|
|
3205
|
-
);
|
|
3206
|
-
socketDiagnostics.destroyed = false;
|
|
3207
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3208
|
-
let handshake = false;
|
|
3209
|
-
let errorCount = 0;
|
|
3210
|
-
const ERROR_LIMIT = 5;
|
|
3211
|
-
if (CadenzaService.get(`Socket handshake with ${URL}`)) {
|
|
3212
|
-
console.error("Socket client already exists", URL);
|
|
3213
|
-
return;
|
|
3214
|
-
}
|
|
3215
|
-
const pendingDelegationIds = /* @__PURE__ */ new Set();
|
|
3216
|
-
const pendingTimers = /* @__PURE__ */ new Set();
|
|
3217
|
-
const syncPendingCounts = () => {
|
|
3218
|
-
socketDiagnostics.pendingDelegations = pendingDelegationIds.size;
|
|
3219
|
-
socketDiagnostics.pendingTimers = pendingTimers.size;
|
|
3220
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3221
|
-
};
|
|
3222
|
-
let handshakeTask = null;
|
|
3223
|
-
let emitWhenReady = null;
|
|
3224
|
-
let transmitTask = null;
|
|
3225
|
-
let delegateTask = null;
|
|
3226
|
-
let socket = null;
|
|
3227
|
-
socket = (0, import_socket2.io)(URL, {
|
|
3228
|
-
reconnection: true,
|
|
3229
|
-
reconnectionAttempts: 5,
|
|
3230
|
-
reconnectionDelay: 2e3,
|
|
3231
|
-
reconnectionDelayMax: 1e4,
|
|
3232
|
-
randomizationFactor: 0.5,
|
|
3233
|
-
transports: ["websocket"],
|
|
3234
|
-
autoConnect: false
|
|
3235
|
-
});
|
|
3236
|
-
emitWhenReady = (event, data, timeoutMs = 6e4, ack) => {
|
|
3237
|
-
return new Promise((resolve) => {
|
|
3238
|
-
const resolveWithError = (errorMessage, fallbackError) => {
|
|
3239
|
-
resolve({
|
|
3240
|
-
...data,
|
|
3241
|
-
errored: true,
|
|
3242
|
-
__error: errorMessage,
|
|
3243
|
-
error: fallbackError instanceof Error ? fallbackError.message : errorMessage,
|
|
3244
|
-
socketId: socket?.id,
|
|
3834
|
+
this.socketClientActor.task(
|
|
3835
|
+
({ state, runtimeState, input, setState, setRuntimeState, emit }) => {
|
|
3836
|
+
const serviceInstanceId = String(input.serviceInstanceId ?? "");
|
|
3837
|
+
const communicationTypes = this.normalizeCommunicationTypes(
|
|
3838
|
+
input.communicationTypes
|
|
3839
|
+
);
|
|
3840
|
+
const serviceName = String(input.serviceName ?? "");
|
|
3841
|
+
const serviceAddress = String(input.serviceAddress ?? "");
|
|
3842
|
+
const protocol = String(input.protocol ?? "http");
|
|
3843
|
+
const normalizedPort = this.resolveServicePort(protocol, input.servicePort);
|
|
3844
|
+
if (!serviceAddress || !normalizedPort) {
|
|
3845
|
+
CadenzaService.log(
|
|
3846
|
+
"Socket client setup skipped due to missing address/port",
|
|
3847
|
+
{
|
|
3245
3848
|
serviceName,
|
|
3246
|
-
|
|
3849
|
+
serviceAddress,
|
|
3850
|
+
servicePort: input.servicePort,
|
|
3851
|
+
protocol
|
|
3852
|
+
},
|
|
3853
|
+
"warning"
|
|
3854
|
+
);
|
|
3855
|
+
return false;
|
|
3856
|
+
}
|
|
3857
|
+
const socketProtocol = protocol === "https" ? "wss" : "ws";
|
|
3858
|
+
const url = `${socketProtocol}://${serviceAddress}:${normalizedPort}`;
|
|
3859
|
+
const fetchId = `${serviceAddress}_${normalizedPort}`;
|
|
3860
|
+
const applySessionOperation = (operation, patch = {}) => {
|
|
3861
|
+
CadenzaService.emit("meta.socket_client.session_operation_requested", {
|
|
3862
|
+
fetchId,
|
|
3863
|
+
operation,
|
|
3864
|
+
patch,
|
|
3865
|
+
serviceInstanceId,
|
|
3866
|
+
communicationTypes,
|
|
3867
|
+
serviceName,
|
|
3868
|
+
serviceAddress,
|
|
3869
|
+
servicePort: normalizedPort,
|
|
3870
|
+
protocol,
|
|
3871
|
+
url
|
|
3872
|
+
});
|
|
3873
|
+
};
|
|
3874
|
+
const upsertDiagnostics = (patch, error) => {
|
|
3875
|
+
CadenzaService.emit("meta.socket_client.diagnostics_upsert_requested", {
|
|
3876
|
+
fetchId,
|
|
3877
|
+
serviceName,
|
|
3878
|
+
url,
|
|
3879
|
+
patch,
|
|
3880
|
+
error
|
|
3881
|
+
});
|
|
3882
|
+
};
|
|
3883
|
+
setState({
|
|
3884
|
+
...state,
|
|
3885
|
+
fetchId,
|
|
3886
|
+
serviceInstanceId,
|
|
3887
|
+
communicationTypes,
|
|
3888
|
+
serviceName,
|
|
3889
|
+
serviceAddress,
|
|
3890
|
+
servicePort: normalizedPort,
|
|
3891
|
+
protocol,
|
|
3892
|
+
url,
|
|
3893
|
+
destroyed: false,
|
|
3894
|
+
updatedAt: Date.now()
|
|
3895
|
+
});
|
|
3896
|
+
let runtimeHandle = runtimeState;
|
|
3897
|
+
if (!runtimeHandle || runtimeHandle.url !== url) {
|
|
3898
|
+
this.destroySocketClientRuntimeHandle(runtimeHandle);
|
|
3899
|
+
runtimeHandle = this.createSocketClientRuntimeHandle(url);
|
|
3900
|
+
setRuntimeState(runtimeHandle);
|
|
3901
|
+
}
|
|
3902
|
+
upsertDiagnostics({
|
|
3903
|
+
destroyed: false,
|
|
3904
|
+
connected: false,
|
|
3905
|
+
handshake: false,
|
|
3906
|
+
socketId: runtimeHandle.socket.id ?? null
|
|
3907
|
+
});
|
|
3908
|
+
applySessionOperation("connect", {
|
|
3909
|
+
destroyed: false,
|
|
3910
|
+
connected: false,
|
|
3911
|
+
handshake: false,
|
|
3912
|
+
socketId: runtimeHandle.socket.id ?? null,
|
|
3913
|
+
pendingDelegations: runtimeHandle.pendingDelegationIds.size,
|
|
3914
|
+
pendingTimers: runtimeHandle.pendingTimers.size,
|
|
3915
|
+
errorCount: runtimeHandle.errorCount
|
|
3916
|
+
});
|
|
3917
|
+
if (runtimeHandle.initialized) {
|
|
3918
|
+
return true;
|
|
3919
|
+
}
|
|
3920
|
+
runtimeHandle.initialized = true;
|
|
3921
|
+
runtimeHandle.handshake = false;
|
|
3922
|
+
runtimeHandle.errorCount = 0;
|
|
3923
|
+
const syncPendingCounts = () => {
|
|
3924
|
+
const pendingDelegations = runtimeHandle.pendingDelegationIds.size;
|
|
3925
|
+
const pendingTimers = runtimeHandle.pendingTimers.size;
|
|
3926
|
+
upsertDiagnostics({
|
|
3927
|
+
pendingDelegations,
|
|
3928
|
+
pendingTimers
|
|
3929
|
+
});
|
|
3930
|
+
applySessionOperation("delegate", {
|
|
3931
|
+
pendingDelegations,
|
|
3932
|
+
pendingTimers
|
|
3933
|
+
});
|
|
3934
|
+
};
|
|
3935
|
+
runtimeHandle.emitWhenReady = (event, data, timeoutMs = 6e4, ack) => {
|
|
3936
|
+
return new Promise((resolve) => {
|
|
3937
|
+
const parsedTimeout = Number(timeoutMs);
|
|
3938
|
+
const normalizedTimeoutMs = Number.isFinite(parsedTimeout) && parsedTimeout > 0 ? Math.trunc(parsedTimeout) : 6e4;
|
|
3939
|
+
let timer = null;
|
|
3940
|
+
let settled = false;
|
|
3941
|
+
const clearPendingTimer = () => {
|
|
3942
|
+
if (!timer) {
|
|
3943
|
+
return;
|
|
3944
|
+
}
|
|
3945
|
+
clearTimeout(timer);
|
|
3946
|
+
runtimeHandle.pendingTimers.delete(timer);
|
|
3947
|
+
syncPendingCounts();
|
|
3948
|
+
timer = null;
|
|
3949
|
+
};
|
|
3950
|
+
const settle = (response) => {
|
|
3951
|
+
if (settled) {
|
|
3952
|
+
return;
|
|
3953
|
+
}
|
|
3954
|
+
settled = true;
|
|
3955
|
+
clearPendingTimer();
|
|
3956
|
+
if (ack) ack(response);
|
|
3957
|
+
resolve(response);
|
|
3958
|
+
};
|
|
3959
|
+
const resolveWithError = (errorMessage, fallbackError) => {
|
|
3960
|
+
settle({
|
|
3961
|
+
...data,
|
|
3962
|
+
errored: true,
|
|
3963
|
+
__error: errorMessage,
|
|
3964
|
+
error: fallbackError instanceof Error ? fallbackError.message : errorMessage,
|
|
3965
|
+
socketId: runtimeHandle.socket.id,
|
|
3966
|
+
serviceName,
|
|
3967
|
+
url
|
|
3968
|
+
});
|
|
3969
|
+
};
|
|
3970
|
+
const tryEmit = async () => {
|
|
3971
|
+
const waitResult = await waitForSocketConnection(
|
|
3972
|
+
runtimeHandle.socket,
|
|
3973
|
+
normalizedTimeoutMs + 10,
|
|
3974
|
+
(reason, error) => {
|
|
3975
|
+
if (reason === "connect_timeout") {
|
|
3976
|
+
return `Socket connect timed out before '${event}'`;
|
|
3977
|
+
}
|
|
3978
|
+
if (reason === "connect_error") {
|
|
3979
|
+
const errMessage = error instanceof Error ? error.message : String(error);
|
|
3980
|
+
return `Socket connect error before '${event}': ${errMessage}`;
|
|
3981
|
+
}
|
|
3982
|
+
return `Socket disconnected before '${event}'`;
|
|
3983
|
+
}
|
|
3984
|
+
);
|
|
3985
|
+
if (!waitResult.ok) {
|
|
3986
|
+
CadenzaService.log(
|
|
3987
|
+
waitResult.error,
|
|
3988
|
+
{
|
|
3989
|
+
socketId: runtimeHandle.socket.id,
|
|
3990
|
+
serviceName,
|
|
3991
|
+
url,
|
|
3992
|
+
event
|
|
3993
|
+
},
|
|
3994
|
+
"error"
|
|
3995
|
+
);
|
|
3996
|
+
upsertDiagnostics({}, waitResult.error);
|
|
3997
|
+
resolveWithError(waitResult.error);
|
|
3998
|
+
return;
|
|
3999
|
+
}
|
|
4000
|
+
timer = setTimeout(() => {
|
|
4001
|
+
if (settled) {
|
|
4002
|
+
return;
|
|
4003
|
+
}
|
|
4004
|
+
clearPendingTimer();
|
|
4005
|
+
const message = `Socket event '${event}' timed out`;
|
|
4006
|
+
CadenzaService.log(
|
|
4007
|
+
message,
|
|
4008
|
+
{ socketId: runtimeHandle.socket.id, serviceName, url },
|
|
4009
|
+
"error"
|
|
4010
|
+
);
|
|
4011
|
+
upsertDiagnostics(
|
|
4012
|
+
{
|
|
4013
|
+
lastHandshakeError: message
|
|
4014
|
+
},
|
|
4015
|
+
message
|
|
4016
|
+
);
|
|
4017
|
+
applySessionOperation("transmit", {
|
|
4018
|
+
lastHandshakeError: message
|
|
4019
|
+
});
|
|
4020
|
+
resolveWithError(message);
|
|
4021
|
+
}, normalizedTimeoutMs + 10);
|
|
4022
|
+
runtimeHandle.pendingTimers.add(timer);
|
|
4023
|
+
syncPendingCounts();
|
|
4024
|
+
runtimeHandle.socket.timeout(normalizedTimeoutMs).emit(event, data, (err, response) => {
|
|
4025
|
+
if (err) {
|
|
4026
|
+
CadenzaService.log(
|
|
4027
|
+
"Socket timeout.",
|
|
4028
|
+
{
|
|
4029
|
+
event,
|
|
4030
|
+
error: err.message,
|
|
4031
|
+
socketId: runtimeHandle.socket.id,
|
|
4032
|
+
serviceName
|
|
4033
|
+
},
|
|
4034
|
+
"warning"
|
|
4035
|
+
);
|
|
4036
|
+
upsertDiagnostics(
|
|
4037
|
+
{
|
|
4038
|
+
lastHandshakeError: err.message
|
|
4039
|
+
},
|
|
4040
|
+
err
|
|
4041
|
+
);
|
|
4042
|
+
applySessionOperation("transmit", {
|
|
4043
|
+
lastHandshakeError: err.message
|
|
4044
|
+
});
|
|
4045
|
+
response = {
|
|
4046
|
+
__error: `Timeout error: ${err}`,
|
|
4047
|
+
errored: true,
|
|
4048
|
+
...data
|
|
4049
|
+
};
|
|
4050
|
+
}
|
|
4051
|
+
settle(response);
|
|
4052
|
+
});
|
|
4053
|
+
};
|
|
4054
|
+
void tryEmit().catch((error) => {
|
|
4055
|
+
CadenzaService.log(
|
|
4056
|
+
"Socket emit failed unexpectedly",
|
|
4057
|
+
{
|
|
4058
|
+
event,
|
|
4059
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4060
|
+
socketId: runtimeHandle.socket.id,
|
|
4061
|
+
serviceName,
|
|
4062
|
+
url
|
|
4063
|
+
},
|
|
4064
|
+
"error"
|
|
4065
|
+
);
|
|
4066
|
+
const message = `Socket event '${event}' failed`;
|
|
4067
|
+
upsertDiagnostics(
|
|
4068
|
+
{
|
|
4069
|
+
lastHandshakeError: error instanceof Error ? error.message : String(error)
|
|
4070
|
+
},
|
|
4071
|
+
error
|
|
4072
|
+
);
|
|
4073
|
+
applySessionOperation("transmit", {
|
|
4074
|
+
lastHandshakeError: error instanceof Error ? error.message : String(error)
|
|
4075
|
+
});
|
|
4076
|
+
resolveWithError(message, error);
|
|
4077
|
+
});
|
|
4078
|
+
});
|
|
4079
|
+
};
|
|
4080
|
+
const socket = runtimeHandle.socket;
|
|
4081
|
+
socket.on("connect", () => {
|
|
4082
|
+
if (runtimeHandle.handshake) return;
|
|
4083
|
+
upsertDiagnostics({
|
|
4084
|
+
connected: true,
|
|
4085
|
+
destroyed: false,
|
|
4086
|
+
socketId: socket.id ?? null
|
|
4087
|
+
});
|
|
4088
|
+
applySessionOperation("connect", {
|
|
4089
|
+
connected: true,
|
|
4090
|
+
destroyed: false,
|
|
4091
|
+
socketId: socket.id ?? null
|
|
4092
|
+
});
|
|
4093
|
+
CadenzaService.emit(`meta.socket_client.connected:${fetchId}`, input);
|
|
4094
|
+
});
|
|
4095
|
+
socket.on("delegation_progress", (delegationCtx) => {
|
|
4096
|
+
CadenzaService.emit(
|
|
4097
|
+
`meta.socket_client.delegation_progress:${delegationCtx.__metadata.__deputyExecId}`,
|
|
4098
|
+
delegationCtx
|
|
4099
|
+
);
|
|
4100
|
+
});
|
|
4101
|
+
socket.on("signal", (signalCtx) => {
|
|
4102
|
+
if (CadenzaService.signalBroker.listObservedSignals().includes(signalCtx.__signalName)) {
|
|
4103
|
+
CadenzaService.emit(signalCtx.__signalName, signalCtx);
|
|
4104
|
+
}
|
|
4105
|
+
});
|
|
4106
|
+
socket.on("status_update", (status) => {
|
|
4107
|
+
CadenzaService.emit("meta.socket_client.status_received", status);
|
|
4108
|
+
});
|
|
4109
|
+
socket.on("connect_error", (err) => {
|
|
4110
|
+
runtimeHandle.handshake = false;
|
|
4111
|
+
upsertDiagnostics(
|
|
4112
|
+
{
|
|
4113
|
+
connected: false,
|
|
4114
|
+
handshake: false,
|
|
4115
|
+
connectErrors: state.connectErrors + 1,
|
|
4116
|
+
lastHandshakeError: err.message
|
|
4117
|
+
},
|
|
4118
|
+
err
|
|
4119
|
+
);
|
|
4120
|
+
applySessionOperation("connect", {
|
|
4121
|
+
connected: false,
|
|
4122
|
+
handshake: false,
|
|
4123
|
+
connectErrors: state.connectErrors + 1,
|
|
4124
|
+
lastHandshakeError: err.message
|
|
4125
|
+
});
|
|
4126
|
+
CadenzaService.log(
|
|
4127
|
+
"Socket connect error",
|
|
4128
|
+
{
|
|
4129
|
+
error: err.message,
|
|
4130
|
+
serviceName,
|
|
4131
|
+
socketId: socket.id,
|
|
4132
|
+
url
|
|
4133
|
+
},
|
|
4134
|
+
"error"
|
|
4135
|
+
);
|
|
4136
|
+
CadenzaService.emit(`meta.socket_client.connect_error:${fetchId}`, err);
|
|
4137
|
+
});
|
|
4138
|
+
socket.on("reconnect_attempt", (attempt) => {
|
|
4139
|
+
upsertDiagnostics({ reconnectAttempts: attempt });
|
|
4140
|
+
applySessionOperation("connect", {
|
|
4141
|
+
reconnectAttempts: attempt
|
|
4142
|
+
});
|
|
4143
|
+
CadenzaService.log(`Reconnect attempt: ${attempt}`);
|
|
4144
|
+
});
|
|
4145
|
+
socket.on("reconnect", (attempt) => {
|
|
4146
|
+
upsertDiagnostics({ connected: true });
|
|
4147
|
+
applySessionOperation("connect", {
|
|
4148
|
+
connected: true
|
|
4149
|
+
});
|
|
4150
|
+
CadenzaService.log(`Socket reconnected after ${attempt} tries`, {
|
|
4151
|
+
socketId: socket.id,
|
|
4152
|
+
url,
|
|
4153
|
+
serviceName
|
|
4154
|
+
});
|
|
4155
|
+
});
|
|
4156
|
+
socket.on("reconnect_error", (err) => {
|
|
4157
|
+
runtimeHandle.handshake = false;
|
|
4158
|
+
upsertDiagnostics(
|
|
4159
|
+
{
|
|
4160
|
+
connected: false,
|
|
4161
|
+
handshake: false,
|
|
4162
|
+
reconnectErrors: state.reconnectErrors + 1,
|
|
4163
|
+
lastHandshakeError: err.message
|
|
4164
|
+
},
|
|
4165
|
+
err
|
|
4166
|
+
);
|
|
4167
|
+
applySessionOperation("connect", {
|
|
4168
|
+
connected: false,
|
|
4169
|
+
handshake: false,
|
|
4170
|
+
reconnectErrors: state.reconnectErrors + 1,
|
|
4171
|
+
lastHandshakeError: err.message
|
|
4172
|
+
});
|
|
4173
|
+
CadenzaService.log(
|
|
4174
|
+
"Socket reconnect failed.",
|
|
4175
|
+
{ error: err.message, serviceName, url, socketId: socket.id },
|
|
4176
|
+
"warning"
|
|
4177
|
+
);
|
|
4178
|
+
});
|
|
4179
|
+
socket.on("error", (err) => {
|
|
4180
|
+
runtimeHandle.errorCount += 1;
|
|
4181
|
+
upsertDiagnostics(
|
|
4182
|
+
{
|
|
4183
|
+
socketErrors: state.socketErrors + 1,
|
|
4184
|
+
lastHandshakeError: this.getErrorMessage(err)
|
|
4185
|
+
},
|
|
4186
|
+
err
|
|
4187
|
+
);
|
|
4188
|
+
applySessionOperation("transmit", {
|
|
4189
|
+
socketErrors: state.socketErrors + 1,
|
|
4190
|
+
errorCount: runtimeHandle.errorCount,
|
|
4191
|
+
lastHandshakeError: this.getErrorMessage(err)
|
|
4192
|
+
});
|
|
4193
|
+
CadenzaService.log(
|
|
4194
|
+
"Socket error",
|
|
4195
|
+
{ error: err, socketId: socket.id, url, serviceName },
|
|
4196
|
+
"error"
|
|
4197
|
+
);
|
|
4198
|
+
CadenzaService.emit("meta.socket_client.error", err);
|
|
4199
|
+
});
|
|
4200
|
+
socket.on("disconnect", () => {
|
|
4201
|
+
const disconnectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4202
|
+
upsertDiagnostics({
|
|
4203
|
+
connected: false,
|
|
4204
|
+
handshake: false,
|
|
4205
|
+
lastDisconnectAt: disconnectedAt
|
|
4206
|
+
});
|
|
4207
|
+
applySessionOperation("connect", {
|
|
4208
|
+
connected: false,
|
|
4209
|
+
handshake: false,
|
|
4210
|
+
lastDisconnectAt: disconnectedAt
|
|
4211
|
+
});
|
|
4212
|
+
CadenzaService.log(
|
|
4213
|
+
"Socket disconnected.",
|
|
4214
|
+
{ url, serviceName, socketId: socket.id },
|
|
4215
|
+
"warning"
|
|
4216
|
+
);
|
|
4217
|
+
CadenzaService.emit(`meta.socket_client.disconnected:${fetchId}`, {
|
|
4218
|
+
serviceName,
|
|
4219
|
+
serviceAddress,
|
|
4220
|
+
servicePort: normalizedPort
|
|
4221
|
+
});
|
|
4222
|
+
runtimeHandle.handshake = false;
|
|
4223
|
+
});
|
|
4224
|
+
socket.connect();
|
|
4225
|
+
runtimeHandle.handshakeTask = CadenzaService.createMetaTask(
|
|
4226
|
+
`Socket handshake with ${url}`,
|
|
4227
|
+
async (_ctx, emitter) => {
|
|
4228
|
+
if (runtimeHandle.handshake) return;
|
|
4229
|
+
runtimeHandle.handshake = true;
|
|
4230
|
+
upsertDiagnostics({
|
|
4231
|
+
handshake: true
|
|
3247
4232
|
});
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
4233
|
+
applySessionOperation("handshake", {
|
|
4234
|
+
handshake: true
|
|
4235
|
+
});
|
|
4236
|
+
await runtimeHandle.emitWhenReady?.(
|
|
4237
|
+
"handshake",
|
|
4238
|
+
{
|
|
4239
|
+
serviceInstanceId: CadenzaService.serviceRegistry.serviceInstanceId,
|
|
4240
|
+
serviceName: CadenzaService.serviceRegistry.serviceName,
|
|
4241
|
+
isFrontend: isBrowser,
|
|
4242
|
+
__status: "success"
|
|
4243
|
+
},
|
|
4244
|
+
1e4,
|
|
4245
|
+
(result) => {
|
|
4246
|
+
if (result.status === "success") {
|
|
4247
|
+
const handshakeAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4248
|
+
upsertDiagnostics({
|
|
4249
|
+
connected: true,
|
|
4250
|
+
handshake: true,
|
|
4251
|
+
lastHandshakeAt: handshakeAt,
|
|
4252
|
+
lastHandshakeError: null,
|
|
4253
|
+
socketId: socket.id ?? null
|
|
4254
|
+
});
|
|
4255
|
+
applySessionOperation("handshake", {
|
|
4256
|
+
connected: true,
|
|
4257
|
+
handshake: true,
|
|
4258
|
+
lastHandshakeAt: handshakeAt,
|
|
4259
|
+
lastHandshakeError: null,
|
|
4260
|
+
socketId: socket.id ?? null
|
|
4261
|
+
});
|
|
4262
|
+
CadenzaService.log("Socket client connected", {
|
|
4263
|
+
result,
|
|
4264
|
+
serviceName,
|
|
4265
|
+
socketId: socket.id,
|
|
4266
|
+
url
|
|
4267
|
+
});
|
|
4268
|
+
} else {
|
|
4269
|
+
const errorMessage = result?.__error ?? result?.error ?? "Socket handshake failed";
|
|
4270
|
+
upsertDiagnostics(
|
|
4271
|
+
{
|
|
4272
|
+
connected: false,
|
|
4273
|
+
handshake: false,
|
|
4274
|
+
lastHandshakeError: errorMessage
|
|
4275
|
+
},
|
|
4276
|
+
errorMessage
|
|
4277
|
+
);
|
|
4278
|
+
applySessionOperation("handshake", {
|
|
4279
|
+
connected: false,
|
|
4280
|
+
handshake: false,
|
|
4281
|
+
lastHandshakeError: errorMessage
|
|
4282
|
+
});
|
|
4283
|
+
CadenzaService.log(
|
|
4284
|
+
"Socket handshake failed",
|
|
4285
|
+
{ result, serviceName, socketId: socket.id, url },
|
|
4286
|
+
"warning"
|
|
4287
|
+
);
|
|
3261
4288
|
}
|
|
3262
|
-
|
|
4289
|
+
void emitter;
|
|
3263
4290
|
}
|
|
3264
4291
|
);
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
fetchId,
|
|
3273
|
-
serviceName,
|
|
3274
|
-
URL,
|
|
3275
|
-
waitResult.error
|
|
3276
|
-
);
|
|
3277
|
-
resolveWithError(waitResult.error);
|
|
4292
|
+
},
|
|
4293
|
+
"Handshakes with socket server"
|
|
4294
|
+
).doOn(`meta.socket_client.connected:${fetchId}`);
|
|
4295
|
+
runtimeHandle.delegateTask = CadenzaService.createMetaTask(
|
|
4296
|
+
`Delegate flow to Socket service ${url}`,
|
|
4297
|
+
async (delegateCtx, emitter) => {
|
|
4298
|
+
if (delegateCtx.__remoteRoutineName === void 0) {
|
|
3278
4299
|
return;
|
|
3279
4300
|
}
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
timer = null;
|
|
3287
|
-
}
|
|
3288
|
-
CadenzaService.log(
|
|
3289
|
-
`Socket event '${event}' timed out`,
|
|
3290
|
-
{ socketId: socket?.id, serviceName, URL },
|
|
3291
|
-
"error"
|
|
3292
|
-
);
|
|
3293
|
-
this.recordSocketClientError(
|
|
3294
|
-
fetchId,
|
|
3295
|
-
serviceName,
|
|
3296
|
-
URL,
|
|
3297
|
-
`Socket event '${event}' timed out`
|
|
3298
|
-
);
|
|
3299
|
-
resolveWithError(`Socket event '${event}' timed out`);
|
|
3300
|
-
}, timeoutMs + 10);
|
|
3301
|
-
pendingTimers.add(timer);
|
|
4301
|
+
delete delegateCtx.__isSubMeta;
|
|
4302
|
+
delete delegateCtx.__broadcast;
|
|
4303
|
+
const deputyExecId = delegateCtx.__metadata?.__deputyExecId;
|
|
4304
|
+
const requestSentAt = Date.now();
|
|
4305
|
+
if (deputyExecId) {
|
|
4306
|
+
runtimeHandle.pendingDelegationIds.add(deputyExecId);
|
|
3302
4307
|
syncPendingCounts();
|
|
3303
4308
|
}
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
4309
|
+
try {
|
|
4310
|
+
const resultContext = await runtimeHandle.emitWhenReady?.(
|
|
4311
|
+
"delegation",
|
|
4312
|
+
delegateCtx,
|
|
4313
|
+
delegateCtx.__timeout ?? 6e4
|
|
4314
|
+
) ?? {
|
|
4315
|
+
errored: true,
|
|
4316
|
+
__error: "Socket delegation returned no response"
|
|
4317
|
+
};
|
|
4318
|
+
const requestDuration = Date.now() - requestSentAt;
|
|
4319
|
+
const metadata = resultContext.__metadata;
|
|
4320
|
+
delete resultContext.__metadata;
|
|
4321
|
+
if (deputyExecId) {
|
|
4322
|
+
emitter(`meta.socket_client.delegated:${deputyExecId}`, {
|
|
4323
|
+
...resultContext,
|
|
4324
|
+
...metadata,
|
|
4325
|
+
__requestDuration: requestDuration
|
|
4326
|
+
});
|
|
3317
4327
|
}
|
|
3318
|
-
if (
|
|
3319
|
-
|
|
3320
|
-
|
|
4328
|
+
if (resultContext?.errored || resultContext?.failed) {
|
|
4329
|
+
const errorMessage = resultContext?.__error ?? resultContext?.error ?? "Socket delegation failed";
|
|
4330
|
+
upsertDiagnostics(
|
|
3321
4331
|
{
|
|
3322
|
-
|
|
3323
|
-
error: err.message,
|
|
3324
|
-
socketId: socket?.id,
|
|
3325
|
-
serviceName
|
|
4332
|
+
lastHandshakeError: String(errorMessage)
|
|
3326
4333
|
},
|
|
3327
|
-
|
|
3328
|
-
);
|
|
3329
|
-
this.recordSocketClientError(
|
|
3330
|
-
fetchId,
|
|
3331
|
-
serviceName,
|
|
3332
|
-
URL,
|
|
3333
|
-
err
|
|
4334
|
+
errorMessage
|
|
3334
4335
|
);
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
...ctx,
|
|
3339
|
-
...ctx.__metadata
|
|
3340
|
-
};
|
|
4336
|
+
applySessionOperation("delegate", {
|
|
4337
|
+
lastHandshakeError: String(errorMessage)
|
|
4338
|
+
});
|
|
3341
4339
|
}
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
socketDiagnostics.socketId = socket?.id ?? null;
|
|
3354
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3355
|
-
CadenzaService.emit(`meta.socket_client.connected:${fetchId}`, ctx);
|
|
3356
|
-
});
|
|
3357
|
-
socket.on("delegation_progress", (ctx2) => {
|
|
3358
|
-
CadenzaService.emit(
|
|
3359
|
-
`meta.socket_client.delegation_progress:${ctx2.__metadata.__deputyExecId}`,
|
|
3360
|
-
ctx2
|
|
3361
|
-
);
|
|
3362
|
-
});
|
|
3363
|
-
socket.on("signal", (ctx2) => {
|
|
3364
|
-
if (CadenzaService.signalBroker.listObservedSignals().includes(ctx2.__signalName)) {
|
|
3365
|
-
CadenzaService.emit(ctx2.__signalName, ctx2);
|
|
3366
|
-
}
|
|
3367
|
-
});
|
|
3368
|
-
socket.on("status_update", (status) => {
|
|
3369
|
-
CadenzaService.emit("meta.socket_client.status_received", status);
|
|
3370
|
-
});
|
|
3371
|
-
socket.on("connect_error", (err) => {
|
|
3372
|
-
handshake = false;
|
|
3373
|
-
socketDiagnostics.connected = false;
|
|
3374
|
-
socketDiagnostics.handshake = false;
|
|
3375
|
-
socketDiagnostics.connectErrors++;
|
|
3376
|
-
socketDiagnostics.lastHandshakeError = err.message;
|
|
3377
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3378
|
-
this.recordSocketClientError(fetchId, serviceName, URL, err);
|
|
3379
|
-
CadenzaService.log(
|
|
3380
|
-
"Socket connect error",
|
|
3381
|
-
{ error: err.message, serviceName, socketId: socket?.id, URL },
|
|
3382
|
-
"error"
|
|
3383
|
-
);
|
|
3384
|
-
CadenzaService.emit(`meta.socket_client.connect_error:${fetchId}`, err);
|
|
3385
|
-
});
|
|
3386
|
-
socket.on("reconnect_attempt", (attempt) => {
|
|
3387
|
-
socketDiagnostics.reconnectAttempts = Math.max(
|
|
3388
|
-
socketDiagnostics.reconnectAttempts,
|
|
3389
|
-
attempt
|
|
3390
|
-
);
|
|
3391
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3392
|
-
CadenzaService.log(`Reconnect attempt: ${attempt}`);
|
|
3393
|
-
});
|
|
3394
|
-
socket.on("reconnect", (attempt) => {
|
|
3395
|
-
socketDiagnostics.connected = true;
|
|
3396
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3397
|
-
CadenzaService.log(`Socket reconnected after ${attempt} tries`, {
|
|
3398
|
-
socketId: socket?.id,
|
|
3399
|
-
URL,
|
|
3400
|
-
serviceName
|
|
3401
|
-
});
|
|
3402
|
-
});
|
|
3403
|
-
socket.on("reconnect_error", (err) => {
|
|
3404
|
-
handshake = false;
|
|
3405
|
-
socketDiagnostics.connected = false;
|
|
3406
|
-
socketDiagnostics.handshake = false;
|
|
3407
|
-
socketDiagnostics.reconnectErrors++;
|
|
3408
|
-
socketDiagnostics.lastHandshakeError = err.message;
|
|
3409
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3410
|
-
this.recordSocketClientError(fetchId, serviceName, URL, err);
|
|
3411
|
-
CadenzaService.log(
|
|
3412
|
-
"Socket reconnect failed.",
|
|
3413
|
-
{ error: err.message, serviceName, URL, socketId: socket?.id },
|
|
3414
|
-
"warning"
|
|
3415
|
-
);
|
|
3416
|
-
});
|
|
3417
|
-
socket.on("error", (err) => {
|
|
3418
|
-
errorCount++;
|
|
3419
|
-
socketDiagnostics.socketErrors++;
|
|
3420
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3421
|
-
this.recordSocketClientError(fetchId, serviceName, URL, err);
|
|
3422
|
-
CadenzaService.log(
|
|
3423
|
-
"Socket error",
|
|
3424
|
-
{ error: err, socketId: socket?.id, URL, serviceName },
|
|
3425
|
-
"error"
|
|
3426
|
-
);
|
|
3427
|
-
CadenzaService.emit("meta.socket_client.error", err);
|
|
3428
|
-
});
|
|
3429
|
-
socket.on("disconnect", () => {
|
|
3430
|
-
const disconnectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3431
|
-
socketDiagnostics.connected = false;
|
|
3432
|
-
socketDiagnostics.handshake = false;
|
|
3433
|
-
socketDiagnostics.lastDisconnectAt = disconnectedAt;
|
|
3434
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3435
|
-
CadenzaService.log(
|
|
3436
|
-
"Socket disconnected.",
|
|
3437
|
-
{ URL, serviceName, socketId: socket?.id },
|
|
3438
|
-
"warning"
|
|
3439
|
-
);
|
|
3440
|
-
CadenzaService.emit(`meta.socket_client.disconnected:${fetchId}`, {
|
|
3441
|
-
serviceName,
|
|
3442
|
-
serviceAddress,
|
|
3443
|
-
servicePort
|
|
3444
|
-
});
|
|
3445
|
-
handshake = false;
|
|
3446
|
-
});
|
|
3447
|
-
socket.connect();
|
|
3448
|
-
handshakeTask = CadenzaService.createMetaTask(
|
|
3449
|
-
`Socket handshake with ${URL}`,
|
|
3450
|
-
async (ctx2, emit) => {
|
|
3451
|
-
if (handshake) return;
|
|
3452
|
-
handshake = true;
|
|
3453
|
-
socketDiagnostics.handshake = true;
|
|
3454
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3455
|
-
await emitWhenReady?.(
|
|
3456
|
-
"handshake",
|
|
3457
|
-
{
|
|
3458
|
-
serviceInstanceId: CadenzaService.serviceRegistry.serviceInstanceId,
|
|
3459
|
-
serviceName: CadenzaService.serviceRegistry.serviceName,
|
|
3460
|
-
isFrontend: isBrowser,
|
|
3461
|
-
__status: "success"
|
|
3462
|
-
},
|
|
3463
|
-
1e4,
|
|
3464
|
-
(result) => {
|
|
3465
|
-
if (result.status === "success") {
|
|
3466
|
-
socketDiagnostics.connected = true;
|
|
3467
|
-
socketDiagnostics.handshake = true;
|
|
3468
|
-
socketDiagnostics.lastHandshakeAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3469
|
-
socketDiagnostics.lastHandshakeError = null;
|
|
3470
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3471
|
-
CadenzaService.log("Socket client connected", {
|
|
3472
|
-
result,
|
|
3473
|
-
serviceName,
|
|
3474
|
-
socketId: socket?.id,
|
|
3475
|
-
URL
|
|
4340
|
+
return resultContext;
|
|
4341
|
+
} catch (error) {
|
|
4342
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4343
|
+
const failedContext = {
|
|
4344
|
+
errored: true,
|
|
4345
|
+
__error: message
|
|
4346
|
+
};
|
|
4347
|
+
if (deputyExecId) {
|
|
4348
|
+
emitter(`meta.socket_client.delegated:${deputyExecId}`, {
|
|
4349
|
+
...failedContext,
|
|
4350
|
+
__requestDuration: Date.now() - requestSentAt
|
|
3476
4351
|
});
|
|
3477
|
-
} else {
|
|
3478
|
-
socketDiagnostics.connected = false;
|
|
3479
|
-
socketDiagnostics.handshake = false;
|
|
3480
|
-
socketDiagnostics.lastHandshakeError = result?.__error ?? result?.error ?? "Socket handshake failed";
|
|
3481
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3482
|
-
this.recordSocketClientError(
|
|
3483
|
-
fetchId,
|
|
3484
|
-
serviceName,
|
|
3485
|
-
URL,
|
|
3486
|
-
socketDiagnostics.lastHandshakeError
|
|
3487
|
-
);
|
|
3488
|
-
CadenzaService.log(
|
|
3489
|
-
"Socket handshake failed",
|
|
3490
|
-
{ result, serviceName, socketId: socket?.id, URL },
|
|
3491
|
-
"warning"
|
|
3492
|
-
);
|
|
3493
4352
|
}
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
delete ctx2.__broadcast;
|
|
3508
|
-
const requestSentAt = Date.now();
|
|
3509
|
-
pendingDelegationIds.add(ctx2.__metadata.__deputyExecId);
|
|
3510
|
-
syncPendingCounts();
|
|
3511
|
-
emitWhenReady?.(
|
|
3512
|
-
"delegation",
|
|
3513
|
-
ctx2,
|
|
3514
|
-
ctx2.__timeout ?? 6e4,
|
|
3515
|
-
(resultContext) => {
|
|
3516
|
-
const requestDuration = Date.now() - requestSentAt;
|
|
3517
|
-
const metadata = resultContext.__metadata;
|
|
3518
|
-
delete resultContext.__metadata;
|
|
3519
|
-
emit(
|
|
3520
|
-
`meta.socket_client.delegated:${ctx2.__metadata.__deputyExecId}`,
|
|
3521
|
-
{
|
|
3522
|
-
...resultContext,
|
|
3523
|
-
...metadata,
|
|
3524
|
-
__requestDuration: requestDuration
|
|
3525
|
-
}
|
|
3526
|
-
);
|
|
3527
|
-
pendingDelegationIds.delete(ctx2.__metadata.__deputyExecId);
|
|
4353
|
+
upsertDiagnostics(
|
|
4354
|
+
{
|
|
4355
|
+
lastHandshakeError: message
|
|
4356
|
+
},
|
|
4357
|
+
error
|
|
4358
|
+
);
|
|
4359
|
+
applySessionOperation("delegate", {
|
|
4360
|
+
lastHandshakeError: message
|
|
4361
|
+
});
|
|
4362
|
+
return failedContext;
|
|
4363
|
+
} finally {
|
|
4364
|
+
if (deputyExecId) {
|
|
4365
|
+
runtimeHandle.pendingDelegationIds.delete(deputyExecId);
|
|
3528
4366
|
syncPendingCounts();
|
|
3529
|
-
if (resultContext?.errored || resultContext?.failed) {
|
|
3530
|
-
this.recordSocketClientError(
|
|
3531
|
-
fetchId,
|
|
3532
|
-
serviceName,
|
|
3533
|
-
URL,
|
|
3534
|
-
resultContext?.__error ?? resultContext?.error ?? "Socket delegation failed"
|
|
3535
|
-
);
|
|
3536
|
-
}
|
|
3537
|
-
resolve(resultContext);
|
|
3538
4367
|
}
|
|
3539
|
-
|
|
3540
|
-
}
|
|
3541
|
-
|
|
3542
|
-
`
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
4368
|
+
}
|
|
4369
|
+
},
|
|
4370
|
+
`Delegate flow to service ${serviceName} with address ${url}`
|
|
4371
|
+
).doOn(`meta.service_registry.selected_instance_for_socket:${fetchId}`).attachSignal(
|
|
4372
|
+
"meta.socket_client.delegated",
|
|
4373
|
+
"meta.socket_shutdown_requested"
|
|
4374
|
+
);
|
|
4375
|
+
runtimeHandle.transmitTask = CadenzaService.createMetaTask(
|
|
4376
|
+
`Transmit signal to socket server ${url}`,
|
|
4377
|
+
async (signalCtx, emitter) => {
|
|
4378
|
+
if (signalCtx.__signalName === void 0) {
|
|
4379
|
+
return;
|
|
4380
|
+
}
|
|
4381
|
+
delete signalCtx.__broadcast;
|
|
4382
|
+
const response = await runtimeHandle.emitWhenReady?.("signal", signalCtx, 5e3) ?? {
|
|
4383
|
+
errored: true,
|
|
4384
|
+
__error: "Socket signal transmission returned no response"
|
|
4385
|
+
};
|
|
4386
|
+
applySessionOperation("transmit", {});
|
|
4387
|
+
if (signalCtx.__routineExecId) {
|
|
4388
|
+
emitter(`meta.socket_client.transmitted:${signalCtx.__routineExecId}`, {
|
|
4389
|
+
...response
|
|
4390
|
+
});
|
|
4391
|
+
}
|
|
4392
|
+
return response;
|
|
4393
|
+
},
|
|
4394
|
+
`Transmits signal to service ${serviceName} with address ${url}`
|
|
4395
|
+
).doOn(`meta.service_registry.selected_instance_for_socket:${fetchId}`).attachSignal("meta.socket_client.transmitted");
|
|
4396
|
+
CadenzaService.createEphemeralMetaTask(
|
|
4397
|
+
`Shutdown SocketClient ${url}`,
|
|
4398
|
+
(_ctx, emitter) => {
|
|
4399
|
+
runtimeHandle.handshake = false;
|
|
4400
|
+
upsertDiagnostics({
|
|
4401
|
+
connected: false,
|
|
4402
|
+
handshake: false,
|
|
4403
|
+
destroyed: true,
|
|
4404
|
+
pendingDelegations: 0,
|
|
4405
|
+
pendingTimers: 0
|
|
4406
|
+
});
|
|
4407
|
+
applySessionOperation("shutdown", {
|
|
4408
|
+
connected: false,
|
|
4409
|
+
handshake: false,
|
|
4410
|
+
destroyed: true,
|
|
4411
|
+
pendingDelegations: 0,
|
|
4412
|
+
pendingTimers: 0
|
|
4413
|
+
});
|
|
4414
|
+
CadenzaService.log("Shutting down socket client", { url, serviceName });
|
|
4415
|
+
emitter(`meta.fetch.handshake_requested:${fetchId}`, {
|
|
4416
|
+
serviceInstanceId,
|
|
4417
|
+
serviceName,
|
|
4418
|
+
communicationTypes,
|
|
4419
|
+
serviceAddress,
|
|
4420
|
+
servicePort: normalizedPort,
|
|
4421
|
+
protocol,
|
|
4422
|
+
handshakeData: {
|
|
4423
|
+
instanceId: CadenzaService.serviceRegistry.serviceInstanceId,
|
|
4424
|
+
serviceName: CadenzaService.serviceRegistry.serviceName
|
|
3561
4425
|
}
|
|
3562
|
-
resolve(response);
|
|
3563
4426
|
});
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
`Shutdown SocketClient ${URL}`,
|
|
3570
|
-
(ctx2, emit) => {
|
|
3571
|
-
handshake = false;
|
|
3572
|
-
socketDiagnostics.connected = false;
|
|
3573
|
-
socketDiagnostics.handshake = false;
|
|
3574
|
-
socketDiagnostics.destroyed = true;
|
|
3575
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3576
|
-
CadenzaService.log("Shutting down socket client", { URL, serviceName });
|
|
3577
|
-
socket?.close();
|
|
3578
|
-
handshakeTask?.destroy();
|
|
3579
|
-
delegateTask?.destroy();
|
|
3580
|
-
transmitTask?.destroy();
|
|
3581
|
-
handshakeTask = null;
|
|
3582
|
-
delegateTask = null;
|
|
3583
|
-
transmitTask = null;
|
|
3584
|
-
emitWhenReady = null;
|
|
3585
|
-
socket = null;
|
|
3586
|
-
emit(`meta.fetch.handshake_requested:${fetchId}`, {
|
|
3587
|
-
serviceInstanceId,
|
|
3588
|
-
serviceName,
|
|
3589
|
-
communicationTypes,
|
|
3590
|
-
serviceAddress,
|
|
3591
|
-
servicePort,
|
|
3592
|
-
protocol,
|
|
3593
|
-
handshakeData: {
|
|
3594
|
-
instanceId: CadenzaService.serviceRegistry.serviceInstanceId,
|
|
3595
|
-
serviceName: CadenzaService.serviceRegistry.serviceName
|
|
4427
|
+
for (const id of runtimeHandle.pendingDelegationIds) {
|
|
4428
|
+
emitter(`meta.socket_client.delegated:${id}`, {
|
|
4429
|
+
errored: true,
|
|
4430
|
+
__error: "Shutting down socket client"
|
|
4431
|
+
});
|
|
3596
4432
|
}
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
errored: true,
|
|
3601
|
-
__error: "Shutting down socket client"
|
|
4433
|
+
this.destroySocketClientRuntimeHandle(runtimeHandle);
|
|
4434
|
+
emitter("meta.socket_client.runtime_clear_requested", {
|
|
4435
|
+
fetchId
|
|
3602
4436
|
});
|
|
3603
|
-
}
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
}
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
`meta.fetch.handshake_failed:${fetchId}`,
|
|
3617
|
-
`meta.socket_client.connect_error:${fetchId}`
|
|
3618
|
-
).attachSignal("meta.fetch.handshake_requested").emits("meta.socket_client_shutdown_complete");
|
|
3619
|
-
return true;
|
|
3620
|
-
},
|
|
3621
|
-
"Connects to a specified socket server"
|
|
4437
|
+
},
|
|
4438
|
+
"Shuts down the socket client"
|
|
4439
|
+
).doOn(
|
|
4440
|
+
`meta.socket_shutdown_requested:${fetchId}`,
|
|
4441
|
+
`meta.socket_client.disconnected:${fetchId}`,
|
|
4442
|
+
`meta.fetch.handshake_failed:${fetchId}`,
|
|
4443
|
+
`meta.socket_client.connect_error:${fetchId}`
|
|
4444
|
+
).attachSignal("meta.fetch.handshake_requested").emits("meta.socket_client_shutdown_complete");
|
|
4445
|
+
return true;
|
|
4446
|
+
},
|
|
4447
|
+
{ mode: "write" }
|
|
4448
|
+
),
|
|
4449
|
+
"Connects to a specified socket server and wires runtime tasks."
|
|
3622
4450
|
).doOn("meta.fetch.handshake_complete").emitsOnFail("meta.socket_client.connect_failed");
|
|
3623
4451
|
}
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
4452
|
+
createInitialSocketServerSessionState(serverKey) {
|
|
4453
|
+
return {
|
|
4454
|
+
serverKey,
|
|
4455
|
+
useSocket: false,
|
|
4456
|
+
status: "inactive",
|
|
4457
|
+
securityProfile: "medium",
|
|
4458
|
+
networkType: "internal",
|
|
4459
|
+
connectionCount: 0,
|
|
4460
|
+
lastStartedAt: null,
|
|
4461
|
+
lastConnectedAt: null,
|
|
4462
|
+
lastDisconnectedAt: null,
|
|
4463
|
+
lastShutdownAt: null,
|
|
4464
|
+
updatedAt: 0
|
|
4465
|
+
};
|
|
3627
4466
|
}
|
|
3628
|
-
|
|
3629
|
-
const detailLevel = ctx.detailLevel === "full" ? "full" : "summary";
|
|
3630
|
-
const includeErrorHistory = Boolean(ctx.includeErrorHistory);
|
|
3631
|
-
const requestedLimit = Number(ctx.errorHistoryLimit);
|
|
3632
|
-
const errorHistoryLimit = Number.isFinite(requestedLimit) ? Math.max(1, Math.min(200, Math.trunc(requestedLimit))) : 10;
|
|
4467
|
+
createInitialSocketClientSessionState() {
|
|
3633
4468
|
return {
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
4469
|
+
fetchId: "",
|
|
4470
|
+
serviceInstanceId: "",
|
|
4471
|
+
communicationTypes: [],
|
|
4472
|
+
serviceName: "",
|
|
4473
|
+
serviceAddress: "",
|
|
4474
|
+
servicePort: 0,
|
|
4475
|
+
protocol: "http",
|
|
4476
|
+
url: "",
|
|
4477
|
+
socketId: null,
|
|
4478
|
+
connected: false,
|
|
4479
|
+
handshake: false,
|
|
4480
|
+
pendingDelegations: 0,
|
|
4481
|
+
pendingTimers: 0,
|
|
4482
|
+
reconnectAttempts: 0,
|
|
4483
|
+
connectErrors: 0,
|
|
4484
|
+
reconnectErrors: 0,
|
|
4485
|
+
socketErrors: 0,
|
|
4486
|
+
errorCount: 0,
|
|
4487
|
+
destroyed: false,
|
|
4488
|
+
lastHandshakeAt: null,
|
|
4489
|
+
lastHandshakeError: null,
|
|
4490
|
+
lastDisconnectAt: null,
|
|
4491
|
+
updatedAt: 0
|
|
3637
4492
|
};
|
|
3638
4493
|
}
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
socketId: null,
|
|
3647
|
-
connected: false,
|
|
3648
|
-
handshake: false,
|
|
3649
|
-
reconnectAttempts: 0,
|
|
3650
|
-
connectErrors: 0,
|
|
3651
|
-
reconnectErrors: 0,
|
|
3652
|
-
socketErrors: 0,
|
|
3653
|
-
pendingDelegations: 0,
|
|
3654
|
-
pendingTimers: 0,
|
|
3655
|
-
destroyed: false,
|
|
3656
|
-
lastHandshakeAt: null,
|
|
3657
|
-
lastHandshakeError: null,
|
|
3658
|
-
lastDisconnectAt: null,
|
|
3659
|
-
lastError: null,
|
|
3660
|
-
lastErrorAt: 0,
|
|
3661
|
-
errorHistory: [],
|
|
3662
|
-
updatedAt: Date.now()
|
|
3663
|
-
};
|
|
3664
|
-
this.socketClientDiagnostics.set(fetchId, state);
|
|
3665
|
-
} else {
|
|
3666
|
-
state.serviceName = serviceName;
|
|
3667
|
-
state.url = url;
|
|
4494
|
+
resolveSocketServerKey(input) {
|
|
4495
|
+
return String(input.serverKey ?? input.__socketServerKey ?? this.socketServerDefaultKey).trim() || this.socketServerDefaultKey;
|
|
4496
|
+
}
|
|
4497
|
+
resolveSocketClientFetchId(input) {
|
|
4498
|
+
const explicitFetchId = String(input.fetchId ?? "").trim();
|
|
4499
|
+
if (explicitFetchId) {
|
|
4500
|
+
return explicitFetchId;
|
|
3668
4501
|
}
|
|
3669
|
-
|
|
4502
|
+
const serviceAddress = String(input.serviceAddress ?? "").trim();
|
|
4503
|
+
const protocol = String(input.protocol ?? "http").trim();
|
|
4504
|
+
const port = this.resolveServicePort(protocol, input.servicePort);
|
|
4505
|
+
if (!serviceAddress || !port) {
|
|
4506
|
+
return void 0;
|
|
4507
|
+
}
|
|
4508
|
+
return `${serviceAddress}_${port}`;
|
|
4509
|
+
}
|
|
4510
|
+
resolveServicePort(protocol, rawPort) {
|
|
4511
|
+
if (protocol === "https") {
|
|
4512
|
+
return 443;
|
|
4513
|
+
}
|
|
4514
|
+
const parsed = Number(rawPort);
|
|
4515
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
4516
|
+
return void 0;
|
|
4517
|
+
}
|
|
4518
|
+
return Math.trunc(parsed);
|
|
4519
|
+
}
|
|
4520
|
+
createSocketServerRuntimeHandleFromContext(context) {
|
|
4521
|
+
const baseServer = context.httpsServer ?? context.httpServer;
|
|
4522
|
+
if (!baseServer) {
|
|
4523
|
+
throw new Error(
|
|
4524
|
+
"Socket server runtime setup requires either httpsServer or httpServer"
|
|
4525
|
+
);
|
|
4526
|
+
}
|
|
4527
|
+
const server = new import_socket.Server(baseServer, {
|
|
4528
|
+
pingInterval: 3e4,
|
|
4529
|
+
pingTimeout: 2e4,
|
|
4530
|
+
maxHttpBufferSize: 1e7,
|
|
4531
|
+
connectionStateRecovery: {
|
|
4532
|
+
maxDisconnectionDuration: 2 * 60 * 1e3,
|
|
4533
|
+
skipMiddlewares: true
|
|
4534
|
+
}
|
|
4535
|
+
});
|
|
4536
|
+
return {
|
|
4537
|
+
server,
|
|
4538
|
+
initialized: false,
|
|
4539
|
+
connectedSocketIds: /* @__PURE__ */ new Set(),
|
|
4540
|
+
broadcastStatusTask: null,
|
|
4541
|
+
shutdownTask: null
|
|
4542
|
+
};
|
|
4543
|
+
}
|
|
4544
|
+
destroySocketServerRuntimeHandle(runtimeHandle) {
|
|
4545
|
+
if (!runtimeHandle) {
|
|
4546
|
+
return;
|
|
4547
|
+
}
|
|
4548
|
+
runtimeHandle.broadcastStatusTask?.destroy();
|
|
4549
|
+
runtimeHandle.shutdownTask?.destroy();
|
|
4550
|
+
runtimeHandle.broadcastStatusTask = null;
|
|
4551
|
+
runtimeHandle.shutdownTask = null;
|
|
4552
|
+
runtimeHandle.connectedSocketIds.clear();
|
|
4553
|
+
runtimeHandle.initialized = false;
|
|
4554
|
+
runtimeHandle.server.close();
|
|
4555
|
+
runtimeHandle.server.removeAllListeners();
|
|
4556
|
+
}
|
|
4557
|
+
createSocketClientRuntimeHandle(url) {
|
|
4558
|
+
return {
|
|
4559
|
+
url,
|
|
4560
|
+
socket: (0, import_socket2.io)(url, {
|
|
4561
|
+
reconnection: true,
|
|
4562
|
+
reconnectionAttempts: 5,
|
|
4563
|
+
reconnectionDelay: 2e3,
|
|
4564
|
+
reconnectionDelayMax: 1e4,
|
|
4565
|
+
randomizationFactor: 0.5,
|
|
4566
|
+
transports: ["websocket"],
|
|
4567
|
+
autoConnect: false
|
|
4568
|
+
}),
|
|
4569
|
+
initialized: false,
|
|
4570
|
+
handshake: false,
|
|
4571
|
+
errorCount: 0,
|
|
4572
|
+
pendingDelegationIds: /* @__PURE__ */ new Set(),
|
|
4573
|
+
pendingTimers: /* @__PURE__ */ new Set(),
|
|
4574
|
+
emitWhenReady: null,
|
|
4575
|
+
handshakeTask: null,
|
|
4576
|
+
delegateTask: null,
|
|
4577
|
+
transmitTask: null
|
|
4578
|
+
};
|
|
4579
|
+
}
|
|
4580
|
+
destroySocketClientRuntimeHandle(runtimeHandle) {
|
|
4581
|
+
if (!runtimeHandle) {
|
|
4582
|
+
return;
|
|
4583
|
+
}
|
|
4584
|
+
runtimeHandle.initialized = false;
|
|
4585
|
+
runtimeHandle.handshake = false;
|
|
4586
|
+
runtimeHandle.emitWhenReady = null;
|
|
4587
|
+
runtimeHandle.handshakeTask?.destroy();
|
|
4588
|
+
runtimeHandle.delegateTask?.destroy();
|
|
4589
|
+
runtimeHandle.transmitTask?.destroy();
|
|
4590
|
+
runtimeHandle.handshakeTask = null;
|
|
4591
|
+
runtimeHandle.delegateTask = null;
|
|
4592
|
+
runtimeHandle.transmitTask = null;
|
|
4593
|
+
for (const timer of runtimeHandle.pendingTimers) {
|
|
4594
|
+
clearTimeout(timer);
|
|
4595
|
+
}
|
|
4596
|
+
runtimeHandle.pendingTimers.clear();
|
|
4597
|
+
runtimeHandle.pendingDelegationIds.clear();
|
|
4598
|
+
runtimeHandle.socket.close();
|
|
4599
|
+
runtimeHandle.socket.removeAllListeners();
|
|
4600
|
+
}
|
|
4601
|
+
normalizeCommunicationTypes(value) {
|
|
4602
|
+
if (!Array.isArray(value)) {
|
|
4603
|
+
return [];
|
|
4604
|
+
}
|
|
4605
|
+
return value.map((item) => String(item)).filter((item) => item.trim().length > 0);
|
|
3670
4606
|
}
|
|
3671
4607
|
getErrorMessage(error) {
|
|
3672
4608
|
if (error instanceof Error) {
|
|
@@ -3681,28 +4617,53 @@ var SocketController = class _SocketController {
|
|
|
3681
4617
|
return String(error);
|
|
3682
4618
|
}
|
|
3683
4619
|
}
|
|
3684
|
-
|
|
3685
|
-
const state
|
|
3686
|
-
|
|
3687
|
-
|
|
3688
|
-
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
4620
|
+
pruneDiagnosticsEntries(entries, now = Date.now()) {
|
|
4621
|
+
for (const [fetchId, state] of Object.entries(entries)) {
|
|
4622
|
+
if (state.destroyed && now - state.updatedAt > this.destroyedDiagnosticsTtlMs) {
|
|
4623
|
+
delete entries[fetchId];
|
|
4624
|
+
}
|
|
4625
|
+
}
|
|
4626
|
+
if (Object.keys(entries).length <= this.diagnosticsMaxClientEntries) {
|
|
4627
|
+
return;
|
|
4628
|
+
}
|
|
4629
|
+
const entriesByEvictionPriority = Object.entries(entries).sort((left, right) => {
|
|
4630
|
+
if (left[1].destroyed !== right[1].destroyed) {
|
|
4631
|
+
return left[1].destroyed ? -1 : 1;
|
|
4632
|
+
}
|
|
4633
|
+
return left[1].updatedAt - right[1].updatedAt;
|
|
3694
4634
|
});
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
state.errorHistory.length - this.diagnosticsErrorHistoryLimit
|
|
3699
|
-
);
|
|
4635
|
+
while (Object.keys(entries).length > this.diagnosticsMaxClientEntries && entriesByEvictionPriority.length > 0) {
|
|
4636
|
+
const [fetchId] = entriesByEvictionPriority.shift();
|
|
4637
|
+
delete entries[fetchId];
|
|
3700
4638
|
}
|
|
3701
4639
|
}
|
|
3702
|
-
|
|
4640
|
+
async getSocketClientDiagnosticsEntry(fetchId) {
|
|
4641
|
+
const normalized = String(fetchId ?? "").trim();
|
|
4642
|
+
if (!normalized) {
|
|
4643
|
+
return void 0;
|
|
4644
|
+
}
|
|
4645
|
+
const snapshot = this.socketClientDiagnosticsActor.getState();
|
|
4646
|
+
const entries = { ...snapshot.entries };
|
|
4647
|
+
this.pruneDiagnosticsEntries(entries);
|
|
4648
|
+
return entries[normalized];
|
|
4649
|
+
}
|
|
4650
|
+
resolveTransportDiagnosticsOptions(ctx) {
|
|
4651
|
+
const detailLevel = ctx.detailLevel === "full" ? "full" : "summary";
|
|
4652
|
+
const includeErrorHistory = Boolean(ctx.includeErrorHistory);
|
|
4653
|
+
const requestedLimit = Number(ctx.errorHistoryLimit);
|
|
4654
|
+
const errorHistoryLimit = Number.isFinite(requestedLimit) ? Math.max(1, Math.min(200, Math.trunc(requestedLimit))) : 10;
|
|
4655
|
+
return {
|
|
4656
|
+
detailLevel,
|
|
4657
|
+
includeErrorHistory,
|
|
4658
|
+
errorHistoryLimit
|
|
4659
|
+
};
|
|
4660
|
+
}
|
|
4661
|
+
collectSocketTransportDiagnostics(ctx, diagnosticsEntries) {
|
|
3703
4662
|
const { detailLevel, includeErrorHistory, errorHistoryLimit } = this.resolveTransportDiagnosticsOptions(ctx);
|
|
3704
4663
|
const serviceName = CadenzaService.serviceRegistry.serviceName ?? "UnknownService";
|
|
3705
|
-
const
|
|
4664
|
+
const entries = { ...diagnosticsEntries };
|
|
4665
|
+
this.pruneDiagnosticsEntries(entries);
|
|
4666
|
+
const states = Object.values(entries).sort(
|
|
3706
4667
|
(a, b) => a.fetchId.localeCompare(b.fetchId)
|
|
3707
4668
|
);
|
|
3708
4669
|
const summary = {
|
|
@@ -3720,10 +4681,7 @@ var SocketController = class _SocketController {
|
|
|
3720
4681
|
0
|
|
3721
4682
|
),
|
|
3722
4683
|
connectErrors: states.reduce((acc, state) => acc + state.connectErrors, 0),
|
|
3723
|
-
reconnectErrors: states.reduce(
|
|
3724
|
-
(acc, state) => acc + state.reconnectErrors,
|
|
3725
|
-
0
|
|
3726
|
-
),
|
|
4684
|
+
reconnectErrors: states.reduce((acc, state) => acc + state.reconnectErrors, 0),
|
|
3727
4685
|
socketErrors: states.reduce((acc, state) => acc + state.socketErrors, 0),
|
|
3728
4686
|
latestError: states.slice().sort((a, b) => b.lastErrorAt - a.lastErrorAt).find((state) => state.lastError)?.lastError ?? null
|
|
3729
4687
|
};
|
|
@@ -7210,6 +8168,14 @@ var CadenzaService = class {
|
|
|
7210
8168
|
options.isMeta = true;
|
|
7211
8169
|
this.createDatabaseService(name, schema, description, options);
|
|
7212
8170
|
}
|
|
8171
|
+
static createActor(spec, options = {}) {
|
|
8172
|
+
this.bootstrap();
|
|
8173
|
+
return new import_core3.Actor(spec, options);
|
|
8174
|
+
}
|
|
8175
|
+
static createActorFromDefinition(definition, options = {}) {
|
|
8176
|
+
this.bootstrap();
|
|
8177
|
+
return import_core3.default.createActorFromDefinition(definition, options);
|
|
8178
|
+
}
|
|
7213
8179
|
/**
|
|
7214
8180
|
* Creates and registers a new task with the provided name, function, and optional details.
|
|
7215
8181
|
*
|
|
@@ -7627,6 +8593,7 @@ var import_core4 = require("@cadenza.io/core");
|
|
|
7627
8593
|
var index_default = CadenzaService;
|
|
7628
8594
|
// Annotate the CommonJS export names for ESM import in node:
|
|
7629
8595
|
0 && (module.exports = {
|
|
8596
|
+
Actor,
|
|
7630
8597
|
DatabaseTask,
|
|
7631
8598
|
DebounceTask,
|
|
7632
8599
|
DeputyTask,
|