@cadenza.io/service 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +69 -29
- package/dist/index.d.ts +69 -29
- package/dist/index.js +1610 -643
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1613 -644
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// src/Cadenza.ts
|
|
2
|
-
import Cadenza
|
|
2
|
+
import Cadenza, {
|
|
3
|
+
Actor
|
|
4
|
+
} from "@cadenza.io/core";
|
|
3
5
|
|
|
4
6
|
// src/graph/definition/DeputyTask.ts
|
|
5
7
|
import { v4 as uuid } from "uuid";
|
|
@@ -251,6 +253,7 @@ var isBrowser = typeof window !== "undefined" && typeof window.document !== "und
|
|
|
251
253
|
var META_INTENT_PREFIX = "meta-";
|
|
252
254
|
var META_RUNTIME_TRANSPORT_DIAGNOSTICS_INTENT = "meta-runtime-transport-diagnostics";
|
|
253
255
|
var META_RUNTIME_STATUS_INTENT = "meta-runtime-status";
|
|
256
|
+
var META_READINESS_INTENT = "meta-readiness";
|
|
254
257
|
function isPlainObject(value) {
|
|
255
258
|
return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
|
|
256
259
|
}
|
|
@@ -314,6 +317,100 @@ function summarizeResponderStatuses(statuses) {
|
|
|
314
317
|
return { responded, failed, timedOut, pending };
|
|
315
318
|
}
|
|
316
319
|
|
|
320
|
+
// src/utils/readiness.ts
|
|
321
|
+
function evaluateDependencyReadiness(input) {
|
|
322
|
+
const missedHeartbeats = Math.max(
|
|
323
|
+
0,
|
|
324
|
+
Math.trunc(Number(input.missedHeartbeats) || 0)
|
|
325
|
+
);
|
|
326
|
+
const stale = missedHeartbeats > 0;
|
|
327
|
+
const timeoutReached = missedHeartbeats >= Math.max(1, input.missThreshold);
|
|
328
|
+
if (!input.exists) {
|
|
329
|
+
return {
|
|
330
|
+
state: "unavailable",
|
|
331
|
+
stale: true,
|
|
332
|
+
blocked: true,
|
|
333
|
+
reason: "missing"
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
if (timeoutReached) {
|
|
337
|
+
return {
|
|
338
|
+
state: "unavailable",
|
|
339
|
+
stale: true,
|
|
340
|
+
blocked: true,
|
|
341
|
+
reason: "heartbeat-timeout"
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
if (input.runtimeState === "unavailable" || !input.acceptingWork) {
|
|
345
|
+
return {
|
|
346
|
+
state: "unavailable",
|
|
347
|
+
stale,
|
|
348
|
+
blocked: true,
|
|
349
|
+
reason: "runtime-unavailable"
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
if (stale) {
|
|
353
|
+
return {
|
|
354
|
+
state: "degraded",
|
|
355
|
+
stale: true,
|
|
356
|
+
blocked: false,
|
|
357
|
+
reason: "heartbeat-stale"
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
if (input.runtimeState === "overloaded") {
|
|
361
|
+
return {
|
|
362
|
+
state: "overloaded",
|
|
363
|
+
stale: false,
|
|
364
|
+
blocked: false,
|
|
365
|
+
reason: "runtime-overloaded"
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
if (input.runtimeState === "degraded") {
|
|
369
|
+
return {
|
|
370
|
+
state: "degraded",
|
|
371
|
+
stale: false,
|
|
372
|
+
blocked: false,
|
|
373
|
+
reason: "runtime-degraded"
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
return {
|
|
377
|
+
state: "ready",
|
|
378
|
+
stale: false,
|
|
379
|
+
blocked: false,
|
|
380
|
+
reason: "runtime-healthy"
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
function summarizeDependencyReadiness(evaluations) {
|
|
384
|
+
const summary = {
|
|
385
|
+
total: evaluations.length,
|
|
386
|
+
ready: 0,
|
|
387
|
+
degraded: 0,
|
|
388
|
+
overloaded: 0,
|
|
389
|
+
unavailable: 0,
|
|
390
|
+
stale: 0
|
|
391
|
+
};
|
|
392
|
+
for (const evaluation of evaluations) {
|
|
393
|
+
if (evaluation.state === "ready") summary.ready++;
|
|
394
|
+
if (evaluation.state === "degraded") summary.degraded++;
|
|
395
|
+
if (evaluation.state === "overloaded") summary.overloaded++;
|
|
396
|
+
if (evaluation.state === "unavailable") summary.unavailable++;
|
|
397
|
+
if (evaluation.stale) summary.stale++;
|
|
398
|
+
}
|
|
399
|
+
return summary;
|
|
400
|
+
}
|
|
401
|
+
function resolveServiceReadinessState(localRuntimeState, localAcceptingWork, dependencySummary) {
|
|
402
|
+
if (localRuntimeState === "unavailable" || !localAcceptingWork) {
|
|
403
|
+
return "blocked";
|
|
404
|
+
}
|
|
405
|
+
if (dependencySummary.unavailable > 0) {
|
|
406
|
+
return "blocked";
|
|
407
|
+
}
|
|
408
|
+
if (dependencySummary.degraded > 0 || dependencySummary.overloaded > 0 || dependencySummary.stale > 0) {
|
|
409
|
+
return "degraded";
|
|
410
|
+
}
|
|
411
|
+
return "ready";
|
|
412
|
+
}
|
|
413
|
+
|
|
317
414
|
// src/utils/runtimeStatus.ts
|
|
318
415
|
function resolveRuntimeStatus(input) {
|
|
319
416
|
const numberOfRunningGraphs = Math.max(
|
|
@@ -395,6 +492,8 @@ var INTERNAL_RUNTIME_STATUS_TASK_NAMES = /* @__PURE__ */ new Set([
|
|
|
395
492
|
"Monitor dependee heartbeat freshness",
|
|
396
493
|
"Resolve runtime status fallback inquiry",
|
|
397
494
|
"Respond runtime status inquiry",
|
|
495
|
+
"Respond readiness inquiry",
|
|
496
|
+
"Collect distributed readiness",
|
|
398
497
|
"Get status"
|
|
399
498
|
]);
|
|
400
499
|
function readPositiveIntegerEnv(name, fallback) {
|
|
@@ -431,6 +530,8 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
431
530
|
this.remoteIntentDeputiesByTask = /* @__PURE__ */ new Map();
|
|
432
531
|
this.dependeesByService = /* @__PURE__ */ new Map();
|
|
433
532
|
this.dependeeByInstance = /* @__PURE__ */ new Map();
|
|
533
|
+
this.readinessDependeesByService = /* @__PURE__ */ new Map();
|
|
534
|
+
this.readinessDependeeByInstance = /* @__PURE__ */ new Map();
|
|
434
535
|
this.lastHeartbeatAtByInstance = /* @__PURE__ */ new Map();
|
|
435
536
|
this.missedHeartbeatsByInstance = /* @__PURE__ */ new Map();
|
|
436
537
|
this.runtimeStatusFallbackInFlightByInstance = /* @__PURE__ */ new Set();
|
|
@@ -546,6 +647,66 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
546
647
|
},
|
|
547
648
|
"Responds to runtime-status inquiries with local service instance status."
|
|
548
649
|
).respondsTo(META_RUNTIME_STATUS_INTENT);
|
|
650
|
+
CadenzaService.defineIntent({
|
|
651
|
+
name: META_READINESS_INTENT,
|
|
652
|
+
description: "Gather service readiness reports derived from local runtime status and required dependees.",
|
|
653
|
+
input: {
|
|
654
|
+
type: "object",
|
|
655
|
+
properties: {
|
|
656
|
+
detailLevel: {
|
|
657
|
+
type: "string",
|
|
658
|
+
constraints: {
|
|
659
|
+
oneOf: ["minimal", "full"]
|
|
660
|
+
}
|
|
661
|
+
},
|
|
662
|
+
includeDependencies: {
|
|
663
|
+
type: "boolean"
|
|
664
|
+
},
|
|
665
|
+
refreshStaleDependencies: {
|
|
666
|
+
type: "boolean"
|
|
667
|
+
},
|
|
668
|
+
targetServiceName: {
|
|
669
|
+
type: "string"
|
|
670
|
+
},
|
|
671
|
+
targetServiceInstanceId: {
|
|
672
|
+
type: "string"
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
},
|
|
676
|
+
output: {
|
|
677
|
+
type: "object",
|
|
678
|
+
properties: {
|
|
679
|
+
readinessReports: {
|
|
680
|
+
type: "array"
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
});
|
|
685
|
+
CadenzaService.createMetaTask(
|
|
686
|
+
"Respond readiness inquiry",
|
|
687
|
+
async (ctx) => {
|
|
688
|
+
const targetServiceName = ctx.targetServiceName;
|
|
689
|
+
const targetServiceInstanceId = ctx.targetServiceInstanceId;
|
|
690
|
+
const report = await this.buildLocalReadinessReport({
|
|
691
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
692
|
+
includeDependencies: ctx.includeDependencies,
|
|
693
|
+
refreshStaleDependencies: ctx.refreshStaleDependencies
|
|
694
|
+
});
|
|
695
|
+
if (!report) {
|
|
696
|
+
return {};
|
|
697
|
+
}
|
|
698
|
+
if (targetServiceName && targetServiceName !== report.serviceName) {
|
|
699
|
+
return {};
|
|
700
|
+
}
|
|
701
|
+
if (targetServiceInstanceId && targetServiceInstanceId !== report.serviceInstanceId) {
|
|
702
|
+
return {};
|
|
703
|
+
}
|
|
704
|
+
return {
|
|
705
|
+
readinessReports: [report]
|
|
706
|
+
};
|
|
707
|
+
},
|
|
708
|
+
"Responds to distributed readiness inquiries using required dependee health."
|
|
709
|
+
).respondsTo(META_READINESS_INTENT);
|
|
549
710
|
this.handleInstanceUpdateTask = CadenzaService.createMetaTask(
|
|
550
711
|
"Handle Instance Update",
|
|
551
712
|
(ctx, emit) => {
|
|
@@ -662,7 +823,11 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
662
823
|
if (!ctx.serviceName || !ctx.serviceInstanceId) {
|
|
663
824
|
return false;
|
|
664
825
|
}
|
|
665
|
-
this.registerDependee(ctx.serviceName, ctx.serviceInstanceId
|
|
826
|
+
this.registerDependee(ctx.serviceName, ctx.serviceInstanceId, {
|
|
827
|
+
requiredForReadiness: this.shouldRequireReadinessFromCommunicationTypes(
|
|
828
|
+
ctx.communicationTypes
|
|
829
|
+
)
|
|
830
|
+
});
|
|
666
831
|
return true;
|
|
667
832
|
},
|
|
668
833
|
"Tracks remote dependency instances for runtime heartbeat monitoring."
|
|
@@ -1347,36 +1512,20 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1347
1512
|
return false;
|
|
1348
1513
|
}
|
|
1349
1514
|
try {
|
|
1350
|
-
const
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
targetServiceName: serviceName,
|
|
1354
|
-
targetServiceInstanceId: serviceInstanceId,
|
|
1355
|
-
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal"
|
|
1356
|
-
},
|
|
1515
|
+
const { report, inquiryMeta } = await this.resolveRuntimeStatusFallbackInquiry(
|
|
1516
|
+
serviceName,
|
|
1517
|
+
serviceInstanceId,
|
|
1357
1518
|
{
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1519
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
1520
|
+
overallTimeoutMs: ctx.overallTimeoutMs,
|
|
1521
|
+
perResponderTimeoutMs: ctx.perResponderTimeoutMs,
|
|
1522
|
+
requireComplete: ctx.requireComplete
|
|
1361
1523
|
}
|
|
1362
1524
|
);
|
|
1363
|
-
const report = this.selectRuntimeStatusReportForTarget(
|
|
1364
|
-
inquiryResult,
|
|
1365
|
-
serviceName,
|
|
1366
|
-
serviceInstanceId
|
|
1367
|
-
);
|
|
1368
|
-
if (!report) {
|
|
1369
|
-
throw new Error(
|
|
1370
|
-
`No runtime status report for ${serviceName}/${serviceInstanceId}`
|
|
1371
|
-
);
|
|
1372
|
-
}
|
|
1373
|
-
this.applyRuntimeStatusReport(report);
|
|
1374
|
-
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
1375
|
-
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
1376
1525
|
return {
|
|
1377
1526
|
...ctx,
|
|
1378
1527
|
runtimeStatusReport: report,
|
|
1379
|
-
__inquiryMeta:
|
|
1528
|
+
__inquiryMeta: inquiryMeta
|
|
1380
1529
|
};
|
|
1381
1530
|
} catch (error) {
|
|
1382
1531
|
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
@@ -1410,6 +1559,27 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1410
1559
|
},
|
|
1411
1560
|
"Runs runtime-status inquiry fallback for a dependee instance after missed heartbeats."
|
|
1412
1561
|
).doOn("meta.service_registry.runtime_status_fallback_requested").emits("meta.service_registry.runtime_status_fallback_resolved").emitsOnFail("meta.service_registry.runtime_status_fallback_failed");
|
|
1562
|
+
this.collectReadinessTask = CadenzaService.createMetaTask(
|
|
1563
|
+
"Collect distributed readiness",
|
|
1564
|
+
async (ctx) => {
|
|
1565
|
+
const inquiryResult = await CadenzaService.inquire(
|
|
1566
|
+
META_READINESS_INTENT,
|
|
1567
|
+
{
|
|
1568
|
+
detailLevel: ctx.detailLevel === "full" ? "full" : "minimal",
|
|
1569
|
+
includeDependencies: ctx.includeDependencies,
|
|
1570
|
+
refreshStaleDependencies: ctx.refreshStaleDependencies,
|
|
1571
|
+
targetServiceName: ctx.targetServiceName,
|
|
1572
|
+
targetServiceInstanceId: ctx.targetServiceInstanceId
|
|
1573
|
+
},
|
|
1574
|
+
ctx.inquiryOptions ?? ctx.__inquiryOptions ?? {}
|
|
1575
|
+
);
|
|
1576
|
+
return {
|
|
1577
|
+
...ctx,
|
|
1578
|
+
...inquiryResult
|
|
1579
|
+
};
|
|
1580
|
+
},
|
|
1581
|
+
"Collects distributed readiness reports from services."
|
|
1582
|
+
).doOn("meta.service_registry.readiness_requested").emits("meta.service_registry.readiness_collected").emitsOnFail("meta.service_registry.readiness_failed");
|
|
1413
1583
|
this.collectTransportDiagnosticsTask = CadenzaService.createMetaTask(
|
|
1414
1584
|
"Collect transport diagnostics",
|
|
1415
1585
|
async (ctx) => {
|
|
@@ -1758,7 +1928,7 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1758
1928
|
}
|
|
1759
1929
|
return this.getInstance(this.serviceName, this.serviceInstanceId);
|
|
1760
1930
|
}
|
|
1761
|
-
registerDependee(serviceName, serviceInstanceId) {
|
|
1931
|
+
registerDependee(serviceName, serviceInstanceId, options = {}) {
|
|
1762
1932
|
if (!serviceName || !serviceInstanceId) {
|
|
1763
1933
|
return;
|
|
1764
1934
|
}
|
|
@@ -1767,6 +1937,13 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1767
1937
|
}
|
|
1768
1938
|
this.dependeesByService.get(serviceName).add(serviceInstanceId);
|
|
1769
1939
|
this.dependeeByInstance.set(serviceInstanceId, serviceName);
|
|
1940
|
+
if (options.requiredForReadiness) {
|
|
1941
|
+
if (!this.readinessDependeesByService.has(serviceName)) {
|
|
1942
|
+
this.readinessDependeesByService.set(serviceName, /* @__PURE__ */ new Set());
|
|
1943
|
+
}
|
|
1944
|
+
this.readinessDependeesByService.get(serviceName).add(serviceInstanceId);
|
|
1945
|
+
this.readinessDependeeByInstance.set(serviceInstanceId, serviceName);
|
|
1946
|
+
}
|
|
1770
1947
|
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
1771
1948
|
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
1772
1949
|
}
|
|
@@ -1779,10 +1956,39 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1779
1956
|
}
|
|
1780
1957
|
}
|
|
1781
1958
|
this.dependeeByInstance.delete(serviceInstanceId);
|
|
1959
|
+
const readinessDependeeServiceName = serviceName ?? this.readinessDependeeByInstance.get(serviceInstanceId);
|
|
1960
|
+
if (readinessDependeeServiceName) {
|
|
1961
|
+
this.readinessDependeesByService.get(readinessDependeeServiceName)?.delete(serviceInstanceId);
|
|
1962
|
+
if (!this.readinessDependeesByService.get(readinessDependeeServiceName)?.size) {
|
|
1963
|
+
this.readinessDependeesByService.delete(readinessDependeeServiceName);
|
|
1964
|
+
}
|
|
1965
|
+
}
|
|
1966
|
+
this.readinessDependeeByInstance.delete(serviceInstanceId);
|
|
1782
1967
|
this.lastHeartbeatAtByInstance.delete(serviceInstanceId);
|
|
1783
1968
|
this.missedHeartbeatsByInstance.delete(serviceInstanceId);
|
|
1784
1969
|
this.runtimeStatusFallbackInFlightByInstance.delete(serviceInstanceId);
|
|
1785
1970
|
}
|
|
1971
|
+
getHeartbeatMisses(serviceInstanceId, now = Date.now()) {
|
|
1972
|
+
const observedMisses = this.missedHeartbeatsByInstance.get(serviceInstanceId) ?? 0;
|
|
1973
|
+
const lastHeartbeatAt = this.lastHeartbeatAtByInstance.get(serviceInstanceId) ?? 0;
|
|
1974
|
+
if (lastHeartbeatAt <= 0) {
|
|
1975
|
+
return Math.max(observedMisses, this.runtimeStatusMissThreshold);
|
|
1976
|
+
}
|
|
1977
|
+
const estimatedMisses = Math.max(
|
|
1978
|
+
0,
|
|
1979
|
+
Math.floor((now - lastHeartbeatAt) / this.runtimeStatusHeartbeatIntervalMs)
|
|
1980
|
+
);
|
|
1981
|
+
return Math.max(observedMisses, estimatedMisses);
|
|
1982
|
+
}
|
|
1983
|
+
shouldRequireReadinessFromCommunicationTypes(communicationTypes) {
|
|
1984
|
+
if (!Array.isArray(communicationTypes)) {
|
|
1985
|
+
return false;
|
|
1986
|
+
}
|
|
1987
|
+
return communicationTypes.some((type) => {
|
|
1988
|
+
const normalized = String(type).toLowerCase();
|
|
1989
|
+
return normalized === "delegation" || normalized === "inquiry";
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
1786
1992
|
resolveRuntimeStatusSnapshot(numberOfRunningGraphs, isActive, isNonResponsive, isBlocked) {
|
|
1787
1993
|
return resolveRuntimeStatus({
|
|
1788
1994
|
numberOfRunningGraphs,
|
|
@@ -1926,6 +2132,166 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1926
2132
|
}
|
|
1927
2133
|
return null;
|
|
1928
2134
|
}
|
|
2135
|
+
async resolveRuntimeStatusFallbackInquiry(serviceName, serviceInstanceId, options = {}) {
|
|
2136
|
+
const inquiryResult = await CadenzaService.inquire(
|
|
2137
|
+
META_RUNTIME_STATUS_INTENT,
|
|
2138
|
+
{
|
|
2139
|
+
targetServiceName: serviceName,
|
|
2140
|
+
targetServiceInstanceId: serviceInstanceId,
|
|
2141
|
+
detailLevel: options.detailLevel ?? "minimal"
|
|
2142
|
+
},
|
|
2143
|
+
{
|
|
2144
|
+
overallTimeoutMs: options.overallTimeoutMs ?? this.runtimeStatusFallbackTimeoutMs,
|
|
2145
|
+
perResponderTimeoutMs: options.perResponderTimeoutMs ?? Math.max(250, Math.floor(this.runtimeStatusFallbackTimeoutMs * 0.75)),
|
|
2146
|
+
requireComplete: options.requireComplete ?? false
|
|
2147
|
+
}
|
|
2148
|
+
);
|
|
2149
|
+
const report = this.selectRuntimeStatusReportForTarget(
|
|
2150
|
+
inquiryResult,
|
|
2151
|
+
serviceName,
|
|
2152
|
+
serviceInstanceId
|
|
2153
|
+
);
|
|
2154
|
+
if (!report) {
|
|
2155
|
+
throw new Error(
|
|
2156
|
+
`No runtime status report for ${serviceName}/${serviceInstanceId}`
|
|
2157
|
+
);
|
|
2158
|
+
}
|
|
2159
|
+
if (!this.applyRuntimeStatusReport(report)) {
|
|
2160
|
+
throw new Error(
|
|
2161
|
+
`No tracked instance for runtime fallback ${serviceName}/${serviceInstanceId}`
|
|
2162
|
+
);
|
|
2163
|
+
}
|
|
2164
|
+
this.lastHeartbeatAtByInstance.set(serviceInstanceId, Date.now());
|
|
2165
|
+
this.missedHeartbeatsByInstance.set(serviceInstanceId, 0);
|
|
2166
|
+
return {
|
|
2167
|
+
report,
|
|
2168
|
+
inquiryMeta: inquiryResult.__inquiryMeta ?? {}
|
|
2169
|
+
};
|
|
2170
|
+
}
|
|
2171
|
+
evaluateDependencyReadinessDetail(serviceName, serviceInstanceId, now = Date.now()) {
|
|
2172
|
+
const instance = this.getInstance(serviceName, serviceInstanceId);
|
|
2173
|
+
const missedHeartbeats = this.getHeartbeatMisses(serviceInstanceId, now);
|
|
2174
|
+
const runtimeState = instance ? instance.runtimeState ?? this.resolveRuntimeStatusSnapshot(
|
|
2175
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
2176
|
+
instance.isActive,
|
|
2177
|
+
instance.isNonResponsive,
|
|
2178
|
+
instance.isBlocked
|
|
2179
|
+
).state : "unavailable";
|
|
2180
|
+
const acceptingWork = instance ? typeof instance.acceptingWork === "boolean" ? instance.acceptingWork : this.resolveRuntimeStatusSnapshot(
|
|
2181
|
+
instance.numberOfRunningGraphs ?? 0,
|
|
2182
|
+
instance.isActive,
|
|
2183
|
+
instance.isNonResponsive,
|
|
2184
|
+
instance.isBlocked
|
|
2185
|
+
).acceptingWork : false;
|
|
2186
|
+
const evaluation = evaluateDependencyReadiness({
|
|
2187
|
+
exists: Boolean(instance),
|
|
2188
|
+
runtimeState,
|
|
2189
|
+
acceptingWork,
|
|
2190
|
+
missedHeartbeats,
|
|
2191
|
+
missThreshold: this.runtimeStatusMissThreshold
|
|
2192
|
+
});
|
|
2193
|
+
const lastHeartbeat = this.lastHeartbeatAtByInstance.get(serviceInstanceId);
|
|
2194
|
+
return {
|
|
2195
|
+
serviceName,
|
|
2196
|
+
serviceInstanceId,
|
|
2197
|
+
dependencyState: evaluation.state,
|
|
2198
|
+
runtimeState,
|
|
2199
|
+
acceptingWork,
|
|
2200
|
+
missedHeartbeats,
|
|
2201
|
+
stale: evaluation.stale,
|
|
2202
|
+
blocked: evaluation.blocked,
|
|
2203
|
+
reason: evaluation.reason,
|
|
2204
|
+
lastHeartbeatAt: lastHeartbeat ? new Date(lastHeartbeat).toISOString() : null,
|
|
2205
|
+
reportedAt: instance?.reportedAt ?? null
|
|
2206
|
+
};
|
|
2207
|
+
}
|
|
2208
|
+
async buildLocalReadinessReport(options = {}) {
|
|
2209
|
+
const localRuntime = this.buildLocalRuntimeStatusReport("minimal");
|
|
2210
|
+
if (!localRuntime) {
|
|
2211
|
+
return null;
|
|
2212
|
+
}
|
|
2213
|
+
const detailLevel = options.detailLevel ?? "minimal";
|
|
2214
|
+
const includeDependencies = options.includeDependencies ?? detailLevel === "full";
|
|
2215
|
+
const refreshStaleDependencies = options.refreshStaleDependencies ?? true;
|
|
2216
|
+
const dependencyPairs = Array.from(this.readinessDependeesByService.entries()).flatMap(
|
|
2217
|
+
([serviceName, instanceIds]) => Array.from(instanceIds).map((serviceInstanceId) => ({
|
|
2218
|
+
serviceName,
|
|
2219
|
+
serviceInstanceId
|
|
2220
|
+
}))
|
|
2221
|
+
).sort((left, right) => {
|
|
2222
|
+
if (left.serviceName !== right.serviceName) {
|
|
2223
|
+
return left.serviceName.localeCompare(right.serviceName);
|
|
2224
|
+
}
|
|
2225
|
+
return left.serviceInstanceId.localeCompare(right.serviceInstanceId);
|
|
2226
|
+
});
|
|
2227
|
+
if (refreshStaleDependencies) {
|
|
2228
|
+
for (const dependency of dependencyPairs) {
|
|
2229
|
+
const misses = this.getHeartbeatMisses(dependency.serviceInstanceId);
|
|
2230
|
+
if (misses < this.runtimeStatusMissThreshold) {
|
|
2231
|
+
continue;
|
|
2232
|
+
}
|
|
2233
|
+
if (this.runtimeStatusFallbackInFlightByInstance.has(
|
|
2234
|
+
dependency.serviceInstanceId
|
|
2235
|
+
)) {
|
|
2236
|
+
continue;
|
|
2237
|
+
}
|
|
2238
|
+
this.runtimeStatusFallbackInFlightByInstance.add(
|
|
2239
|
+
dependency.serviceInstanceId
|
|
2240
|
+
);
|
|
2241
|
+
try {
|
|
2242
|
+
await this.resolveRuntimeStatusFallbackInquiry(
|
|
2243
|
+
dependency.serviceName,
|
|
2244
|
+
dependency.serviceInstanceId
|
|
2245
|
+
);
|
|
2246
|
+
} catch (error) {
|
|
2247
|
+
CadenzaService.log(
|
|
2248
|
+
"Readiness dependency fallback failed.",
|
|
2249
|
+
{
|
|
2250
|
+
serviceName: dependency.serviceName,
|
|
2251
|
+
serviceInstanceId: dependency.serviceInstanceId,
|
|
2252
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2253
|
+
},
|
|
2254
|
+
"warning"
|
|
2255
|
+
);
|
|
2256
|
+
} finally {
|
|
2257
|
+
this.runtimeStatusFallbackInFlightByInstance.delete(
|
|
2258
|
+
dependency.serviceInstanceId
|
|
2259
|
+
);
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
const now = Date.now();
|
|
2264
|
+
const dependencyDetails = dependencyPairs.map(
|
|
2265
|
+
(dependency) => this.evaluateDependencyReadinessDetail(
|
|
2266
|
+
dependency.serviceName,
|
|
2267
|
+
dependency.serviceInstanceId,
|
|
2268
|
+
now
|
|
2269
|
+
)
|
|
2270
|
+
);
|
|
2271
|
+
const dependencySummary = summarizeDependencyReadiness(
|
|
2272
|
+
dependencyDetails.map((detail) => ({
|
|
2273
|
+
state: detail.dependencyState,
|
|
2274
|
+
stale: detail.stale,
|
|
2275
|
+
blocked: detail.blocked,
|
|
2276
|
+
reason: detail.reason
|
|
2277
|
+
}))
|
|
2278
|
+
);
|
|
2279
|
+
const readinessState = resolveServiceReadinessState(
|
|
2280
|
+
localRuntime.state,
|
|
2281
|
+
localRuntime.acceptingWork,
|
|
2282
|
+
dependencySummary
|
|
2283
|
+
);
|
|
2284
|
+
return {
|
|
2285
|
+
serviceName: localRuntime.serviceName,
|
|
2286
|
+
serviceInstanceId: localRuntime.serviceInstanceId,
|
|
2287
|
+
reportedAt: new Date(now).toISOString(),
|
|
2288
|
+
readinessState,
|
|
2289
|
+
runtimeState: localRuntime.state,
|
|
2290
|
+
acceptingWork: localRuntime.acceptingWork,
|
|
2291
|
+
dependencySummary,
|
|
2292
|
+
...includeDependencies ? { dependencies: dependencyDetails } : {}
|
|
2293
|
+
};
|
|
2294
|
+
}
|
|
1929
2295
|
reset() {
|
|
1930
2296
|
this.instances.clear();
|
|
1931
2297
|
this.deputies.clear();
|
|
@@ -1935,6 +2301,8 @@ var ServiceRegistry = class _ServiceRegistry {
|
|
|
1935
2301
|
this.remoteIntentDeputiesByTask.clear();
|
|
1936
2302
|
this.dependeesByService.clear();
|
|
1937
2303
|
this.dependeeByInstance.clear();
|
|
2304
|
+
this.readinessDependeesByService.clear();
|
|
2305
|
+
this.readinessDependeeByInstance.clear();
|
|
1938
2306
|
this.lastHeartbeatAtByInstance.clear();
|
|
1939
2307
|
this.missedHeartbeatsByInstance.clear();
|
|
1940
2308
|
this.runtimeStatusFallbackInFlightByInstance.clear();
|
|
@@ -2060,6 +2428,8 @@ var RestController = class _RestController {
|
|
|
2060
2428
|
constructor() {
|
|
2061
2429
|
this.fetchClientDiagnostics = /* @__PURE__ */ new Map();
|
|
2062
2430
|
this.diagnosticsErrorHistoryLimit = 100;
|
|
2431
|
+
this.diagnosticsMaxClientEntries = 500;
|
|
2432
|
+
this.destroyedDiagnosticsTtlMs = 15 * 6e4;
|
|
2063
2433
|
/**
|
|
2064
2434
|
* Fetches data from the given URL with a specified timeout. This function performs
|
|
2065
2435
|
* a fetch request with the ability to cancel the request if it exceeds the provided timeout duration.
|
|
@@ -2713,6 +3083,28 @@ var RestController = class _RestController {
|
|
|
2713
3083
|
if (!this._instance) this._instance = new _RestController();
|
|
2714
3084
|
return this._instance;
|
|
2715
3085
|
}
|
|
3086
|
+
pruneFetchClientDiagnostics(now = Date.now()) {
|
|
3087
|
+
for (const [fetchId, state] of this.fetchClientDiagnostics.entries()) {
|
|
3088
|
+
if (state.destroyed && now - state.updatedAt > this.destroyedDiagnosticsTtlMs) {
|
|
3089
|
+
this.fetchClientDiagnostics.delete(fetchId);
|
|
3090
|
+
}
|
|
3091
|
+
}
|
|
3092
|
+
if (this.fetchClientDiagnostics.size <= this.diagnosticsMaxClientEntries) {
|
|
3093
|
+
return;
|
|
3094
|
+
}
|
|
3095
|
+
const entriesByEvictionPriority = Array.from(
|
|
3096
|
+
this.fetchClientDiagnostics.entries()
|
|
3097
|
+
).sort((left, right) => {
|
|
3098
|
+
if (left[1].destroyed !== right[1].destroyed) {
|
|
3099
|
+
return left[1].destroyed ? -1 : 1;
|
|
3100
|
+
}
|
|
3101
|
+
return left[1].updatedAt - right[1].updatedAt;
|
|
3102
|
+
});
|
|
3103
|
+
while (this.fetchClientDiagnostics.size > this.diagnosticsMaxClientEntries && entriesByEvictionPriority.length > 0) {
|
|
3104
|
+
const [fetchId] = entriesByEvictionPriority.shift();
|
|
3105
|
+
this.fetchClientDiagnostics.delete(fetchId);
|
|
3106
|
+
}
|
|
3107
|
+
}
|
|
2716
3108
|
resolveTransportDiagnosticsOptions(ctx) {
|
|
2717
3109
|
const detailLevel = ctx.detailLevel === "full" ? "full" : "summary";
|
|
2718
3110
|
const includeErrorHistory = Boolean(ctx.includeErrorHistory);
|
|
@@ -2725,6 +3117,8 @@ var RestController = class _RestController {
|
|
|
2725
3117
|
};
|
|
2726
3118
|
}
|
|
2727
3119
|
ensureFetchClientDiagnostics(fetchId, serviceName, url) {
|
|
3120
|
+
const now = Date.now();
|
|
3121
|
+
this.pruneFetchClientDiagnostics(now);
|
|
2728
3122
|
let state = this.fetchClientDiagnostics.get(fetchId);
|
|
2729
3123
|
if (!state) {
|
|
2730
3124
|
state = {
|
|
@@ -2744,13 +3138,14 @@ var RestController = class _RestController {
|
|
|
2744
3138
|
signalFailures: 0,
|
|
2745
3139
|
statusChecks: 0,
|
|
2746
3140
|
statusFailures: 0,
|
|
2747
|
-
updatedAt:
|
|
3141
|
+
updatedAt: now
|
|
2748
3142
|
};
|
|
2749
3143
|
this.fetchClientDiagnostics.set(fetchId, state);
|
|
2750
3144
|
} else {
|
|
2751
3145
|
state.serviceName = serviceName;
|
|
2752
3146
|
state.url = url;
|
|
2753
3147
|
}
|
|
3148
|
+
this.pruneFetchClientDiagnostics(now);
|
|
2754
3149
|
return state;
|
|
2755
3150
|
}
|
|
2756
3151
|
getErrorMessage(error) {
|
|
@@ -2782,6 +3177,7 @@ var RestController = class _RestController {
|
|
|
2782
3177
|
}
|
|
2783
3178
|
}
|
|
2784
3179
|
collectFetchTransportDiagnostics(ctx) {
|
|
3180
|
+
this.pruneFetchClientDiagnostics();
|
|
2785
3181
|
const { detailLevel, includeErrorHistory, errorHistoryLimit } = this.resolveTransportDiagnosticsOptions(ctx);
|
|
2786
3182
|
const serviceName = CadenzaService.serviceRegistry.serviceName ?? "UnknownService";
|
|
2787
3183
|
const states = Array.from(this.fetchClientDiagnostics.values()).sort(
|
|
@@ -2898,55 +3294,225 @@ var waitForSocketConnection = async (socket, timeoutMs, createError) => {
|
|
|
2898
3294
|
|
|
2899
3295
|
// src/network/SocketController.ts
|
|
2900
3296
|
var SocketController = class _SocketController {
|
|
2901
|
-
/**
|
|
2902
|
-
* Constructs the `SocketServer`, setting up a WebSocket server with specific configurations,
|
|
2903
|
-
* including connection state recovery, rate limiting, CORS handling, and custom event handling.
|
|
2904
|
-
* This class sets up the communication infrastructure for scalable, resilient, and secure WebSocket-based interactions
|
|
2905
|
-
* using metadata-driven task execution with `Cadenza`.
|
|
2906
|
-
*
|
|
2907
|
-
* It provides support for:
|
|
2908
|
-
* - Origin-based access control for connections.
|
|
2909
|
-
* - Optional payload sanitization.
|
|
2910
|
-
* - Configurable rate limiting and behavior on limit breaches (soft/hard disconnects).
|
|
2911
|
-
* - Event handlers for connection, handshake, delegation, signaling, status checks, and disconnection.
|
|
2912
|
-
*
|
|
2913
|
-
* The server can handle both internal and external interactions depending on the provided configurations,
|
|
2914
|
-
* and integrates directly with Cadenza's task workflow engine.
|
|
2915
|
-
*
|
|
2916
|
-
* Initializes the `SocketServer` to be ready for WebSocket communication.
|
|
2917
|
-
*/
|
|
2918
3297
|
constructor() {
|
|
2919
|
-
this.socketClientDiagnostics = /* @__PURE__ */ new Map();
|
|
2920
3298
|
this.diagnosticsErrorHistoryLimit = 100;
|
|
3299
|
+
this.diagnosticsMaxClientEntries = 500;
|
|
3300
|
+
this.destroyedDiagnosticsTtlMs = 15 * 6e4;
|
|
3301
|
+
this.socketServerDefaultKey = "socket-server-default";
|
|
3302
|
+
this.socketServerActor = CadenzaService.createActor(
|
|
3303
|
+
{
|
|
3304
|
+
name: "SocketServerActor",
|
|
3305
|
+
description: "Holds durable socket server session state and runtime socket server handle",
|
|
3306
|
+
defaultKey: this.socketServerDefaultKey,
|
|
3307
|
+
keyResolver: (input) => this.resolveSocketServerKey(input),
|
|
3308
|
+
loadPolicy: "lazy",
|
|
3309
|
+
writeContract: "overwrite",
|
|
3310
|
+
initState: this.createInitialSocketServerSessionState(
|
|
3311
|
+
this.socketServerDefaultKey
|
|
3312
|
+
)
|
|
3313
|
+
},
|
|
3314
|
+
{ isMeta: true }
|
|
3315
|
+
);
|
|
3316
|
+
this.socketClientActor = CadenzaService.createActor(
|
|
3317
|
+
{
|
|
3318
|
+
name: "SocketClientActor",
|
|
3319
|
+
description: "Holds durable socket client session state and runtime socket connection handles",
|
|
3320
|
+
defaultKey: "socket-client-default",
|
|
3321
|
+
keyResolver: (input) => this.resolveSocketClientFetchId(input),
|
|
3322
|
+
loadPolicy: "lazy",
|
|
3323
|
+
writeContract: "overwrite",
|
|
3324
|
+
initState: this.createInitialSocketClientSessionState()
|
|
3325
|
+
},
|
|
3326
|
+
{ isMeta: true }
|
|
3327
|
+
);
|
|
3328
|
+
this.socketClientDiagnosticsActor = CadenzaService.createActor(
|
|
3329
|
+
{
|
|
3330
|
+
name: "SocketClientDiagnosticsActor",
|
|
3331
|
+
description: "Tracks socket client diagnostics snapshots per fetchId for transport observability",
|
|
3332
|
+
defaultKey: "socket-client-diagnostics",
|
|
3333
|
+
loadPolicy: "eager",
|
|
3334
|
+
writeContract: "overwrite",
|
|
3335
|
+
initState: {
|
|
3336
|
+
entries: {}
|
|
3337
|
+
}
|
|
3338
|
+
},
|
|
3339
|
+
{ isMeta: true }
|
|
3340
|
+
);
|
|
3341
|
+
this.registerDiagnosticsTasks();
|
|
3342
|
+
this.registerSocketServerTasks();
|
|
3343
|
+
this.registerSocketClientTasks();
|
|
2921
3344
|
CadenzaService.createMetaTask(
|
|
2922
3345
|
"Collect socket transport diagnostics",
|
|
2923
|
-
|
|
3346
|
+
this.socketClientDiagnosticsActor.task(
|
|
3347
|
+
({ state, input }) => this.collectSocketTransportDiagnostics(input, state.entries),
|
|
3348
|
+
{ mode: "read" }
|
|
3349
|
+
),
|
|
2924
3350
|
"Responds to distributed transport diagnostics inquiries with socket client data."
|
|
2925
3351
|
).respondsTo(META_RUNTIME_TRANSPORT_DIAGNOSTICS_INTENT);
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
3352
|
+
}
|
|
3353
|
+
static get instance() {
|
|
3354
|
+
if (!this._instance) this._instance = new _SocketController();
|
|
3355
|
+
return this._instance;
|
|
3356
|
+
}
|
|
3357
|
+
registerDiagnosticsTasks() {
|
|
3358
|
+
CadenzaService.createThrottledMetaTask(
|
|
3359
|
+
"SocketClientDiagnosticsActor.Upsert",
|
|
3360
|
+
this.socketClientDiagnosticsActor.task(
|
|
3361
|
+
({ state, input, setState }) => {
|
|
3362
|
+
const fetchId = String(input.fetchId ?? "").trim();
|
|
3363
|
+
if (!fetchId) {
|
|
2931
3364
|
return;
|
|
2932
3365
|
}
|
|
2933
|
-
const
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
3366
|
+
const now = Date.now();
|
|
3367
|
+
const entries = { ...state.entries };
|
|
3368
|
+
const existing = entries[fetchId];
|
|
3369
|
+
const base = existing ? {
|
|
3370
|
+
...existing,
|
|
3371
|
+
errorHistory: [...existing.errorHistory]
|
|
3372
|
+
} : {
|
|
3373
|
+
fetchId,
|
|
3374
|
+
serviceName: String(input.serviceName ?? ""),
|
|
3375
|
+
url: String(input.url ?? ""),
|
|
3376
|
+
socketId: null,
|
|
3377
|
+
connected: false,
|
|
3378
|
+
handshake: false,
|
|
3379
|
+
reconnectAttempts: 0,
|
|
3380
|
+
connectErrors: 0,
|
|
3381
|
+
reconnectErrors: 0,
|
|
3382
|
+
socketErrors: 0,
|
|
3383
|
+
pendingDelegations: 0,
|
|
3384
|
+
pendingTimers: 0,
|
|
3385
|
+
destroyed: false,
|
|
3386
|
+
lastHandshakeAt: null,
|
|
3387
|
+
lastHandshakeError: null,
|
|
3388
|
+
lastDisconnectAt: null,
|
|
3389
|
+
lastError: null,
|
|
3390
|
+
lastErrorAt: 0,
|
|
3391
|
+
errorHistory: [],
|
|
3392
|
+
updatedAt: now
|
|
3393
|
+
};
|
|
3394
|
+
if (input.serviceName !== void 0) {
|
|
3395
|
+
base.serviceName = String(input.serviceName);
|
|
3396
|
+
}
|
|
3397
|
+
if (input.url !== void 0) {
|
|
3398
|
+
base.url = String(input.url);
|
|
3399
|
+
}
|
|
3400
|
+
const patch = input.patch && typeof input.patch === "object" ? input.patch : {};
|
|
3401
|
+
Object.assign(base, patch);
|
|
3402
|
+
base.fetchId = fetchId;
|
|
3403
|
+
base.updatedAt = now;
|
|
3404
|
+
const errorMessage = input.error !== void 0 ? this.getErrorMessage(input.error) : void 0;
|
|
3405
|
+
if (errorMessage) {
|
|
3406
|
+
base.lastError = errorMessage;
|
|
3407
|
+
base.lastErrorAt = now;
|
|
3408
|
+
base.errorHistory.push({
|
|
3409
|
+
at: new Date(now).toISOString(),
|
|
3410
|
+
message: errorMessage
|
|
3411
|
+
});
|
|
3412
|
+
if (base.errorHistory.length > this.diagnosticsErrorHistoryLimit) {
|
|
3413
|
+
base.errorHistory.splice(
|
|
3414
|
+
0,
|
|
3415
|
+
base.errorHistory.length - this.diagnosticsErrorHistoryLimit
|
|
3416
|
+
);
|
|
2943
3417
|
}
|
|
3418
|
+
}
|
|
3419
|
+
entries[fetchId] = base;
|
|
3420
|
+
this.pruneDiagnosticsEntries(entries, now);
|
|
3421
|
+
setState({ entries });
|
|
3422
|
+
},
|
|
3423
|
+
{ mode: "write" }
|
|
3424
|
+
),
|
|
3425
|
+
(context) => String(context?.fetchId ?? "default"),
|
|
3426
|
+
"Upserts socket client diagnostics in actor state."
|
|
3427
|
+
).doOn("meta.socket_client.diagnostics_upsert_requested");
|
|
3428
|
+
}
|
|
3429
|
+
registerSocketServerTasks() {
|
|
3430
|
+
CadenzaService.createThrottledMetaTask(
|
|
3431
|
+
"SocketServerActor.PatchSession",
|
|
3432
|
+
this.socketServerActor.task(
|
|
3433
|
+
({ state, input, setState }) => {
|
|
3434
|
+
const patch = input.patch && typeof input.patch === "object" ? input.patch : {};
|
|
3435
|
+
setState({
|
|
3436
|
+
...state,
|
|
3437
|
+
...patch,
|
|
3438
|
+
updatedAt: Date.now()
|
|
3439
|
+
});
|
|
3440
|
+
},
|
|
3441
|
+
{ mode: "write" }
|
|
3442
|
+
),
|
|
3443
|
+
(context) => String(context?.serverKey ?? this.socketServerDefaultKey),
|
|
3444
|
+
"Applies partial durable session updates for socket server actor."
|
|
3445
|
+
).doOn("meta.socket_server.session_patch_requested");
|
|
3446
|
+
CadenzaService.createMetaTask(
|
|
3447
|
+
"SocketServerActor.ClearRuntime",
|
|
3448
|
+
this.socketServerActor.task(
|
|
3449
|
+
({ setRuntimeState }) => {
|
|
3450
|
+
setRuntimeState(null);
|
|
3451
|
+
},
|
|
3452
|
+
{ mode: "write" }
|
|
3453
|
+
),
|
|
3454
|
+
"Clears socket server runtime handle after shutdown."
|
|
3455
|
+
).doOn("meta.socket_server.runtime_clear_requested");
|
|
3456
|
+
const setupSocketServerTask = CadenzaService.createMetaTask(
|
|
3457
|
+
"Setup SocketServer",
|
|
3458
|
+
this.socketServerActor.task(
|
|
3459
|
+
({ state, runtimeState, input, actor, setState, setRuntimeState, emit }) => {
|
|
3460
|
+
const serverKey = this.resolveSocketServerKey(input) ?? actor.key ?? this.socketServerDefaultKey;
|
|
3461
|
+
const shouldUseSocket = Boolean(input.__useSocket);
|
|
3462
|
+
if (!shouldUseSocket) {
|
|
3463
|
+
this.destroySocketServerRuntimeHandle(runtimeState);
|
|
3464
|
+
setRuntimeState(null);
|
|
3465
|
+
setState({
|
|
3466
|
+
...state,
|
|
3467
|
+
serverKey,
|
|
3468
|
+
useSocket: false,
|
|
3469
|
+
status: "inactive",
|
|
3470
|
+
connectionCount: 0,
|
|
3471
|
+
lastShutdownAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3472
|
+
updatedAt: Date.now()
|
|
3473
|
+
});
|
|
3474
|
+
return;
|
|
3475
|
+
}
|
|
3476
|
+
let runtimeHandle = runtimeState;
|
|
3477
|
+
if (!runtimeHandle) {
|
|
3478
|
+
runtimeHandle = this.createSocketServerRuntimeHandleFromContext(input);
|
|
3479
|
+
setRuntimeState(runtimeHandle);
|
|
3480
|
+
}
|
|
3481
|
+
const profile = String(input.__securityProfile ?? state.securityProfile ?? "medium");
|
|
3482
|
+
const networkType = String(input.__networkType ?? state.networkType ?? "internal");
|
|
3483
|
+
const schedulePatch = (patch) => {
|
|
3484
|
+
CadenzaService.emit("meta.socket_server.session_patch_requested", {
|
|
3485
|
+
serverKey,
|
|
3486
|
+
patch
|
|
3487
|
+
});
|
|
3488
|
+
};
|
|
3489
|
+
if (runtimeHandle.initialized) {
|
|
3490
|
+
schedulePatch({
|
|
3491
|
+
status: "active",
|
|
3492
|
+
useSocket: true,
|
|
3493
|
+
securityProfile: profile,
|
|
3494
|
+
networkType,
|
|
3495
|
+
connectionCount: runtimeHandle.connectedSocketIds.size,
|
|
3496
|
+
lastStartedAt: state.lastStartedAt ?? (/* @__PURE__ */ new Date()).toISOString()
|
|
3497
|
+
});
|
|
3498
|
+
return;
|
|
3499
|
+
}
|
|
3500
|
+
const server = runtimeHandle.server;
|
|
3501
|
+
runtimeHandle.initialized = true;
|
|
3502
|
+
setState({
|
|
3503
|
+
...state,
|
|
3504
|
+
serverKey,
|
|
3505
|
+
useSocket: true,
|
|
3506
|
+
status: "active",
|
|
3507
|
+
securityProfile: profile,
|
|
3508
|
+
networkType,
|
|
3509
|
+
connectionCount: runtimeHandle.connectedSocketIds.size,
|
|
3510
|
+
lastStartedAt: state.lastStartedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
3511
|
+
updatedAt: Date.now()
|
|
2944
3512
|
});
|
|
2945
|
-
const profile = ctx.__securityProfile ?? "medium";
|
|
2946
3513
|
server.use((socket, next) => {
|
|
2947
3514
|
const origin = socket?.handshake?.headers?.origin;
|
|
2948
3515
|
const allowedOrigins = ["*"];
|
|
2949
|
-
const networkType = ctx.__networkType ?? "internal";
|
|
2950
3516
|
let effectiveOrigin = origin || "unknown";
|
|
2951
3517
|
if (networkType === "internal") effectiveOrigin = "internal";
|
|
2952
3518
|
if (profile !== "low" && !allowedOrigins.includes(effectiveOrigin) && !allowedOrigins.includes("*")) {
|
|
@@ -2957,7 +3523,9 @@ var SocketController = class _SocketController {
|
|
|
2957
3523
|
medium: { points: 1e4, duration: 10 },
|
|
2958
3524
|
high: { points: 1e3, duration: 60, blockDuration: 300 }
|
|
2959
3525
|
};
|
|
2960
|
-
const limiter = new RateLimiterMemory2(
|
|
3526
|
+
const limiter = new RateLimiterMemory2(
|
|
3527
|
+
limiterOptions[profile] ?? limiterOptions.medium
|
|
3528
|
+
);
|
|
2961
3529
|
const clientKey = socket?.handshake?.address || "unknown";
|
|
2962
3530
|
socket.use((packet, packetNext) => {
|
|
2963
3531
|
limiter.consume(clientKey).then(() => packetNext()).catch((rej) => {
|
|
@@ -2992,116 +3560,111 @@ var SocketController = class _SocketController {
|
|
|
2992
3560
|
});
|
|
2993
3561
|
next();
|
|
2994
3562
|
});
|
|
2995
|
-
if (!server) {
|
|
2996
|
-
CadenzaService.log("Socket setup error: No server", {}, "error");
|
|
2997
|
-
return { ...ctx, __error: "No server", errored: true };
|
|
2998
|
-
}
|
|
2999
3563
|
server.on("connection", (ws) => {
|
|
3564
|
+
runtimeHandle.connectedSocketIds.add(ws.id);
|
|
3565
|
+
schedulePatch({
|
|
3566
|
+
connectionCount: runtimeHandle.connectedSocketIds.size,
|
|
3567
|
+
lastConnectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3568
|
+
status: "active"
|
|
3569
|
+
});
|
|
3000
3570
|
try {
|
|
3001
|
-
ws.on(
|
|
3002
|
-
"
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
}
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
if (ctx3.__routineExecId) {
|
|
3022
|
-
emit(
|
|
3023
|
-
`meta.socket_client.transmitted:${ctx3.__routineExecId}`,
|
|
3024
|
-
{}
|
|
3025
|
-
);
|
|
3026
|
-
}
|
|
3571
|
+
ws.on("handshake", (ctx, callback) => {
|
|
3572
|
+
CadenzaService.log("SocketServer: New connection", {
|
|
3573
|
+
...ctx,
|
|
3574
|
+
socketId: ws.id
|
|
3575
|
+
});
|
|
3576
|
+
callback({
|
|
3577
|
+
status: "success",
|
|
3578
|
+
serviceName: CadenzaService.serviceRegistry.serviceName
|
|
3579
|
+
});
|
|
3580
|
+
if (ctx.isFrontend) {
|
|
3581
|
+
const fetchId = `browser:${ctx.serviceInstanceId}`;
|
|
3582
|
+
CadenzaService.createMetaTask(
|
|
3583
|
+
`Transmit signal to ${fetchId}`,
|
|
3584
|
+
(c, emitter) => {
|
|
3585
|
+
if (c.__signalName === void 0) {
|
|
3586
|
+
return;
|
|
3587
|
+
}
|
|
3588
|
+
ws.emit("signal", c);
|
|
3589
|
+
if (c.__routineExecId) {
|
|
3590
|
+
emitter(`meta.socket_client.transmitted:${c.__routineExecId}`, {});
|
|
3027
3591
|
}
|
|
3028
|
-
).doOn(
|
|
3029
|
-
`meta.service_registry.selected_instance_for_socket:${fetchId}`
|
|
3030
|
-
).attachSignal("meta.socket_client.transmitted");
|
|
3031
|
-
}
|
|
3032
|
-
CadenzaService.emit("meta.socket.handshake", ctx2);
|
|
3033
|
-
}
|
|
3034
|
-
);
|
|
3035
|
-
ws.on(
|
|
3036
|
-
"delegation",
|
|
3037
|
-
(ctx2, callback) => {
|
|
3038
|
-
const deputyExecId = ctx2.__metadata.__deputyExecId;
|
|
3039
|
-
CadenzaService.createEphemeralMetaTask(
|
|
3040
|
-
"Resolve delegation",
|
|
3041
|
-
(ctx3) => {
|
|
3042
|
-
callback(ctx3);
|
|
3043
|
-
},
|
|
3044
|
-
"Resolves a delegation request using the provided callback from the client (.emitWithAck())",
|
|
3045
|
-
{ register: false }
|
|
3046
|
-
).doOn(`meta.node.graph_completed:${deputyExecId}`).emits(`meta.socket.delegation_resolved:${deputyExecId}`);
|
|
3047
|
-
CadenzaService.createEphemeralMetaTask(
|
|
3048
|
-
"Delegation progress update",
|
|
3049
|
-
(ctx3) => {
|
|
3050
|
-
if (ctx3.__progress !== void 0)
|
|
3051
|
-
ws.emit("delegation_progress", ctx3);
|
|
3052
3592
|
},
|
|
3053
|
-
"
|
|
3054
|
-
|
|
3055
|
-
once: false,
|
|
3056
|
-
destroyCondition: (ctx3) => ctx3.data.progress === 1 || ctx3.data?.progress === void 0,
|
|
3057
|
-
register: false
|
|
3058
|
-
}
|
|
3059
|
-
).doOn(
|
|
3060
|
-
`meta.node.routine_execution_progress:${deputyExecId}`,
|
|
3061
|
-
`meta.node.graph_completed:${deputyExecId}`
|
|
3062
|
-
).emitsOnFail(`meta.socket.progress_failed:${deputyExecId}`);
|
|
3063
|
-
CadenzaService.emit("meta.socket.delegation_requested", {
|
|
3064
|
-
...ctx2,
|
|
3065
|
-
__name: ctx2.__remoteRoutineName
|
|
3066
|
-
});
|
|
3593
|
+
"Transmit frontend bound signal through active websocket."
|
|
3594
|
+
).doOn(`meta.service_registry.selected_instance_for_socket:${fetchId}`).attachSignal("meta.socket_client.transmitted");
|
|
3067
3595
|
}
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3596
|
+
CadenzaService.emit("meta.socket.handshake", ctx);
|
|
3597
|
+
});
|
|
3598
|
+
ws.on("delegation", (ctx, callback) => {
|
|
3599
|
+
const deputyExecId = ctx.__metadata.__deputyExecId;
|
|
3600
|
+
CadenzaService.createEphemeralMetaTask(
|
|
3601
|
+
"Resolve delegation",
|
|
3602
|
+
(delegationCtx) => {
|
|
3603
|
+
callback(delegationCtx);
|
|
3604
|
+
},
|
|
3605
|
+
"Resolves a delegation request using client callback.",
|
|
3606
|
+
{ register: false }
|
|
3607
|
+
).doOn(`meta.node.graph_completed:${deputyExecId}`).emits(`meta.socket.delegation_resolved:${deputyExecId}`);
|
|
3608
|
+
CadenzaService.createEphemeralMetaTask(
|
|
3609
|
+
"Delegation progress update",
|
|
3610
|
+
(progressCtx) => {
|
|
3611
|
+
if (progressCtx.__progress !== void 0) {
|
|
3612
|
+
ws.emit("delegation_progress", progressCtx);
|
|
3613
|
+
}
|
|
3614
|
+
},
|
|
3615
|
+
"Updates delegation progress to client.",
|
|
3616
|
+
{
|
|
3617
|
+
once: false,
|
|
3618
|
+
destroyCondition: (progressCtx) => progressCtx.data.progress === 1 || progressCtx.data?.progress === void 0,
|
|
3619
|
+
register: false
|
|
3089
3620
|
}
|
|
3621
|
+
).doOn(
|
|
3622
|
+
`meta.node.routine_execution_progress:${deputyExecId}`,
|
|
3623
|
+
`meta.node.graph_completed:${deputyExecId}`
|
|
3624
|
+
).emitsOnFail(`meta.socket.progress_failed:${deputyExecId}`);
|
|
3625
|
+
CadenzaService.emit("meta.socket.delegation_requested", {
|
|
3626
|
+
...ctx,
|
|
3627
|
+
__name: ctx.__remoteRoutineName
|
|
3628
|
+
});
|
|
3629
|
+
});
|
|
3630
|
+
ws.on("signal", (ctx, callback) => {
|
|
3631
|
+
if (CadenzaService.signalBroker.listObservedSignals().includes(ctx.__signalName)) {
|
|
3632
|
+
callback({
|
|
3633
|
+
__status: "success",
|
|
3634
|
+
__signalName: ctx.__signalName
|
|
3635
|
+
});
|
|
3636
|
+
CadenzaService.emit(ctx.__signalName, ctx);
|
|
3637
|
+
} else {
|
|
3638
|
+
CadenzaService.log(
|
|
3639
|
+
`No such signal ${ctx.__signalName} on ${ctx.__serviceName}`,
|
|
3640
|
+
"warning"
|
|
3641
|
+
);
|
|
3642
|
+
callback({
|
|
3643
|
+
...ctx,
|
|
3644
|
+
__status: "error",
|
|
3645
|
+
__error: `No such signal: ${ctx.__signalName}`,
|
|
3646
|
+
errored: true
|
|
3647
|
+
});
|
|
3090
3648
|
}
|
|
3091
|
-
);
|
|
3649
|
+
});
|
|
3092
3650
|
ws.on(
|
|
3093
3651
|
"status_check",
|
|
3094
|
-
(
|
|
3652
|
+
(ctx, callback) => {
|
|
3095
3653
|
CadenzaService.createEphemeralMetaTask(
|
|
3096
3654
|
"Resolve status check",
|
|
3097
3655
|
callback,
|
|
3098
3656
|
"Resolves a status check request",
|
|
3099
3657
|
{ register: false }
|
|
3100
3658
|
).doAfter(CadenzaService.serviceRegistry.getStatusTask);
|
|
3101
|
-
CadenzaService.emit("meta.socket.status_check_requested",
|
|
3659
|
+
CadenzaService.emit("meta.socket.status_check_requested", ctx);
|
|
3102
3660
|
}
|
|
3103
3661
|
);
|
|
3104
3662
|
ws.on("disconnect", () => {
|
|
3663
|
+
runtimeHandle.connectedSocketIds.delete(ws.id);
|
|
3664
|
+
schedulePatch({
|
|
3665
|
+
connectionCount: runtimeHandle.connectedSocketIds.size,
|
|
3666
|
+
lastDisconnectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3667
|
+
});
|
|
3105
3668
|
CadenzaService.log(
|
|
3106
3669
|
"Socket client disconnected",
|
|
3107
3670
|
{ socketId: ws.id },
|
|
@@ -3111,514 +3674,888 @@ var SocketController = class _SocketController {
|
|
|
3111
3674
|
__wsId: ws.id
|
|
3112
3675
|
});
|
|
3113
3676
|
});
|
|
3114
|
-
} catch (
|
|
3677
|
+
} catch (error) {
|
|
3115
3678
|
CadenzaService.log(
|
|
3116
3679
|
"SocketServer: Error in socket event",
|
|
3117
|
-
{ error
|
|
3680
|
+
{ error },
|
|
3118
3681
|
"error"
|
|
3119
3682
|
);
|
|
3120
3683
|
}
|
|
3121
3684
|
CadenzaService.emit("meta.socket.connected", { __wsId: ws.id });
|
|
3122
3685
|
});
|
|
3123
|
-
CadenzaService.createMetaTask(
|
|
3124
|
-
|
|
3125
|
-
(
|
|
3686
|
+
runtimeHandle.broadcastStatusTask = CadenzaService.createMetaTask(
|
|
3687
|
+
`Broadcast status ${serverKey}`,
|
|
3688
|
+
(ctx) => server.emit("status_update", ctx),
|
|
3126
3689
|
"Broadcasts the status of the server to all clients"
|
|
3127
3690
|
).doOn("meta.service.updated");
|
|
3128
|
-
CadenzaService.createMetaTask(
|
|
3129
|
-
|
|
3130
|
-
() =>
|
|
3691
|
+
runtimeHandle.shutdownTask = CadenzaService.createMetaTask(
|
|
3692
|
+
`Shutdown SocketServer ${serverKey}`,
|
|
3693
|
+
async () => {
|
|
3694
|
+
this.destroySocketServerRuntimeHandle(runtimeHandle);
|
|
3695
|
+
CadenzaService.emit("meta.socket_server.runtime_clear_requested", {
|
|
3696
|
+
serverKey
|
|
3697
|
+
});
|
|
3698
|
+
CadenzaService.emit("meta.socket_server.session_patch_requested", {
|
|
3699
|
+
serverKey,
|
|
3700
|
+
patch: {
|
|
3701
|
+
useSocket: false,
|
|
3702
|
+
status: "shutdown",
|
|
3703
|
+
connectionCount: 0,
|
|
3704
|
+
lastShutdownAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3705
|
+
}
|
|
3706
|
+
});
|
|
3707
|
+
},
|
|
3131
3708
|
"Shuts down the socket server"
|
|
3132
3709
|
).doOn("meta.socket_server_shutdown_requested").emits("meta.socket.shutdown");
|
|
3133
|
-
return
|
|
3134
|
-
}
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3710
|
+
return true;
|
|
3711
|
+
},
|
|
3712
|
+
{ mode: "write" }
|
|
3713
|
+
),
|
|
3714
|
+
"Initializes socket server runtime through actor state."
|
|
3715
|
+
);
|
|
3716
|
+
setupSocketServerTask.doOn("global.meta.rest.network_configured");
|
|
3717
|
+
}
|
|
3718
|
+
registerSocketClientTasks() {
|
|
3719
|
+
CadenzaService.createThrottledMetaTask(
|
|
3720
|
+
"SocketClientActor.ApplySessionOperation",
|
|
3721
|
+
this.socketClientActor.task(
|
|
3722
|
+
({ state, input, setState }) => {
|
|
3723
|
+
const operation = String(
|
|
3724
|
+
input.operation ?? "transmit"
|
|
3725
|
+
);
|
|
3726
|
+
const patch = input.patch && typeof input.patch === "object" ? input.patch : {};
|
|
3727
|
+
let next = {
|
|
3728
|
+
...state,
|
|
3729
|
+
...patch,
|
|
3730
|
+
communicationTypes: patch.communicationTypes !== void 0 ? this.normalizeCommunicationTypes(patch.communicationTypes) : state.communicationTypes,
|
|
3731
|
+
updatedAt: Date.now()
|
|
3732
|
+
};
|
|
3733
|
+
if (input.serviceName !== void 0) {
|
|
3734
|
+
next.serviceName = String(input.serviceName);
|
|
3735
|
+
}
|
|
3736
|
+
if (input.serviceAddress !== void 0) {
|
|
3737
|
+
next.serviceAddress = String(input.serviceAddress);
|
|
3738
|
+
}
|
|
3739
|
+
if (input.serviceInstanceId !== void 0) {
|
|
3740
|
+
next.serviceInstanceId = String(input.serviceInstanceId);
|
|
3741
|
+
}
|
|
3742
|
+
if (input.protocol !== void 0) {
|
|
3743
|
+
next.protocol = String(input.protocol);
|
|
3744
|
+
}
|
|
3745
|
+
if (input.url !== void 0) {
|
|
3746
|
+
next.url = String(input.url);
|
|
3747
|
+
}
|
|
3748
|
+
if (input.servicePort !== void 0) {
|
|
3749
|
+
next.servicePort = Number(input.servicePort);
|
|
3750
|
+
}
|
|
3751
|
+
if (input.fetchId !== void 0) {
|
|
3752
|
+
next.fetchId = String(input.fetchId);
|
|
3753
|
+
}
|
|
3754
|
+
if (operation === "connect") {
|
|
3755
|
+
next.destroyed = false;
|
|
3756
|
+
} else if (operation === "handshake") {
|
|
3757
|
+
next.destroyed = false;
|
|
3758
|
+
next.connected = patch.connected ?? true;
|
|
3759
|
+
next.handshake = patch.handshake ?? true;
|
|
3760
|
+
} else if (operation === "shutdown") {
|
|
3761
|
+
next.connected = false;
|
|
3762
|
+
next.handshake = false;
|
|
3763
|
+
next.destroyed = true;
|
|
3764
|
+
next.pendingDelegations = 0;
|
|
3765
|
+
next.pendingTimers = 0;
|
|
3766
|
+
}
|
|
3767
|
+
setState(next);
|
|
3768
|
+
return next;
|
|
3769
|
+
},
|
|
3770
|
+
{ mode: "write" }
|
|
3771
|
+
),
|
|
3772
|
+
(context) => String(this.resolveSocketClientFetchId(context ?? {}) ?? "default"),
|
|
3773
|
+
"Applies socket client session operation patch in actor durable state."
|
|
3774
|
+
).doOn("meta.socket_client.session_operation_requested");
|
|
3775
|
+
CadenzaService.createMetaTask(
|
|
3776
|
+
"SocketClientActor.ClearRuntime",
|
|
3777
|
+
this.socketClientActor.task(
|
|
3778
|
+
({ setRuntimeState }) => {
|
|
3779
|
+
setRuntimeState(null);
|
|
3780
|
+
},
|
|
3781
|
+
{ mode: "write" }
|
|
3782
|
+
),
|
|
3783
|
+
"Clears socket client runtime handle."
|
|
3784
|
+
).doOn("meta.socket_client.runtime_clear_requested");
|
|
3138
3785
|
CadenzaService.createMetaTask(
|
|
3139
3786
|
"Connect to socket server",
|
|
3140
|
-
(
|
|
3141
|
-
|
|
3142
|
-
serviceInstanceId
|
|
3143
|
-
communicationTypes
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
fetchId,
|
|
3155
|
-
serviceName,
|
|
3156
|
-
URL
|
|
3157
|
-
);
|
|
3158
|
-
socketDiagnostics.destroyed = false;
|
|
3159
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3160
|
-
let handshake = false;
|
|
3161
|
-
let errorCount = 0;
|
|
3162
|
-
const ERROR_LIMIT = 5;
|
|
3163
|
-
if (CadenzaService.get(`Socket handshake with ${URL}`)) {
|
|
3164
|
-
console.error("Socket client already exists", URL);
|
|
3165
|
-
return;
|
|
3166
|
-
}
|
|
3167
|
-
const pendingDelegationIds = /* @__PURE__ */ new Set();
|
|
3168
|
-
const pendingTimers = /* @__PURE__ */ new Set();
|
|
3169
|
-
const syncPendingCounts = () => {
|
|
3170
|
-
socketDiagnostics.pendingDelegations = pendingDelegationIds.size;
|
|
3171
|
-
socketDiagnostics.pendingTimers = pendingTimers.size;
|
|
3172
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3173
|
-
};
|
|
3174
|
-
let handshakeTask = null;
|
|
3175
|
-
let emitWhenReady = null;
|
|
3176
|
-
let transmitTask = null;
|
|
3177
|
-
let delegateTask = null;
|
|
3178
|
-
let socket = null;
|
|
3179
|
-
socket = io(URL, {
|
|
3180
|
-
reconnection: true,
|
|
3181
|
-
reconnectionAttempts: 5,
|
|
3182
|
-
reconnectionDelay: 2e3,
|
|
3183
|
-
reconnectionDelayMax: 1e4,
|
|
3184
|
-
randomizationFactor: 0.5,
|
|
3185
|
-
transports: ["websocket"],
|
|
3186
|
-
autoConnect: false
|
|
3187
|
-
});
|
|
3188
|
-
emitWhenReady = (event, data, timeoutMs = 6e4, ack) => {
|
|
3189
|
-
return new Promise((resolve) => {
|
|
3190
|
-
const resolveWithError = (errorMessage, fallbackError) => {
|
|
3191
|
-
resolve({
|
|
3192
|
-
...data,
|
|
3193
|
-
errored: true,
|
|
3194
|
-
__error: errorMessage,
|
|
3195
|
-
error: fallbackError instanceof Error ? fallbackError.message : errorMessage,
|
|
3196
|
-
socketId: socket?.id,
|
|
3787
|
+
this.socketClientActor.task(
|
|
3788
|
+
({ state, runtimeState, input, setState, setRuntimeState, emit }) => {
|
|
3789
|
+
const serviceInstanceId = String(input.serviceInstanceId ?? "");
|
|
3790
|
+
const communicationTypes = this.normalizeCommunicationTypes(
|
|
3791
|
+
input.communicationTypes
|
|
3792
|
+
);
|
|
3793
|
+
const serviceName = String(input.serviceName ?? "");
|
|
3794
|
+
const serviceAddress = String(input.serviceAddress ?? "");
|
|
3795
|
+
const protocol = String(input.protocol ?? "http");
|
|
3796
|
+
const normalizedPort = this.resolveServicePort(protocol, input.servicePort);
|
|
3797
|
+
if (!serviceAddress || !normalizedPort) {
|
|
3798
|
+
CadenzaService.log(
|
|
3799
|
+
"Socket client setup skipped due to missing address/port",
|
|
3800
|
+
{
|
|
3197
3801
|
serviceName,
|
|
3198
|
-
|
|
3802
|
+
serviceAddress,
|
|
3803
|
+
servicePort: input.servicePort,
|
|
3804
|
+
protocol
|
|
3805
|
+
},
|
|
3806
|
+
"warning"
|
|
3807
|
+
);
|
|
3808
|
+
return false;
|
|
3809
|
+
}
|
|
3810
|
+
const socketProtocol = protocol === "https" ? "wss" : "ws";
|
|
3811
|
+
const url = `${socketProtocol}://${serviceAddress}:${normalizedPort}`;
|
|
3812
|
+
const fetchId = `${serviceAddress}_${normalizedPort}`;
|
|
3813
|
+
const applySessionOperation = (operation, patch = {}) => {
|
|
3814
|
+
CadenzaService.emit("meta.socket_client.session_operation_requested", {
|
|
3815
|
+
fetchId,
|
|
3816
|
+
operation,
|
|
3817
|
+
patch,
|
|
3818
|
+
serviceInstanceId,
|
|
3819
|
+
communicationTypes,
|
|
3820
|
+
serviceName,
|
|
3821
|
+
serviceAddress,
|
|
3822
|
+
servicePort: normalizedPort,
|
|
3823
|
+
protocol,
|
|
3824
|
+
url
|
|
3825
|
+
});
|
|
3826
|
+
};
|
|
3827
|
+
const upsertDiagnostics = (patch, error) => {
|
|
3828
|
+
CadenzaService.emit("meta.socket_client.diagnostics_upsert_requested", {
|
|
3829
|
+
fetchId,
|
|
3830
|
+
serviceName,
|
|
3831
|
+
url,
|
|
3832
|
+
patch,
|
|
3833
|
+
error
|
|
3834
|
+
});
|
|
3835
|
+
};
|
|
3836
|
+
setState({
|
|
3837
|
+
...state,
|
|
3838
|
+
fetchId,
|
|
3839
|
+
serviceInstanceId,
|
|
3840
|
+
communicationTypes,
|
|
3841
|
+
serviceName,
|
|
3842
|
+
serviceAddress,
|
|
3843
|
+
servicePort: normalizedPort,
|
|
3844
|
+
protocol,
|
|
3845
|
+
url,
|
|
3846
|
+
destroyed: false,
|
|
3847
|
+
updatedAt: Date.now()
|
|
3848
|
+
});
|
|
3849
|
+
let runtimeHandle = runtimeState;
|
|
3850
|
+
if (!runtimeHandle || runtimeHandle.url !== url) {
|
|
3851
|
+
this.destroySocketClientRuntimeHandle(runtimeHandle);
|
|
3852
|
+
runtimeHandle = this.createSocketClientRuntimeHandle(url);
|
|
3853
|
+
setRuntimeState(runtimeHandle);
|
|
3854
|
+
}
|
|
3855
|
+
upsertDiagnostics({
|
|
3856
|
+
destroyed: false,
|
|
3857
|
+
connected: false,
|
|
3858
|
+
handshake: false,
|
|
3859
|
+
socketId: runtimeHandle.socket.id ?? null
|
|
3860
|
+
});
|
|
3861
|
+
applySessionOperation("connect", {
|
|
3862
|
+
destroyed: false,
|
|
3863
|
+
connected: false,
|
|
3864
|
+
handshake: false,
|
|
3865
|
+
socketId: runtimeHandle.socket.id ?? null,
|
|
3866
|
+
pendingDelegations: runtimeHandle.pendingDelegationIds.size,
|
|
3867
|
+
pendingTimers: runtimeHandle.pendingTimers.size,
|
|
3868
|
+
errorCount: runtimeHandle.errorCount
|
|
3869
|
+
});
|
|
3870
|
+
if (runtimeHandle.initialized) {
|
|
3871
|
+
return true;
|
|
3872
|
+
}
|
|
3873
|
+
runtimeHandle.initialized = true;
|
|
3874
|
+
runtimeHandle.handshake = false;
|
|
3875
|
+
runtimeHandle.errorCount = 0;
|
|
3876
|
+
const syncPendingCounts = () => {
|
|
3877
|
+
const pendingDelegations = runtimeHandle.pendingDelegationIds.size;
|
|
3878
|
+
const pendingTimers = runtimeHandle.pendingTimers.size;
|
|
3879
|
+
upsertDiagnostics({
|
|
3880
|
+
pendingDelegations,
|
|
3881
|
+
pendingTimers
|
|
3882
|
+
});
|
|
3883
|
+
applySessionOperation("delegate", {
|
|
3884
|
+
pendingDelegations,
|
|
3885
|
+
pendingTimers
|
|
3886
|
+
});
|
|
3887
|
+
};
|
|
3888
|
+
runtimeHandle.emitWhenReady = (event, data, timeoutMs = 6e4, ack) => {
|
|
3889
|
+
return new Promise((resolve) => {
|
|
3890
|
+
const parsedTimeout = Number(timeoutMs);
|
|
3891
|
+
const normalizedTimeoutMs = Number.isFinite(parsedTimeout) && parsedTimeout > 0 ? Math.trunc(parsedTimeout) : 6e4;
|
|
3892
|
+
let timer = null;
|
|
3893
|
+
let settled = false;
|
|
3894
|
+
const clearPendingTimer = () => {
|
|
3895
|
+
if (!timer) {
|
|
3896
|
+
return;
|
|
3897
|
+
}
|
|
3898
|
+
clearTimeout(timer);
|
|
3899
|
+
runtimeHandle.pendingTimers.delete(timer);
|
|
3900
|
+
syncPendingCounts();
|
|
3901
|
+
timer = null;
|
|
3902
|
+
};
|
|
3903
|
+
const settle = (response) => {
|
|
3904
|
+
if (settled) {
|
|
3905
|
+
return;
|
|
3906
|
+
}
|
|
3907
|
+
settled = true;
|
|
3908
|
+
clearPendingTimer();
|
|
3909
|
+
if (ack) ack(response);
|
|
3910
|
+
resolve(response);
|
|
3911
|
+
};
|
|
3912
|
+
const resolveWithError = (errorMessage, fallbackError) => {
|
|
3913
|
+
settle({
|
|
3914
|
+
...data,
|
|
3915
|
+
errored: true,
|
|
3916
|
+
__error: errorMessage,
|
|
3917
|
+
error: fallbackError instanceof Error ? fallbackError.message : errorMessage,
|
|
3918
|
+
socketId: runtimeHandle.socket.id,
|
|
3919
|
+
serviceName,
|
|
3920
|
+
url
|
|
3921
|
+
});
|
|
3922
|
+
};
|
|
3923
|
+
const tryEmit = async () => {
|
|
3924
|
+
const waitResult = await waitForSocketConnection(
|
|
3925
|
+
runtimeHandle.socket,
|
|
3926
|
+
normalizedTimeoutMs + 10,
|
|
3927
|
+
(reason, error) => {
|
|
3928
|
+
if (reason === "connect_timeout") {
|
|
3929
|
+
return `Socket connect timed out before '${event}'`;
|
|
3930
|
+
}
|
|
3931
|
+
if (reason === "connect_error") {
|
|
3932
|
+
const errMessage = error instanceof Error ? error.message : String(error);
|
|
3933
|
+
return `Socket connect error before '${event}': ${errMessage}`;
|
|
3934
|
+
}
|
|
3935
|
+
return `Socket disconnected before '${event}'`;
|
|
3936
|
+
}
|
|
3937
|
+
);
|
|
3938
|
+
if (!waitResult.ok) {
|
|
3939
|
+
CadenzaService.log(
|
|
3940
|
+
waitResult.error,
|
|
3941
|
+
{
|
|
3942
|
+
socketId: runtimeHandle.socket.id,
|
|
3943
|
+
serviceName,
|
|
3944
|
+
url,
|
|
3945
|
+
event
|
|
3946
|
+
},
|
|
3947
|
+
"error"
|
|
3948
|
+
);
|
|
3949
|
+
upsertDiagnostics({}, waitResult.error);
|
|
3950
|
+
resolveWithError(waitResult.error);
|
|
3951
|
+
return;
|
|
3952
|
+
}
|
|
3953
|
+
timer = setTimeout(() => {
|
|
3954
|
+
if (settled) {
|
|
3955
|
+
return;
|
|
3956
|
+
}
|
|
3957
|
+
clearPendingTimer();
|
|
3958
|
+
const message = `Socket event '${event}' timed out`;
|
|
3959
|
+
CadenzaService.log(
|
|
3960
|
+
message,
|
|
3961
|
+
{ socketId: runtimeHandle.socket.id, serviceName, url },
|
|
3962
|
+
"error"
|
|
3963
|
+
);
|
|
3964
|
+
upsertDiagnostics(
|
|
3965
|
+
{
|
|
3966
|
+
lastHandshakeError: message
|
|
3967
|
+
},
|
|
3968
|
+
message
|
|
3969
|
+
);
|
|
3970
|
+
applySessionOperation("transmit", {
|
|
3971
|
+
lastHandshakeError: message
|
|
3972
|
+
});
|
|
3973
|
+
resolveWithError(message);
|
|
3974
|
+
}, normalizedTimeoutMs + 10);
|
|
3975
|
+
runtimeHandle.pendingTimers.add(timer);
|
|
3976
|
+
syncPendingCounts();
|
|
3977
|
+
runtimeHandle.socket.timeout(normalizedTimeoutMs).emit(event, data, (err, response) => {
|
|
3978
|
+
if (err) {
|
|
3979
|
+
CadenzaService.log(
|
|
3980
|
+
"Socket timeout.",
|
|
3981
|
+
{
|
|
3982
|
+
event,
|
|
3983
|
+
error: err.message,
|
|
3984
|
+
socketId: runtimeHandle.socket.id,
|
|
3985
|
+
serviceName
|
|
3986
|
+
},
|
|
3987
|
+
"warning"
|
|
3988
|
+
);
|
|
3989
|
+
upsertDiagnostics(
|
|
3990
|
+
{
|
|
3991
|
+
lastHandshakeError: err.message
|
|
3992
|
+
},
|
|
3993
|
+
err
|
|
3994
|
+
);
|
|
3995
|
+
applySessionOperation("transmit", {
|
|
3996
|
+
lastHandshakeError: err.message
|
|
3997
|
+
});
|
|
3998
|
+
response = {
|
|
3999
|
+
__error: `Timeout error: ${err}`,
|
|
4000
|
+
errored: true,
|
|
4001
|
+
...data
|
|
4002
|
+
};
|
|
4003
|
+
}
|
|
4004
|
+
settle(response);
|
|
4005
|
+
});
|
|
4006
|
+
};
|
|
4007
|
+
void tryEmit().catch((error) => {
|
|
4008
|
+
CadenzaService.log(
|
|
4009
|
+
"Socket emit failed unexpectedly",
|
|
4010
|
+
{
|
|
4011
|
+
event,
|
|
4012
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4013
|
+
socketId: runtimeHandle.socket.id,
|
|
4014
|
+
serviceName,
|
|
4015
|
+
url
|
|
4016
|
+
},
|
|
4017
|
+
"error"
|
|
4018
|
+
);
|
|
4019
|
+
const message = `Socket event '${event}' failed`;
|
|
4020
|
+
upsertDiagnostics(
|
|
4021
|
+
{
|
|
4022
|
+
lastHandshakeError: error instanceof Error ? error.message : String(error)
|
|
4023
|
+
},
|
|
4024
|
+
error
|
|
4025
|
+
);
|
|
4026
|
+
applySessionOperation("transmit", {
|
|
4027
|
+
lastHandshakeError: error instanceof Error ? error.message : String(error)
|
|
4028
|
+
});
|
|
4029
|
+
resolveWithError(message, error);
|
|
4030
|
+
});
|
|
4031
|
+
});
|
|
4032
|
+
};
|
|
4033
|
+
const socket = runtimeHandle.socket;
|
|
4034
|
+
socket.on("connect", () => {
|
|
4035
|
+
if (runtimeHandle.handshake) return;
|
|
4036
|
+
upsertDiagnostics({
|
|
4037
|
+
connected: true,
|
|
4038
|
+
destroyed: false,
|
|
4039
|
+
socketId: socket.id ?? null
|
|
4040
|
+
});
|
|
4041
|
+
applySessionOperation("connect", {
|
|
4042
|
+
connected: true,
|
|
4043
|
+
destroyed: false,
|
|
4044
|
+
socketId: socket.id ?? null
|
|
4045
|
+
});
|
|
4046
|
+
CadenzaService.emit(`meta.socket_client.connected:${fetchId}`, input);
|
|
4047
|
+
});
|
|
4048
|
+
socket.on("delegation_progress", (delegationCtx) => {
|
|
4049
|
+
CadenzaService.emit(
|
|
4050
|
+
`meta.socket_client.delegation_progress:${delegationCtx.__metadata.__deputyExecId}`,
|
|
4051
|
+
delegationCtx
|
|
4052
|
+
);
|
|
4053
|
+
});
|
|
4054
|
+
socket.on("signal", (signalCtx) => {
|
|
4055
|
+
if (CadenzaService.signalBroker.listObservedSignals().includes(signalCtx.__signalName)) {
|
|
4056
|
+
CadenzaService.emit(signalCtx.__signalName, signalCtx);
|
|
4057
|
+
}
|
|
4058
|
+
});
|
|
4059
|
+
socket.on("status_update", (status) => {
|
|
4060
|
+
CadenzaService.emit("meta.socket_client.status_received", status);
|
|
4061
|
+
});
|
|
4062
|
+
socket.on("connect_error", (err) => {
|
|
4063
|
+
runtimeHandle.handshake = false;
|
|
4064
|
+
upsertDiagnostics(
|
|
4065
|
+
{
|
|
4066
|
+
connected: false,
|
|
4067
|
+
handshake: false,
|
|
4068
|
+
connectErrors: state.connectErrors + 1,
|
|
4069
|
+
lastHandshakeError: err.message
|
|
4070
|
+
},
|
|
4071
|
+
err
|
|
4072
|
+
);
|
|
4073
|
+
applySessionOperation("connect", {
|
|
4074
|
+
connected: false,
|
|
4075
|
+
handshake: false,
|
|
4076
|
+
connectErrors: state.connectErrors + 1,
|
|
4077
|
+
lastHandshakeError: err.message
|
|
4078
|
+
});
|
|
4079
|
+
CadenzaService.log(
|
|
4080
|
+
"Socket connect error",
|
|
4081
|
+
{
|
|
4082
|
+
error: err.message,
|
|
4083
|
+
serviceName,
|
|
4084
|
+
socketId: socket.id,
|
|
4085
|
+
url
|
|
4086
|
+
},
|
|
4087
|
+
"error"
|
|
4088
|
+
);
|
|
4089
|
+
CadenzaService.emit(`meta.socket_client.connect_error:${fetchId}`, err);
|
|
4090
|
+
});
|
|
4091
|
+
socket.on("reconnect_attempt", (attempt) => {
|
|
4092
|
+
upsertDiagnostics({ reconnectAttempts: attempt });
|
|
4093
|
+
applySessionOperation("connect", {
|
|
4094
|
+
reconnectAttempts: attempt
|
|
4095
|
+
});
|
|
4096
|
+
CadenzaService.log(`Reconnect attempt: ${attempt}`);
|
|
4097
|
+
});
|
|
4098
|
+
socket.on("reconnect", (attempt) => {
|
|
4099
|
+
upsertDiagnostics({ connected: true });
|
|
4100
|
+
applySessionOperation("connect", {
|
|
4101
|
+
connected: true
|
|
4102
|
+
});
|
|
4103
|
+
CadenzaService.log(`Socket reconnected after ${attempt} tries`, {
|
|
4104
|
+
socketId: socket.id,
|
|
4105
|
+
url,
|
|
4106
|
+
serviceName
|
|
4107
|
+
});
|
|
4108
|
+
});
|
|
4109
|
+
socket.on("reconnect_error", (err) => {
|
|
4110
|
+
runtimeHandle.handshake = false;
|
|
4111
|
+
upsertDiagnostics(
|
|
4112
|
+
{
|
|
4113
|
+
connected: false,
|
|
4114
|
+
handshake: false,
|
|
4115
|
+
reconnectErrors: state.reconnectErrors + 1,
|
|
4116
|
+
lastHandshakeError: err.message
|
|
4117
|
+
},
|
|
4118
|
+
err
|
|
4119
|
+
);
|
|
4120
|
+
applySessionOperation("connect", {
|
|
4121
|
+
connected: false,
|
|
4122
|
+
handshake: false,
|
|
4123
|
+
reconnectErrors: state.reconnectErrors + 1,
|
|
4124
|
+
lastHandshakeError: err.message
|
|
4125
|
+
});
|
|
4126
|
+
CadenzaService.log(
|
|
4127
|
+
"Socket reconnect failed.",
|
|
4128
|
+
{ error: err.message, serviceName, url, socketId: socket.id },
|
|
4129
|
+
"warning"
|
|
4130
|
+
);
|
|
4131
|
+
});
|
|
4132
|
+
socket.on("error", (err) => {
|
|
4133
|
+
runtimeHandle.errorCount += 1;
|
|
4134
|
+
upsertDiagnostics(
|
|
4135
|
+
{
|
|
4136
|
+
socketErrors: state.socketErrors + 1,
|
|
4137
|
+
lastHandshakeError: this.getErrorMessage(err)
|
|
4138
|
+
},
|
|
4139
|
+
err
|
|
4140
|
+
);
|
|
4141
|
+
applySessionOperation("transmit", {
|
|
4142
|
+
socketErrors: state.socketErrors + 1,
|
|
4143
|
+
errorCount: runtimeHandle.errorCount,
|
|
4144
|
+
lastHandshakeError: this.getErrorMessage(err)
|
|
4145
|
+
});
|
|
4146
|
+
CadenzaService.log(
|
|
4147
|
+
"Socket error",
|
|
4148
|
+
{ error: err, socketId: socket.id, url, serviceName },
|
|
4149
|
+
"error"
|
|
4150
|
+
);
|
|
4151
|
+
CadenzaService.emit("meta.socket_client.error", err);
|
|
4152
|
+
});
|
|
4153
|
+
socket.on("disconnect", () => {
|
|
4154
|
+
const disconnectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4155
|
+
upsertDiagnostics({
|
|
4156
|
+
connected: false,
|
|
4157
|
+
handshake: false,
|
|
4158
|
+
lastDisconnectAt: disconnectedAt
|
|
4159
|
+
});
|
|
4160
|
+
applySessionOperation("connect", {
|
|
4161
|
+
connected: false,
|
|
4162
|
+
handshake: false,
|
|
4163
|
+
lastDisconnectAt: disconnectedAt
|
|
4164
|
+
});
|
|
4165
|
+
CadenzaService.log(
|
|
4166
|
+
"Socket disconnected.",
|
|
4167
|
+
{ url, serviceName, socketId: socket.id },
|
|
4168
|
+
"warning"
|
|
4169
|
+
);
|
|
4170
|
+
CadenzaService.emit(`meta.socket_client.disconnected:${fetchId}`, {
|
|
4171
|
+
serviceName,
|
|
4172
|
+
serviceAddress,
|
|
4173
|
+
servicePort: normalizedPort
|
|
4174
|
+
});
|
|
4175
|
+
runtimeHandle.handshake = false;
|
|
4176
|
+
});
|
|
4177
|
+
socket.connect();
|
|
4178
|
+
runtimeHandle.handshakeTask = CadenzaService.createMetaTask(
|
|
4179
|
+
`Socket handshake with ${url}`,
|
|
4180
|
+
async (_ctx, emitter) => {
|
|
4181
|
+
if (runtimeHandle.handshake) return;
|
|
4182
|
+
runtimeHandle.handshake = true;
|
|
4183
|
+
upsertDiagnostics({
|
|
4184
|
+
handshake: true
|
|
3199
4185
|
});
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
4186
|
+
applySessionOperation("handshake", {
|
|
4187
|
+
handshake: true
|
|
4188
|
+
});
|
|
4189
|
+
await runtimeHandle.emitWhenReady?.(
|
|
4190
|
+
"handshake",
|
|
4191
|
+
{
|
|
4192
|
+
serviceInstanceId: CadenzaService.serviceRegistry.serviceInstanceId,
|
|
4193
|
+
serviceName: CadenzaService.serviceRegistry.serviceName,
|
|
4194
|
+
isFrontend: isBrowser,
|
|
4195
|
+
__status: "success"
|
|
4196
|
+
},
|
|
4197
|
+
1e4,
|
|
4198
|
+
(result) => {
|
|
4199
|
+
if (result.status === "success") {
|
|
4200
|
+
const handshakeAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4201
|
+
upsertDiagnostics({
|
|
4202
|
+
connected: true,
|
|
4203
|
+
handshake: true,
|
|
4204
|
+
lastHandshakeAt: handshakeAt,
|
|
4205
|
+
lastHandshakeError: null,
|
|
4206
|
+
socketId: socket.id ?? null
|
|
4207
|
+
});
|
|
4208
|
+
applySessionOperation("handshake", {
|
|
4209
|
+
connected: true,
|
|
4210
|
+
handshake: true,
|
|
4211
|
+
lastHandshakeAt: handshakeAt,
|
|
4212
|
+
lastHandshakeError: null,
|
|
4213
|
+
socketId: socket.id ?? null
|
|
4214
|
+
});
|
|
4215
|
+
CadenzaService.log("Socket client connected", {
|
|
4216
|
+
result,
|
|
4217
|
+
serviceName,
|
|
4218
|
+
socketId: socket.id,
|
|
4219
|
+
url
|
|
4220
|
+
});
|
|
4221
|
+
} else {
|
|
4222
|
+
const errorMessage = result?.__error ?? result?.error ?? "Socket handshake failed";
|
|
4223
|
+
upsertDiagnostics(
|
|
4224
|
+
{
|
|
4225
|
+
connected: false,
|
|
4226
|
+
handshake: false,
|
|
4227
|
+
lastHandshakeError: errorMessage
|
|
4228
|
+
},
|
|
4229
|
+
errorMessage
|
|
4230
|
+
);
|
|
4231
|
+
applySessionOperation("handshake", {
|
|
4232
|
+
connected: false,
|
|
4233
|
+
handshake: false,
|
|
4234
|
+
lastHandshakeError: errorMessage
|
|
4235
|
+
});
|
|
4236
|
+
CadenzaService.log(
|
|
4237
|
+
"Socket handshake failed",
|
|
4238
|
+
{ result, serviceName, socketId: socket.id, url },
|
|
4239
|
+
"warning"
|
|
4240
|
+
);
|
|
3213
4241
|
}
|
|
3214
|
-
|
|
4242
|
+
void emitter;
|
|
3215
4243
|
}
|
|
3216
4244
|
);
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
fetchId,
|
|
3225
|
-
serviceName,
|
|
3226
|
-
URL,
|
|
3227
|
-
waitResult.error
|
|
3228
|
-
);
|
|
3229
|
-
resolveWithError(waitResult.error);
|
|
4245
|
+
},
|
|
4246
|
+
"Handshakes with socket server"
|
|
4247
|
+
).doOn(`meta.socket_client.connected:${fetchId}`);
|
|
4248
|
+
runtimeHandle.delegateTask = CadenzaService.createMetaTask(
|
|
4249
|
+
`Delegate flow to Socket service ${url}`,
|
|
4250
|
+
async (delegateCtx, emitter) => {
|
|
4251
|
+
if (delegateCtx.__remoteRoutineName === void 0) {
|
|
3230
4252
|
return;
|
|
3231
4253
|
}
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
timer = null;
|
|
3239
|
-
}
|
|
3240
|
-
CadenzaService.log(
|
|
3241
|
-
`Socket event '${event}' timed out`,
|
|
3242
|
-
{ socketId: socket?.id, serviceName, URL },
|
|
3243
|
-
"error"
|
|
3244
|
-
);
|
|
3245
|
-
this.recordSocketClientError(
|
|
3246
|
-
fetchId,
|
|
3247
|
-
serviceName,
|
|
3248
|
-
URL,
|
|
3249
|
-
`Socket event '${event}' timed out`
|
|
3250
|
-
);
|
|
3251
|
-
resolveWithError(`Socket event '${event}' timed out`);
|
|
3252
|
-
}, timeoutMs + 10);
|
|
3253
|
-
pendingTimers.add(timer);
|
|
4254
|
+
delete delegateCtx.__isSubMeta;
|
|
4255
|
+
delete delegateCtx.__broadcast;
|
|
4256
|
+
const deputyExecId = delegateCtx.__metadata?.__deputyExecId;
|
|
4257
|
+
const requestSentAt = Date.now();
|
|
4258
|
+
if (deputyExecId) {
|
|
4259
|
+
runtimeHandle.pendingDelegationIds.add(deputyExecId);
|
|
3254
4260
|
syncPendingCounts();
|
|
3255
4261
|
}
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
4262
|
+
try {
|
|
4263
|
+
const resultContext = await runtimeHandle.emitWhenReady?.(
|
|
4264
|
+
"delegation",
|
|
4265
|
+
delegateCtx,
|
|
4266
|
+
delegateCtx.__timeout ?? 6e4
|
|
4267
|
+
) ?? {
|
|
4268
|
+
errored: true,
|
|
4269
|
+
__error: "Socket delegation returned no response"
|
|
4270
|
+
};
|
|
4271
|
+
const requestDuration = Date.now() - requestSentAt;
|
|
4272
|
+
const metadata = resultContext.__metadata;
|
|
4273
|
+
delete resultContext.__metadata;
|
|
4274
|
+
if (deputyExecId) {
|
|
4275
|
+
emitter(`meta.socket_client.delegated:${deputyExecId}`, {
|
|
4276
|
+
...resultContext,
|
|
4277
|
+
...metadata,
|
|
4278
|
+
__requestDuration: requestDuration
|
|
4279
|
+
});
|
|
3269
4280
|
}
|
|
3270
|
-
if (
|
|
3271
|
-
|
|
3272
|
-
|
|
4281
|
+
if (resultContext?.errored || resultContext?.failed) {
|
|
4282
|
+
const errorMessage = resultContext?.__error ?? resultContext?.error ?? "Socket delegation failed";
|
|
4283
|
+
upsertDiagnostics(
|
|
3273
4284
|
{
|
|
3274
|
-
|
|
3275
|
-
error: err.message,
|
|
3276
|
-
socketId: socket?.id,
|
|
3277
|
-
serviceName
|
|
4285
|
+
lastHandshakeError: String(errorMessage)
|
|
3278
4286
|
},
|
|
3279
|
-
|
|
3280
|
-
);
|
|
3281
|
-
this.recordSocketClientError(
|
|
3282
|
-
fetchId,
|
|
3283
|
-
serviceName,
|
|
3284
|
-
URL,
|
|
3285
|
-
err
|
|
4287
|
+
errorMessage
|
|
3286
4288
|
);
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
...ctx,
|
|
3291
|
-
...ctx.__metadata
|
|
3292
|
-
};
|
|
4289
|
+
applySessionOperation("delegate", {
|
|
4290
|
+
lastHandshakeError: String(errorMessage)
|
|
4291
|
+
});
|
|
3293
4292
|
}
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
socketDiagnostics.socketId = socket?.id ?? null;
|
|
3306
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3307
|
-
CadenzaService.emit(`meta.socket_client.connected:${fetchId}`, ctx);
|
|
3308
|
-
});
|
|
3309
|
-
socket.on("delegation_progress", (ctx2) => {
|
|
3310
|
-
CadenzaService.emit(
|
|
3311
|
-
`meta.socket_client.delegation_progress:${ctx2.__metadata.__deputyExecId}`,
|
|
3312
|
-
ctx2
|
|
3313
|
-
);
|
|
3314
|
-
});
|
|
3315
|
-
socket.on("signal", (ctx2) => {
|
|
3316
|
-
if (CadenzaService.signalBroker.listObservedSignals().includes(ctx2.__signalName)) {
|
|
3317
|
-
CadenzaService.emit(ctx2.__signalName, ctx2);
|
|
3318
|
-
}
|
|
3319
|
-
});
|
|
3320
|
-
socket.on("status_update", (status) => {
|
|
3321
|
-
CadenzaService.emit("meta.socket_client.status_received", status);
|
|
3322
|
-
});
|
|
3323
|
-
socket.on("connect_error", (err) => {
|
|
3324
|
-
handshake = false;
|
|
3325
|
-
socketDiagnostics.connected = false;
|
|
3326
|
-
socketDiagnostics.handshake = false;
|
|
3327
|
-
socketDiagnostics.connectErrors++;
|
|
3328
|
-
socketDiagnostics.lastHandshakeError = err.message;
|
|
3329
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3330
|
-
this.recordSocketClientError(fetchId, serviceName, URL, err);
|
|
3331
|
-
CadenzaService.log(
|
|
3332
|
-
"Socket connect error",
|
|
3333
|
-
{ error: err.message, serviceName, socketId: socket?.id, URL },
|
|
3334
|
-
"error"
|
|
3335
|
-
);
|
|
3336
|
-
CadenzaService.emit(`meta.socket_client.connect_error:${fetchId}`, err);
|
|
3337
|
-
});
|
|
3338
|
-
socket.on("reconnect_attempt", (attempt) => {
|
|
3339
|
-
socketDiagnostics.reconnectAttempts = Math.max(
|
|
3340
|
-
socketDiagnostics.reconnectAttempts,
|
|
3341
|
-
attempt
|
|
3342
|
-
);
|
|
3343
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3344
|
-
CadenzaService.log(`Reconnect attempt: ${attempt}`);
|
|
3345
|
-
});
|
|
3346
|
-
socket.on("reconnect", (attempt) => {
|
|
3347
|
-
socketDiagnostics.connected = true;
|
|
3348
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3349
|
-
CadenzaService.log(`Socket reconnected after ${attempt} tries`, {
|
|
3350
|
-
socketId: socket?.id,
|
|
3351
|
-
URL,
|
|
3352
|
-
serviceName
|
|
3353
|
-
});
|
|
3354
|
-
});
|
|
3355
|
-
socket.on("reconnect_error", (err) => {
|
|
3356
|
-
handshake = false;
|
|
3357
|
-
socketDiagnostics.connected = false;
|
|
3358
|
-
socketDiagnostics.handshake = false;
|
|
3359
|
-
socketDiagnostics.reconnectErrors++;
|
|
3360
|
-
socketDiagnostics.lastHandshakeError = err.message;
|
|
3361
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3362
|
-
this.recordSocketClientError(fetchId, serviceName, URL, err);
|
|
3363
|
-
CadenzaService.log(
|
|
3364
|
-
"Socket reconnect failed.",
|
|
3365
|
-
{ error: err.message, serviceName, URL, socketId: socket?.id },
|
|
3366
|
-
"warning"
|
|
3367
|
-
);
|
|
3368
|
-
});
|
|
3369
|
-
socket.on("error", (err) => {
|
|
3370
|
-
errorCount++;
|
|
3371
|
-
socketDiagnostics.socketErrors++;
|
|
3372
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3373
|
-
this.recordSocketClientError(fetchId, serviceName, URL, err);
|
|
3374
|
-
CadenzaService.log(
|
|
3375
|
-
"Socket error",
|
|
3376
|
-
{ error: err, socketId: socket?.id, URL, serviceName },
|
|
3377
|
-
"error"
|
|
3378
|
-
);
|
|
3379
|
-
CadenzaService.emit("meta.socket_client.error", err);
|
|
3380
|
-
});
|
|
3381
|
-
socket.on("disconnect", () => {
|
|
3382
|
-
const disconnectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3383
|
-
socketDiagnostics.connected = false;
|
|
3384
|
-
socketDiagnostics.handshake = false;
|
|
3385
|
-
socketDiagnostics.lastDisconnectAt = disconnectedAt;
|
|
3386
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3387
|
-
CadenzaService.log(
|
|
3388
|
-
"Socket disconnected.",
|
|
3389
|
-
{ URL, serviceName, socketId: socket?.id },
|
|
3390
|
-
"warning"
|
|
3391
|
-
);
|
|
3392
|
-
CadenzaService.emit(`meta.socket_client.disconnected:${fetchId}`, {
|
|
3393
|
-
serviceName,
|
|
3394
|
-
serviceAddress,
|
|
3395
|
-
servicePort
|
|
3396
|
-
});
|
|
3397
|
-
handshake = false;
|
|
3398
|
-
});
|
|
3399
|
-
socket.connect();
|
|
3400
|
-
handshakeTask = CadenzaService.createMetaTask(
|
|
3401
|
-
`Socket handshake with ${URL}`,
|
|
3402
|
-
async (ctx2, emit) => {
|
|
3403
|
-
if (handshake) return;
|
|
3404
|
-
handshake = true;
|
|
3405
|
-
socketDiagnostics.handshake = true;
|
|
3406
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3407
|
-
await emitWhenReady?.(
|
|
3408
|
-
"handshake",
|
|
3409
|
-
{
|
|
3410
|
-
serviceInstanceId: CadenzaService.serviceRegistry.serviceInstanceId,
|
|
3411
|
-
serviceName: CadenzaService.serviceRegistry.serviceName,
|
|
3412
|
-
isFrontend: isBrowser,
|
|
3413
|
-
__status: "success"
|
|
3414
|
-
},
|
|
3415
|
-
1e4,
|
|
3416
|
-
(result) => {
|
|
3417
|
-
if (result.status === "success") {
|
|
3418
|
-
socketDiagnostics.connected = true;
|
|
3419
|
-
socketDiagnostics.handshake = true;
|
|
3420
|
-
socketDiagnostics.lastHandshakeAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3421
|
-
socketDiagnostics.lastHandshakeError = null;
|
|
3422
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3423
|
-
CadenzaService.log("Socket client connected", {
|
|
3424
|
-
result,
|
|
3425
|
-
serviceName,
|
|
3426
|
-
socketId: socket?.id,
|
|
3427
|
-
URL
|
|
4293
|
+
return resultContext;
|
|
4294
|
+
} catch (error) {
|
|
4295
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4296
|
+
const failedContext = {
|
|
4297
|
+
errored: true,
|
|
4298
|
+
__error: message
|
|
4299
|
+
};
|
|
4300
|
+
if (deputyExecId) {
|
|
4301
|
+
emitter(`meta.socket_client.delegated:${deputyExecId}`, {
|
|
4302
|
+
...failedContext,
|
|
4303
|
+
__requestDuration: Date.now() - requestSentAt
|
|
3428
4304
|
});
|
|
3429
|
-
} else {
|
|
3430
|
-
socketDiagnostics.connected = false;
|
|
3431
|
-
socketDiagnostics.handshake = false;
|
|
3432
|
-
socketDiagnostics.lastHandshakeError = result?.__error ?? result?.error ?? "Socket handshake failed";
|
|
3433
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3434
|
-
this.recordSocketClientError(
|
|
3435
|
-
fetchId,
|
|
3436
|
-
serviceName,
|
|
3437
|
-
URL,
|
|
3438
|
-
socketDiagnostics.lastHandshakeError
|
|
3439
|
-
);
|
|
3440
|
-
CadenzaService.log(
|
|
3441
|
-
"Socket handshake failed",
|
|
3442
|
-
{ result, serviceName, socketId: socket?.id, URL },
|
|
3443
|
-
"warning"
|
|
3444
|
-
);
|
|
3445
4305
|
}
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
|
|
3455
|
-
|
|
3456
|
-
|
|
3457
|
-
|
|
3458
|
-
|
|
3459
|
-
delete ctx2.__broadcast;
|
|
3460
|
-
const requestSentAt = Date.now();
|
|
3461
|
-
pendingDelegationIds.add(ctx2.__metadata.__deputyExecId);
|
|
3462
|
-
syncPendingCounts();
|
|
3463
|
-
emitWhenReady?.(
|
|
3464
|
-
"delegation",
|
|
3465
|
-
ctx2,
|
|
3466
|
-
ctx2.__timeout ?? 6e4,
|
|
3467
|
-
(resultContext) => {
|
|
3468
|
-
const requestDuration = Date.now() - requestSentAt;
|
|
3469
|
-
const metadata = resultContext.__metadata;
|
|
3470
|
-
delete resultContext.__metadata;
|
|
3471
|
-
emit(
|
|
3472
|
-
`meta.socket_client.delegated:${ctx2.__metadata.__deputyExecId}`,
|
|
3473
|
-
{
|
|
3474
|
-
...resultContext,
|
|
3475
|
-
...metadata,
|
|
3476
|
-
__requestDuration: requestDuration
|
|
3477
|
-
}
|
|
3478
|
-
);
|
|
3479
|
-
pendingDelegationIds.delete(ctx2.__metadata.__deputyExecId);
|
|
4306
|
+
upsertDiagnostics(
|
|
4307
|
+
{
|
|
4308
|
+
lastHandshakeError: message
|
|
4309
|
+
},
|
|
4310
|
+
error
|
|
4311
|
+
);
|
|
4312
|
+
applySessionOperation("delegate", {
|
|
4313
|
+
lastHandshakeError: message
|
|
4314
|
+
});
|
|
4315
|
+
return failedContext;
|
|
4316
|
+
} finally {
|
|
4317
|
+
if (deputyExecId) {
|
|
4318
|
+
runtimeHandle.pendingDelegationIds.delete(deputyExecId);
|
|
3480
4319
|
syncPendingCounts();
|
|
3481
|
-
if (resultContext?.errored || resultContext?.failed) {
|
|
3482
|
-
this.recordSocketClientError(
|
|
3483
|
-
fetchId,
|
|
3484
|
-
serviceName,
|
|
3485
|
-
URL,
|
|
3486
|
-
resultContext?.__error ?? resultContext?.error ?? "Socket delegation failed"
|
|
3487
|
-
);
|
|
3488
|
-
}
|
|
3489
|
-
resolve(resultContext);
|
|
3490
4320
|
}
|
|
3491
|
-
|
|
3492
|
-
}
|
|
3493
|
-
|
|
3494
|
-
`
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
4321
|
+
}
|
|
4322
|
+
},
|
|
4323
|
+
`Delegate flow to service ${serviceName} with address ${url}`
|
|
4324
|
+
).doOn(`meta.service_registry.selected_instance_for_socket:${fetchId}`).attachSignal(
|
|
4325
|
+
"meta.socket_client.delegated",
|
|
4326
|
+
"meta.socket_shutdown_requested"
|
|
4327
|
+
);
|
|
4328
|
+
runtimeHandle.transmitTask = CadenzaService.createMetaTask(
|
|
4329
|
+
`Transmit signal to socket server ${url}`,
|
|
4330
|
+
async (signalCtx, emitter) => {
|
|
4331
|
+
if (signalCtx.__signalName === void 0) {
|
|
4332
|
+
return;
|
|
4333
|
+
}
|
|
4334
|
+
delete signalCtx.__broadcast;
|
|
4335
|
+
const response = await runtimeHandle.emitWhenReady?.("signal", signalCtx, 5e3) ?? {
|
|
4336
|
+
errored: true,
|
|
4337
|
+
__error: "Socket signal transmission returned no response"
|
|
4338
|
+
};
|
|
4339
|
+
applySessionOperation("transmit", {});
|
|
4340
|
+
if (signalCtx.__routineExecId) {
|
|
4341
|
+
emitter(`meta.socket_client.transmitted:${signalCtx.__routineExecId}`, {
|
|
4342
|
+
...response
|
|
4343
|
+
});
|
|
4344
|
+
}
|
|
4345
|
+
return response;
|
|
4346
|
+
},
|
|
4347
|
+
`Transmits signal to service ${serviceName} with address ${url}`
|
|
4348
|
+
).doOn(`meta.service_registry.selected_instance_for_socket:${fetchId}`).attachSignal("meta.socket_client.transmitted");
|
|
4349
|
+
CadenzaService.createEphemeralMetaTask(
|
|
4350
|
+
`Shutdown SocketClient ${url}`,
|
|
4351
|
+
(_ctx, emitter) => {
|
|
4352
|
+
runtimeHandle.handshake = false;
|
|
4353
|
+
upsertDiagnostics({
|
|
4354
|
+
connected: false,
|
|
4355
|
+
handshake: false,
|
|
4356
|
+
destroyed: true,
|
|
4357
|
+
pendingDelegations: 0,
|
|
4358
|
+
pendingTimers: 0
|
|
4359
|
+
});
|
|
4360
|
+
applySessionOperation("shutdown", {
|
|
4361
|
+
connected: false,
|
|
4362
|
+
handshake: false,
|
|
4363
|
+
destroyed: true,
|
|
4364
|
+
pendingDelegations: 0,
|
|
4365
|
+
pendingTimers: 0
|
|
4366
|
+
});
|
|
4367
|
+
CadenzaService.log("Shutting down socket client", { url, serviceName });
|
|
4368
|
+
emitter(`meta.fetch.handshake_requested:${fetchId}`, {
|
|
4369
|
+
serviceInstanceId,
|
|
4370
|
+
serviceName,
|
|
4371
|
+
communicationTypes,
|
|
4372
|
+
serviceAddress,
|
|
4373
|
+
servicePort: normalizedPort,
|
|
4374
|
+
protocol,
|
|
4375
|
+
handshakeData: {
|
|
4376
|
+
instanceId: CadenzaService.serviceRegistry.serviceInstanceId,
|
|
4377
|
+
serviceName: CadenzaService.serviceRegistry.serviceName
|
|
3513
4378
|
}
|
|
3514
|
-
resolve(response);
|
|
3515
4379
|
});
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
`Shutdown SocketClient ${URL}`,
|
|
3522
|
-
(ctx2, emit) => {
|
|
3523
|
-
handshake = false;
|
|
3524
|
-
socketDiagnostics.connected = false;
|
|
3525
|
-
socketDiagnostics.handshake = false;
|
|
3526
|
-
socketDiagnostics.destroyed = true;
|
|
3527
|
-
socketDiagnostics.updatedAt = Date.now();
|
|
3528
|
-
CadenzaService.log("Shutting down socket client", { URL, serviceName });
|
|
3529
|
-
socket?.close();
|
|
3530
|
-
handshakeTask?.destroy();
|
|
3531
|
-
delegateTask?.destroy();
|
|
3532
|
-
transmitTask?.destroy();
|
|
3533
|
-
handshakeTask = null;
|
|
3534
|
-
delegateTask = null;
|
|
3535
|
-
transmitTask = null;
|
|
3536
|
-
emitWhenReady = null;
|
|
3537
|
-
socket = null;
|
|
3538
|
-
emit(`meta.fetch.handshake_requested:${fetchId}`, {
|
|
3539
|
-
serviceInstanceId,
|
|
3540
|
-
serviceName,
|
|
3541
|
-
communicationTypes,
|
|
3542
|
-
serviceAddress,
|
|
3543
|
-
servicePort,
|
|
3544
|
-
protocol,
|
|
3545
|
-
handshakeData: {
|
|
3546
|
-
instanceId: CadenzaService.serviceRegistry.serviceInstanceId,
|
|
3547
|
-
serviceName: CadenzaService.serviceRegistry.serviceName
|
|
4380
|
+
for (const id of runtimeHandle.pendingDelegationIds) {
|
|
4381
|
+
emitter(`meta.socket_client.delegated:${id}`, {
|
|
4382
|
+
errored: true,
|
|
4383
|
+
__error: "Shutting down socket client"
|
|
4384
|
+
});
|
|
3548
4385
|
}
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
errored: true,
|
|
3553
|
-
__error: "Shutting down socket client"
|
|
4386
|
+
this.destroySocketClientRuntimeHandle(runtimeHandle);
|
|
4387
|
+
emitter("meta.socket_client.runtime_clear_requested", {
|
|
4388
|
+
fetchId
|
|
3554
4389
|
});
|
|
3555
|
-
}
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
}
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
`meta.fetch.handshake_failed:${fetchId}`,
|
|
3569
|
-
`meta.socket_client.connect_error:${fetchId}`
|
|
3570
|
-
).attachSignal("meta.fetch.handshake_requested").emits("meta.socket_client_shutdown_complete");
|
|
3571
|
-
return true;
|
|
3572
|
-
},
|
|
3573
|
-
"Connects to a specified socket server"
|
|
4390
|
+
},
|
|
4391
|
+
"Shuts down the socket client"
|
|
4392
|
+
).doOn(
|
|
4393
|
+
`meta.socket_shutdown_requested:${fetchId}`,
|
|
4394
|
+
`meta.socket_client.disconnected:${fetchId}`,
|
|
4395
|
+
`meta.fetch.handshake_failed:${fetchId}`,
|
|
4396
|
+
`meta.socket_client.connect_error:${fetchId}`
|
|
4397
|
+
).attachSignal("meta.fetch.handshake_requested").emits("meta.socket_client_shutdown_complete");
|
|
4398
|
+
return true;
|
|
4399
|
+
},
|
|
4400
|
+
{ mode: "write" }
|
|
4401
|
+
),
|
|
4402
|
+
"Connects to a specified socket server and wires runtime tasks."
|
|
3574
4403
|
).doOn("meta.fetch.handshake_complete").emitsOnFail("meta.socket_client.connect_failed");
|
|
3575
4404
|
}
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
4405
|
+
createInitialSocketServerSessionState(serverKey) {
|
|
4406
|
+
return {
|
|
4407
|
+
serverKey,
|
|
4408
|
+
useSocket: false,
|
|
4409
|
+
status: "inactive",
|
|
4410
|
+
securityProfile: "medium",
|
|
4411
|
+
networkType: "internal",
|
|
4412
|
+
connectionCount: 0,
|
|
4413
|
+
lastStartedAt: null,
|
|
4414
|
+
lastConnectedAt: null,
|
|
4415
|
+
lastDisconnectedAt: null,
|
|
4416
|
+
lastShutdownAt: null,
|
|
4417
|
+
updatedAt: 0
|
|
4418
|
+
};
|
|
3579
4419
|
}
|
|
3580
|
-
|
|
3581
|
-
const detailLevel = ctx.detailLevel === "full" ? "full" : "summary";
|
|
3582
|
-
const includeErrorHistory = Boolean(ctx.includeErrorHistory);
|
|
3583
|
-
const requestedLimit = Number(ctx.errorHistoryLimit);
|
|
3584
|
-
const errorHistoryLimit = Number.isFinite(requestedLimit) ? Math.max(1, Math.min(200, Math.trunc(requestedLimit))) : 10;
|
|
4420
|
+
createInitialSocketClientSessionState() {
|
|
3585
4421
|
return {
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
4422
|
+
fetchId: "",
|
|
4423
|
+
serviceInstanceId: "",
|
|
4424
|
+
communicationTypes: [],
|
|
4425
|
+
serviceName: "",
|
|
4426
|
+
serviceAddress: "",
|
|
4427
|
+
servicePort: 0,
|
|
4428
|
+
protocol: "http",
|
|
4429
|
+
url: "",
|
|
4430
|
+
socketId: null,
|
|
4431
|
+
connected: false,
|
|
4432
|
+
handshake: false,
|
|
4433
|
+
pendingDelegations: 0,
|
|
4434
|
+
pendingTimers: 0,
|
|
4435
|
+
reconnectAttempts: 0,
|
|
4436
|
+
connectErrors: 0,
|
|
4437
|
+
reconnectErrors: 0,
|
|
4438
|
+
socketErrors: 0,
|
|
4439
|
+
errorCount: 0,
|
|
4440
|
+
destroyed: false,
|
|
4441
|
+
lastHandshakeAt: null,
|
|
4442
|
+
lastHandshakeError: null,
|
|
4443
|
+
lastDisconnectAt: null,
|
|
4444
|
+
updatedAt: 0
|
|
3589
4445
|
};
|
|
3590
4446
|
}
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
socketId: null,
|
|
3599
|
-
connected: false,
|
|
3600
|
-
handshake: false,
|
|
3601
|
-
reconnectAttempts: 0,
|
|
3602
|
-
connectErrors: 0,
|
|
3603
|
-
reconnectErrors: 0,
|
|
3604
|
-
socketErrors: 0,
|
|
3605
|
-
pendingDelegations: 0,
|
|
3606
|
-
pendingTimers: 0,
|
|
3607
|
-
destroyed: false,
|
|
3608
|
-
lastHandshakeAt: null,
|
|
3609
|
-
lastHandshakeError: null,
|
|
3610
|
-
lastDisconnectAt: null,
|
|
3611
|
-
lastError: null,
|
|
3612
|
-
lastErrorAt: 0,
|
|
3613
|
-
errorHistory: [],
|
|
3614
|
-
updatedAt: Date.now()
|
|
3615
|
-
};
|
|
3616
|
-
this.socketClientDiagnostics.set(fetchId, state);
|
|
3617
|
-
} else {
|
|
3618
|
-
state.serviceName = serviceName;
|
|
3619
|
-
state.url = url;
|
|
4447
|
+
resolveSocketServerKey(input) {
|
|
4448
|
+
return String(input.serverKey ?? input.__socketServerKey ?? this.socketServerDefaultKey).trim() || this.socketServerDefaultKey;
|
|
4449
|
+
}
|
|
4450
|
+
resolveSocketClientFetchId(input) {
|
|
4451
|
+
const explicitFetchId = String(input.fetchId ?? "").trim();
|
|
4452
|
+
if (explicitFetchId) {
|
|
4453
|
+
return explicitFetchId;
|
|
3620
4454
|
}
|
|
3621
|
-
|
|
4455
|
+
const serviceAddress = String(input.serviceAddress ?? "").trim();
|
|
4456
|
+
const protocol = String(input.protocol ?? "http").trim();
|
|
4457
|
+
const port = this.resolveServicePort(protocol, input.servicePort);
|
|
4458
|
+
if (!serviceAddress || !port) {
|
|
4459
|
+
return void 0;
|
|
4460
|
+
}
|
|
4461
|
+
return `${serviceAddress}_${port}`;
|
|
4462
|
+
}
|
|
4463
|
+
resolveServicePort(protocol, rawPort) {
|
|
4464
|
+
if (protocol === "https") {
|
|
4465
|
+
return 443;
|
|
4466
|
+
}
|
|
4467
|
+
const parsed = Number(rawPort);
|
|
4468
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
4469
|
+
return void 0;
|
|
4470
|
+
}
|
|
4471
|
+
return Math.trunc(parsed);
|
|
4472
|
+
}
|
|
4473
|
+
createSocketServerRuntimeHandleFromContext(context) {
|
|
4474
|
+
const baseServer = context.httpsServer ?? context.httpServer;
|
|
4475
|
+
if (!baseServer) {
|
|
4476
|
+
throw new Error(
|
|
4477
|
+
"Socket server runtime setup requires either httpsServer or httpServer"
|
|
4478
|
+
);
|
|
4479
|
+
}
|
|
4480
|
+
const server = new Server(baseServer, {
|
|
4481
|
+
pingInterval: 3e4,
|
|
4482
|
+
pingTimeout: 2e4,
|
|
4483
|
+
maxHttpBufferSize: 1e7,
|
|
4484
|
+
connectionStateRecovery: {
|
|
4485
|
+
maxDisconnectionDuration: 2 * 60 * 1e3,
|
|
4486
|
+
skipMiddlewares: true
|
|
4487
|
+
}
|
|
4488
|
+
});
|
|
4489
|
+
return {
|
|
4490
|
+
server,
|
|
4491
|
+
initialized: false,
|
|
4492
|
+
connectedSocketIds: /* @__PURE__ */ new Set(),
|
|
4493
|
+
broadcastStatusTask: null,
|
|
4494
|
+
shutdownTask: null
|
|
4495
|
+
};
|
|
4496
|
+
}
|
|
4497
|
+
destroySocketServerRuntimeHandle(runtimeHandle) {
|
|
4498
|
+
if (!runtimeHandle) {
|
|
4499
|
+
return;
|
|
4500
|
+
}
|
|
4501
|
+
runtimeHandle.broadcastStatusTask?.destroy();
|
|
4502
|
+
runtimeHandle.shutdownTask?.destroy();
|
|
4503
|
+
runtimeHandle.broadcastStatusTask = null;
|
|
4504
|
+
runtimeHandle.shutdownTask = null;
|
|
4505
|
+
runtimeHandle.connectedSocketIds.clear();
|
|
4506
|
+
runtimeHandle.initialized = false;
|
|
4507
|
+
runtimeHandle.server.close();
|
|
4508
|
+
runtimeHandle.server.removeAllListeners();
|
|
4509
|
+
}
|
|
4510
|
+
createSocketClientRuntimeHandle(url) {
|
|
4511
|
+
return {
|
|
4512
|
+
url,
|
|
4513
|
+
socket: io(url, {
|
|
4514
|
+
reconnection: true,
|
|
4515
|
+
reconnectionAttempts: 5,
|
|
4516
|
+
reconnectionDelay: 2e3,
|
|
4517
|
+
reconnectionDelayMax: 1e4,
|
|
4518
|
+
randomizationFactor: 0.5,
|
|
4519
|
+
transports: ["websocket"],
|
|
4520
|
+
autoConnect: false
|
|
4521
|
+
}),
|
|
4522
|
+
initialized: false,
|
|
4523
|
+
handshake: false,
|
|
4524
|
+
errorCount: 0,
|
|
4525
|
+
pendingDelegationIds: /* @__PURE__ */ new Set(),
|
|
4526
|
+
pendingTimers: /* @__PURE__ */ new Set(),
|
|
4527
|
+
emitWhenReady: null,
|
|
4528
|
+
handshakeTask: null,
|
|
4529
|
+
delegateTask: null,
|
|
4530
|
+
transmitTask: null
|
|
4531
|
+
};
|
|
4532
|
+
}
|
|
4533
|
+
destroySocketClientRuntimeHandle(runtimeHandle) {
|
|
4534
|
+
if (!runtimeHandle) {
|
|
4535
|
+
return;
|
|
4536
|
+
}
|
|
4537
|
+
runtimeHandle.initialized = false;
|
|
4538
|
+
runtimeHandle.handshake = false;
|
|
4539
|
+
runtimeHandle.emitWhenReady = null;
|
|
4540
|
+
runtimeHandle.handshakeTask?.destroy();
|
|
4541
|
+
runtimeHandle.delegateTask?.destroy();
|
|
4542
|
+
runtimeHandle.transmitTask?.destroy();
|
|
4543
|
+
runtimeHandle.handshakeTask = null;
|
|
4544
|
+
runtimeHandle.delegateTask = null;
|
|
4545
|
+
runtimeHandle.transmitTask = null;
|
|
4546
|
+
for (const timer of runtimeHandle.pendingTimers) {
|
|
4547
|
+
clearTimeout(timer);
|
|
4548
|
+
}
|
|
4549
|
+
runtimeHandle.pendingTimers.clear();
|
|
4550
|
+
runtimeHandle.pendingDelegationIds.clear();
|
|
4551
|
+
runtimeHandle.socket.close();
|
|
4552
|
+
runtimeHandle.socket.removeAllListeners();
|
|
4553
|
+
}
|
|
4554
|
+
normalizeCommunicationTypes(value) {
|
|
4555
|
+
if (!Array.isArray(value)) {
|
|
4556
|
+
return [];
|
|
4557
|
+
}
|
|
4558
|
+
return value.map((item) => String(item)).filter((item) => item.trim().length > 0);
|
|
3622
4559
|
}
|
|
3623
4560
|
getErrorMessage(error) {
|
|
3624
4561
|
if (error instanceof Error) {
|
|
@@ -3633,28 +4570,53 @@ var SocketController = class _SocketController {
|
|
|
3633
4570
|
return String(error);
|
|
3634
4571
|
}
|
|
3635
4572
|
}
|
|
3636
|
-
|
|
3637
|
-
const state
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
4573
|
+
pruneDiagnosticsEntries(entries, now = Date.now()) {
|
|
4574
|
+
for (const [fetchId, state] of Object.entries(entries)) {
|
|
4575
|
+
if (state.destroyed && now - state.updatedAt > this.destroyedDiagnosticsTtlMs) {
|
|
4576
|
+
delete entries[fetchId];
|
|
4577
|
+
}
|
|
4578
|
+
}
|
|
4579
|
+
if (Object.keys(entries).length <= this.diagnosticsMaxClientEntries) {
|
|
4580
|
+
return;
|
|
4581
|
+
}
|
|
4582
|
+
const entriesByEvictionPriority = Object.entries(entries).sort((left, right) => {
|
|
4583
|
+
if (left[1].destroyed !== right[1].destroyed) {
|
|
4584
|
+
return left[1].destroyed ? -1 : 1;
|
|
4585
|
+
}
|
|
4586
|
+
return left[1].updatedAt - right[1].updatedAt;
|
|
3646
4587
|
});
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
state.errorHistory.length - this.diagnosticsErrorHistoryLimit
|
|
3651
|
-
);
|
|
4588
|
+
while (Object.keys(entries).length > this.diagnosticsMaxClientEntries && entriesByEvictionPriority.length > 0) {
|
|
4589
|
+
const [fetchId] = entriesByEvictionPriority.shift();
|
|
4590
|
+
delete entries[fetchId];
|
|
3652
4591
|
}
|
|
3653
4592
|
}
|
|
3654
|
-
|
|
4593
|
+
async getSocketClientDiagnosticsEntry(fetchId) {
|
|
4594
|
+
const normalized = String(fetchId ?? "").trim();
|
|
4595
|
+
if (!normalized) {
|
|
4596
|
+
return void 0;
|
|
4597
|
+
}
|
|
4598
|
+
const snapshot = this.socketClientDiagnosticsActor.getState();
|
|
4599
|
+
const entries = { ...snapshot.entries };
|
|
4600
|
+
this.pruneDiagnosticsEntries(entries);
|
|
4601
|
+
return entries[normalized];
|
|
4602
|
+
}
|
|
4603
|
+
resolveTransportDiagnosticsOptions(ctx) {
|
|
4604
|
+
const detailLevel = ctx.detailLevel === "full" ? "full" : "summary";
|
|
4605
|
+
const includeErrorHistory = Boolean(ctx.includeErrorHistory);
|
|
4606
|
+
const requestedLimit = Number(ctx.errorHistoryLimit);
|
|
4607
|
+
const errorHistoryLimit = Number.isFinite(requestedLimit) ? Math.max(1, Math.min(200, Math.trunc(requestedLimit))) : 10;
|
|
4608
|
+
return {
|
|
4609
|
+
detailLevel,
|
|
4610
|
+
includeErrorHistory,
|
|
4611
|
+
errorHistoryLimit
|
|
4612
|
+
};
|
|
4613
|
+
}
|
|
4614
|
+
collectSocketTransportDiagnostics(ctx, diagnosticsEntries) {
|
|
3655
4615
|
const { detailLevel, includeErrorHistory, errorHistoryLimit } = this.resolveTransportDiagnosticsOptions(ctx);
|
|
3656
4616
|
const serviceName = CadenzaService.serviceRegistry.serviceName ?? "UnknownService";
|
|
3657
|
-
const
|
|
4617
|
+
const entries = { ...diagnosticsEntries };
|
|
4618
|
+
this.pruneDiagnosticsEntries(entries);
|
|
4619
|
+
const states = Object.values(entries).sort(
|
|
3658
4620
|
(a, b) => a.fetchId.localeCompare(b.fetchId)
|
|
3659
4621
|
);
|
|
3660
4622
|
const summary = {
|
|
@@ -3672,10 +4634,7 @@ var SocketController = class _SocketController {
|
|
|
3672
4634
|
0
|
|
3673
4635
|
),
|
|
3674
4636
|
connectErrors: states.reduce((acc, state) => acc + state.connectErrors, 0),
|
|
3675
|
-
reconnectErrors: states.reduce(
|
|
3676
|
-
(acc, state) => acc + state.reconnectErrors,
|
|
3677
|
-
0
|
|
3678
|
-
),
|
|
4637
|
+
reconnectErrors: states.reduce((acc, state) => acc + state.reconnectErrors, 0),
|
|
3679
4638
|
socketErrors: states.reduce((acc, state) => acc + state.socketErrors, 0),
|
|
3680
4639
|
latestError: states.slice().sort((a, b) => b.lastErrorAt - a.lastErrorAt).find((state) => state.lastError)?.lastError ?? null
|
|
3681
4640
|
};
|
|
@@ -7162,6 +8121,14 @@ var CadenzaService = class {
|
|
|
7162
8121
|
options.isMeta = true;
|
|
7163
8122
|
this.createDatabaseService(name, schema, description, options);
|
|
7164
8123
|
}
|
|
8124
|
+
static createActor(spec, options = {}) {
|
|
8125
|
+
this.bootstrap();
|
|
8126
|
+
return new Actor(spec, options);
|
|
8127
|
+
}
|
|
8128
|
+
static createActorFromDefinition(definition, options = {}) {
|
|
8129
|
+
this.bootstrap();
|
|
8130
|
+
return Cadenza.createActorFromDefinition(definition, options);
|
|
8131
|
+
}
|
|
7165
8132
|
/**
|
|
7166
8133
|
* Creates and registers a new task with the provided name, function, and optional details.
|
|
7167
8134
|
*
|
|
@@ -7576,6 +8543,7 @@ CadenzaService.warnedInvalidMetaIntentResponderKeys = /* @__PURE__ */ new Set();
|
|
|
7576
8543
|
|
|
7577
8544
|
// src/index.ts
|
|
7578
8545
|
import {
|
|
8546
|
+
Actor as Actor2,
|
|
7579
8547
|
DebounceTask as DebounceTask2,
|
|
7580
8548
|
EphemeralTask as EphemeralTask2,
|
|
7581
8549
|
GraphRoutine as GraphRoutine2,
|
|
@@ -7583,6 +8551,7 @@ import {
|
|
|
7583
8551
|
} from "@cadenza.io/core";
|
|
7584
8552
|
var index_default = CadenzaService;
|
|
7585
8553
|
export {
|
|
8554
|
+
Actor2 as Actor,
|
|
7586
8555
|
DatabaseTask,
|
|
7587
8556
|
DebounceTask2 as DebounceTask,
|
|
7588
8557
|
DeputyTask,
|