@dudousxd/nestjs-durable-dashboard 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/group-by-partition.d.ts +21 -0
- package/dist/client/group-by-partition.d.ts.map +1 -0
- package/dist/client/group-by-partition.js +47 -0
- package/dist/client/group-by-partition.js.map +1 -0
- package/dist/client/pivot-by-worker.d.ts +24 -0
- package/dist/client/pivot-by-worker.d.ts.map +1 -0
- package/dist/client/pivot-by-worker.js +55 -0
- package/dist/client/pivot-by-worker.js.map +1 -0
- package/dist/client/queue-partition.d.ts +10 -0
- package/dist/client/queue-partition.d.ts.map +1 -0
- package/dist/client/queue-partition.js +16 -0
- package/dist/client/queue-partition.js.map +1 -0
- package/dist/client/summarize-health.d.ts +19 -0
- package/dist/client/summarize-health.d.ts.map +1 -0
- package/dist/client/summarize-health.js +22 -0
- package/dist/client/summarize-health.js.map +1 -0
- package/dist/server/index.cjs +68 -51
- package/dist/server/index.cjs.map +1 -1
- package/dist/server/index.d.cts +31 -14
- package/dist/server/index.d.ts +31 -14
- package/dist/server/index.js +69 -52
- package/dist/server/index.js.map +1 -1
- package/dist/spa/assets/index-B1CZbiKg.js +23 -0
- package/dist/spa/assets/index-DdQ96lMA.css +1 -0
- package/dist/spa/assets/{preview-B6V1trqM.js → preview-DW_bfkiV.js} +1 -1
- package/dist/spa/index.html +3 -3
- package/dist/spa/preview.html +3 -3
- package/package.json +2 -2
- package/dist/spa/assets/index-5kwYNGTJ.css +0 -1
- package/dist/spa/assets/index-DLFzyBlt.js +0 -23
- /package/dist/spa/assets/{index-CeRgaeH7.js → index-Dq-p-Rzl.js} +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { GroupHealth } from './durable-client';
|
|
2
|
+
/** Rolled-up worker health for one tenant-isolation partition (see `partitionOf`). */
|
|
3
|
+
export interface PartitionView {
|
|
4
|
+
partition: string;
|
|
5
|
+
/** Distinct `instanceId`s across every group's `liveWorkers` in this partition. */
|
|
6
|
+
workerCount: number;
|
|
7
|
+
/** Number of groups (queues) bucketed into this partition. */
|
|
8
|
+
handlerCount: number;
|
|
9
|
+
/** Sum of `depth` across the partition's groups. */
|
|
10
|
+
totalDepth: number;
|
|
11
|
+
/** Groups with a nonzero backlog and no live worker. */
|
|
12
|
+
starvedCount: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Bucket `GroupHealth[]` by `partitionOf(group.group)` and roll up counts. A worker that serves
|
|
16
|
+
* multiple queues in the same partition is counted once toward `workerCount` — we dedupe by
|
|
17
|
+
* `instanceId` across the whole partition, not per group. `'default'` (the operator/control-plane
|
|
18
|
+
* partition) sorts first; the rest follow alphabetically.
|
|
19
|
+
*/
|
|
20
|
+
export declare function groupByPartition(groups: GroupHealth[]): PartitionView[];
|
|
21
|
+
//# sourceMappingURL=group-by-partition.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"group-by-partition.d.ts","sourceRoot":"","sources":["../../src/client/group-by-partition.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,sFAAsF;AACtF,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,mFAAmF;IACnF,WAAW,EAAE,MAAM,CAAC;IACpB,8DAA8D;IAC9D,YAAY,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,UAAU,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,aAAa,EAAE,CAsCvE"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { partitionOf } from './queue-partition';
|
|
2
|
+
/**
|
|
3
|
+
* Bucket `GroupHealth[]` by `partitionOf(group.group)` and roll up counts. A worker that serves
|
|
4
|
+
* multiple queues in the same partition is counted once toward `workerCount` — we dedupe by
|
|
5
|
+
* `instanceId` across the whole partition, not per group. `'default'` (the operator/control-plane
|
|
6
|
+
* partition) sorts first; the rest follow alphabetically.
|
|
7
|
+
*/
|
|
8
|
+
export function groupByPartition(groups) {
|
|
9
|
+
const byPartition = new Map();
|
|
10
|
+
for (const group of groups) {
|
|
11
|
+
const partition = partitionOf(group.group);
|
|
12
|
+
const existing = byPartition.get(partition);
|
|
13
|
+
if (existing) {
|
|
14
|
+
existing.push(group);
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
byPartition.set(partition, [group]);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
const views = [...byPartition.entries()].map(([partition, partitionGroups]) => {
|
|
21
|
+
const workerIds = new Set();
|
|
22
|
+
let totalDepth = 0;
|
|
23
|
+
let starvedCount = 0;
|
|
24
|
+
for (const group of partitionGroups) {
|
|
25
|
+
totalDepth += group.depth;
|
|
26
|
+
if (group.depth > 0 && group.liveWorkers.length === 0)
|
|
27
|
+
starvedCount += 1;
|
|
28
|
+
for (const worker of group.liveWorkers)
|
|
29
|
+
workerIds.add(worker.instanceId);
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
partition,
|
|
33
|
+
workerCount: workerIds.size,
|
|
34
|
+
handlerCount: partitionGroups.length,
|
|
35
|
+
totalDepth,
|
|
36
|
+
starvedCount,
|
|
37
|
+
};
|
|
38
|
+
});
|
|
39
|
+
return views.sort((a, b) => {
|
|
40
|
+
if (a.partition === 'default')
|
|
41
|
+
return -1;
|
|
42
|
+
if (b.partition === 'default')
|
|
43
|
+
return 1;
|
|
44
|
+
return a.partition.localeCompare(b.partition);
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=group-by-partition.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"group-by-partition.js","sourceRoot":"","sources":["../../src/client/group-by-partition.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAehD;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAqB;IACpD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAyB,CAAC;IAErD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,EAAE;QAC5E,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,UAAU,IAAI,KAAK,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,YAAY,IAAI,CAAC,CAAC;YACzE,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,WAAW;gBAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3E,CAAC;QAED,OAAO;YACL,SAAS;YACT,WAAW,EAAE,SAAS,CAAC,IAAI;YAC3B,YAAY,EAAE,eAAe,CAAC,MAAM;YACpC,UAAU;YACV,YAAY;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACzB,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO,CAAC,CAAC;QACxC,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { GroupHealth, WorkerStatus } from './durable-client';
|
|
2
|
+
/** One real worker pod, pivoted out of the `group → liveWorkers` shape the engine reports. A single
|
|
3
|
+
* process subscribes to many handler queues and heartbeats into each, so `GroupHealth.liveWorkers`
|
|
4
|
+
* repeats the same `instanceId` under every group it serves — this collapses those back to one row. */
|
|
5
|
+
export interface WorkerView {
|
|
6
|
+
instanceId: string;
|
|
7
|
+
/** The partition this worker serves; its groups share one (see `partitionOf`). */
|
|
8
|
+
partition: string;
|
|
9
|
+
runtime?: 'node' | 'python' | undefined;
|
|
10
|
+
/** `baseHandlerName()` of every group this worker appears in, deduped and sorted ascending. */
|
|
11
|
+
handlers: string[];
|
|
12
|
+
/** From the heartbeat with the greatest `lastBeatAt`. */
|
|
13
|
+
status?: WorkerStatus | undefined;
|
|
14
|
+
/** Max `lastBeatAt` across this worker's heartbeats. */
|
|
15
|
+
lastBeatAt: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Pivot per-group worker health into per-worker health: `group → workers` becomes
|
|
19
|
+
* `worker → the groups it serves`. Each worker's `status`/`runtime` come from its most recent
|
|
20
|
+
* heartbeat (greatest `lastBeatAt`); `partition` follows that same latest heartbeat's group, which
|
|
21
|
+
* also covers the (should-never-happen) case of a worker whose groups disagree on partition.
|
|
22
|
+
*/
|
|
23
|
+
export declare function pivotByWorker(groups: GroupHealth[]): WorkerView[];
|
|
24
|
+
//# sourceMappingURL=pivot-by-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pivot-by-worker.d.ts","sourceRoot":"","sources":["../../src/client/pivot-by-worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAGlE;;wGAEwG;AACxG,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;IACxC,+FAA+F;IAC/F,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,yDAAyD;IACzD,MAAM,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IAClC,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;CACpB;AAYD;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,UAAU,EAAE,CA8CjE"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { baseHandlerName, partitionOf } from './queue-partition';
|
|
2
|
+
/**
|
|
3
|
+
* Pivot per-group worker health into per-worker health: `group → workers` becomes
|
|
4
|
+
* `worker → the groups it serves`. Each worker's `status`/`runtime` come from its most recent
|
|
5
|
+
* heartbeat (greatest `lastBeatAt`); `partition` follows that same latest heartbeat's group, which
|
|
6
|
+
* also covers the (should-never-happen) case of a worker whose groups disagree on partition.
|
|
7
|
+
*/
|
|
8
|
+
export function pivotByWorker(groups) {
|
|
9
|
+
const byInstance = new Map();
|
|
10
|
+
for (const group of groups) {
|
|
11
|
+
for (const heartbeat of group.liveWorkers) {
|
|
12
|
+
const handler = baseHandlerName(heartbeat.group);
|
|
13
|
+
const existing = byInstance.get(heartbeat.instanceId);
|
|
14
|
+
if (existing === undefined) {
|
|
15
|
+
byInstance.set(heartbeat.instanceId, {
|
|
16
|
+
instanceId: heartbeat.instanceId,
|
|
17
|
+
partition: partitionOf(heartbeat.group),
|
|
18
|
+
runtime: heartbeat.status?.runtime,
|
|
19
|
+
handlers: new Set([handler]),
|
|
20
|
+
status: heartbeat.status,
|
|
21
|
+
lastBeatAt: heartbeat.lastBeatAt,
|
|
22
|
+
});
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
existing.handlers.add(handler);
|
|
26
|
+
if (heartbeat.lastBeatAt >= existing.lastBeatAt) {
|
|
27
|
+
existing.lastBeatAt = heartbeat.lastBeatAt;
|
|
28
|
+
existing.partition = partitionOf(heartbeat.group);
|
|
29
|
+
existing.runtime = heartbeat.status?.runtime;
|
|
30
|
+
existing.status = heartbeat.status;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return [...byInstance.values()]
|
|
35
|
+
.map((worker) => ({
|
|
36
|
+
instanceId: worker.instanceId,
|
|
37
|
+
partition: worker.partition,
|
|
38
|
+
runtime: worker.runtime,
|
|
39
|
+
handlers: [...worker.handlers].sort((a, b) => a.localeCompare(b)),
|
|
40
|
+
status: worker.status,
|
|
41
|
+
lastBeatAt: worker.lastBeatAt,
|
|
42
|
+
}))
|
|
43
|
+
.sort((a, b) => {
|
|
44
|
+
if (a.partition !== b.partition) {
|
|
45
|
+
// 'default' always sorts first; everything else falls back to alphabetical.
|
|
46
|
+
if (a.partition === 'default')
|
|
47
|
+
return -1;
|
|
48
|
+
if (b.partition === 'default')
|
|
49
|
+
return 1;
|
|
50
|
+
return a.partition.localeCompare(b.partition);
|
|
51
|
+
}
|
|
52
|
+
return a.instanceId.localeCompare(b.instanceId);
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=pivot-by-worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pivot-by-worker.js","sourceRoot":"","sources":["../../src/client/pivot-by-worker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AA4BjE;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,MAAqB;IACjD,MAAM,UAAU,GAAG,IAAI,GAAG,EAA6B,CAAC;IAExD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YACtD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE;oBACnC,UAAU,EAAE,SAAS,CAAC,UAAU;oBAChC,SAAS,EAAE,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC;oBACvC,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,OAAO;oBAClC,QAAQ,EAAE,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;oBAC5B,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,UAAU,EAAE,SAAS,CAAC,UAAU;iBACjC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/B,IAAI,SAAS,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;gBAChD,QAAQ,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;gBAC3C,QAAQ,CAAC,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBAClD,QAAQ,CAAC,OAAO,GAAG,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC;gBAC7C,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;YACrC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;SAC5B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,QAAQ,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;YAChC,4EAA4E;YAC5E,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC;YACxC,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Route-by-handler queue tokens are `<handler>@<partition>` (see core `tenantGroup`); the operator's
|
|
3
|
+
* default partition carries no suffix. A handler/workflow name never contains `@`, so the last `@`
|
|
4
|
+
* unambiguously separates the partition.
|
|
5
|
+
*/
|
|
6
|
+
/** The isolation partition a worker-queue token belongs to (`'default'` when it has no suffix). */
|
|
7
|
+
export declare function partitionOf(group: string): string;
|
|
8
|
+
/** The handler/workflow name with any `@<partition>` suffix stripped. */
|
|
9
|
+
export declare function baseHandlerName(group: string): string;
|
|
10
|
+
//# sourceMappingURL=queue-partition.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue-partition.d.ts","sourceRoot":"","sources":["../../src/client/queue-partition.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,mGAAmG;AACnG,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGjD;AAED,yEAAyE;AACzE,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGrD"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Route-by-handler queue tokens are `<handler>@<partition>` (see core `tenantGroup`); the operator's
|
|
3
|
+
* default partition carries no suffix. A handler/workflow name never contains `@`, so the last `@`
|
|
4
|
+
* unambiguously separates the partition.
|
|
5
|
+
*/
|
|
6
|
+
/** The isolation partition a worker-queue token belongs to (`'default'` when it has no suffix). */
|
|
7
|
+
export function partitionOf(group) {
|
|
8
|
+
const at = group.lastIndexOf('@');
|
|
9
|
+
return at === -1 ? 'default' : group.slice(at + 1);
|
|
10
|
+
}
|
|
11
|
+
/** The handler/workflow name with any `@<partition>` suffix stripped. */
|
|
12
|
+
export function baseHandlerName(group) {
|
|
13
|
+
const at = group.lastIndexOf('@');
|
|
14
|
+
return at === -1 ? group : group.slice(0, at);
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=queue-partition.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue-partition.js","sourceRoot":"","sources":["../../src/client/queue-partition.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,mGAAmG;AACnG,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,EAAE,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,MAAM,EAAE,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAChD,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { GroupHealth } from './durable-client';
|
|
2
|
+
/** Health-first summary of the whole `/durable` worker fleet, across every queue. */
|
|
3
|
+
export interface HealthSummary {
|
|
4
|
+
/** Total number of groups (queues), healthy or not. */
|
|
5
|
+
queueCount: number;
|
|
6
|
+
/** Distinct `instanceId`s across every group's `liveWorkers` — a worker on many queues counts once. */
|
|
7
|
+
workerCount: number;
|
|
8
|
+
/** Groups with backlog and no worker consuming it, sorted by `depth` descending. */
|
|
9
|
+
starved: GroupHealth[];
|
|
10
|
+
/** True when `starved` is empty — the whole fleet is draining. */
|
|
11
|
+
allDraining: boolean;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Reduce the (potentially ~40-queue) `GroupHealth` list to the one actionable signal: which
|
|
15
|
+
* queues have backlog with nobody consuming it. `Array.prototype.sort` is stable (ES2019+), so
|
|
16
|
+
* groups tied on `depth` keep their input order.
|
|
17
|
+
*/
|
|
18
|
+
export declare function summarizeHealth(groups: GroupHealth[]): HealthSummary;
|
|
19
|
+
//# sourceMappingURL=summarize-health.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summarize-health.d.ts","sourceRoot":"","sources":["../../src/client/summarize-health.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,qFAAqF;AACrF,MAAM,WAAW,aAAa;IAC5B,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,uGAAuG;IACvG,WAAW,EAAE,MAAM,CAAC;IACpB,oFAAoF;IACpF,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,kEAAkE;IAClE,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,aAAa,CAgBpE"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reduce the (potentially ~40-queue) `GroupHealth` list to the one actionable signal: which
|
|
3
|
+
* queues have backlog with nobody consuming it. `Array.prototype.sort` is stable (ES2019+), so
|
|
4
|
+
* groups tied on `depth` keep their input order.
|
|
5
|
+
*/
|
|
6
|
+
export function summarizeHealth(groups) {
|
|
7
|
+
const instanceIds = new Set();
|
|
8
|
+
for (const group of groups) {
|
|
9
|
+
for (const worker of group.liveWorkers)
|
|
10
|
+
instanceIds.add(worker.instanceId);
|
|
11
|
+
}
|
|
12
|
+
const starved = groups
|
|
13
|
+
.filter((group) => group.depth > 0 && group.liveWorkers.length === 0)
|
|
14
|
+
.sort((a, b) => b.depth - a.depth);
|
|
15
|
+
return {
|
|
16
|
+
queueCount: groups.length,
|
|
17
|
+
workerCount: instanceIds.size,
|
|
18
|
+
starved,
|
|
19
|
+
allDraining: starved.length === 0,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=summarize-health.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summarize-health.js","sourceRoot":"","sources":["../../src/client/summarize-health.ts"],"names":[],"mappings":"AAcA;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAqB;IACnD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,WAAW;YAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,OAAO,GAAG,MAAM;SACnB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC;SACpE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAErC,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,WAAW,EAAE,WAAW,CAAC,IAAI;QAC7B,OAAO;QACP,WAAW,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC;KAClC,CAAC;AACJ,CAAC"}
|
package/dist/server/index.cjs
CHANGED
|
@@ -36,6 +36,11 @@ module.exports = __toCommonJS(index_exports);
|
|
|
36
36
|
var import_nestjs_durable_core = require("@dudousxd/nestjs-durable-core");
|
|
37
37
|
var import_common = require("@nestjs/common");
|
|
38
38
|
var import_rxjs = require("rxjs");
|
|
39
|
+
|
|
40
|
+
// src/server/tokens.ts
|
|
41
|
+
var RUN_GATEWAY = Symbol.for("nestjs-durable:run-gateway");
|
|
42
|
+
|
|
43
|
+
// src/server/dashboard.service.ts
|
|
39
44
|
function _ts_decorate(decorators, target, key, desc) {
|
|
40
45
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
41
46
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -57,33 +62,52 @@ var DashboardService = class {
|
|
|
57
62
|
static {
|
|
58
63
|
__name(this, "DashboardService");
|
|
59
64
|
}
|
|
65
|
+
gateway;
|
|
60
66
|
store;
|
|
61
67
|
engine;
|
|
62
|
-
/** Prometheus counters accumulated from engine events since boot (per process)
|
|
68
|
+
/** Prometheus counters accumulated from engine events since boot (per process); undefined on a
|
|
69
|
+
* store-less tenant (no engine to accumulate from). */
|
|
63
70
|
metricsCollector;
|
|
64
|
-
constructor(store, engine) {
|
|
71
|
+
constructor(gateway, store, engine) {
|
|
72
|
+
this.gateway = gateway;
|
|
65
73
|
this.store = store;
|
|
66
74
|
this.engine = engine;
|
|
67
|
-
this.metricsCollector = (0, import_nestjs_durable_core.collectMetrics)(this.engine);
|
|
75
|
+
this.metricsCollector = this.store && this.engine ? (0, import_nestjs_durable_core.collectMetrics)(this.engine) : void 0;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* The operator-only ops need the FULL engine + canonical store, which only the control plane has.
|
|
79
|
+
* On a tenant the `WorkflowEngine` token is a store-less start client that implements none of these
|
|
80
|
+
* methods, so presence of the canonical store is the reliable discriminator — gate on it (never
|
|
81
|
+
* on engine presence, which is truthy on a tenant too).
|
|
82
|
+
*/
|
|
83
|
+
controlPlane() {
|
|
84
|
+
if (!this.store || !this.engine) {
|
|
85
|
+
throw new Error("This durable dashboard operation requires the control plane (not available on a tenant deployment).");
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
store: this.store,
|
|
89
|
+
engine: this.engine
|
|
90
|
+
};
|
|
68
91
|
}
|
|
69
92
|
/**
|
|
70
93
|
* Prometheus-text metrics for a `/metrics` scrape: the event-counters (runs/steps by outcome,
|
|
71
94
|
* per-workflow counts) plus live **backlog gauges** queried at scrape time — `durable_pending_runs`
|
|
72
95
|
* (the dispatch backlog: the key health signal of the dispatch model) and `durable_dead_runs` (DLQ
|
|
73
|
-
* size). Capped per status so the scrape can't load an unbounded result set.
|
|
96
|
+
* size). Capped per status so the scrape can't load an unbounded result set. Control-plane-only.
|
|
74
97
|
*/
|
|
75
98
|
async metrics() {
|
|
99
|
+
const { store, engine } = this.controlPlane();
|
|
76
100
|
const cap = 1e4;
|
|
77
101
|
const [pending, running, dead] = await Promise.all([
|
|
78
|
-
|
|
102
|
+
store.listRuns({
|
|
79
103
|
status: "pending",
|
|
80
104
|
limit: cap
|
|
81
105
|
}),
|
|
82
|
-
|
|
106
|
+
store.listRuns({
|
|
83
107
|
status: "running",
|
|
84
108
|
limit: cap
|
|
85
109
|
}),
|
|
86
|
-
|
|
110
|
+
store.listRuns({
|
|
87
111
|
status: "dead",
|
|
88
112
|
limit: cap
|
|
89
113
|
})
|
|
@@ -96,7 +120,7 @@ var DashboardService = class {
|
|
|
96
120
|
"# TYPE durable_dead_runs gauge",
|
|
97
121
|
`durable_dead_runs ${dead.length}`
|
|
98
122
|
];
|
|
99
|
-
const health = await
|
|
123
|
+
const health = await engine.workerHealth();
|
|
100
124
|
if (health.length > 0) {
|
|
101
125
|
gauges.push("# TYPE durable_group_queue_depth gauge");
|
|
102
126
|
for (const h of health) {
|
|
@@ -107,40 +131,30 @@ var DashboardService = class {
|
|
|
107
131
|
gauges.push(`durable_group_live_workers{group="${h.group}"} ${h.liveWorkers.length}`);
|
|
108
132
|
}
|
|
109
133
|
}
|
|
110
|
-
return `${this.metricsCollector
|
|
134
|
+
return `${this.metricsCollector?.prometheus() ?? ""}${gauges.join("\n")}
|
|
111
135
|
`;
|
|
112
136
|
}
|
|
113
137
|
listRuns(query) {
|
|
114
|
-
return this.
|
|
138
|
+
return this.gateway.listRuns(query);
|
|
115
139
|
}
|
|
116
140
|
/** Per-group worker health (queue backlog + live worker heartbeats) for the Workers panel. The
|
|
117
|
-
* alert state a row turns red on is `depth > 0 && liveWorkers.length === 0`. */
|
|
118
|
-
workerHealth() {
|
|
119
|
-
return this.engine.workerHealth();
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
if (!run) return null;
|
|
124
|
-
const [timeline, children] = await Promise.all([
|
|
125
|
-
this.store.listCheckpoints(runId),
|
|
126
|
-
this.engine.getRunChildren(runId)
|
|
127
|
-
]);
|
|
128
|
-
return {
|
|
129
|
-
run,
|
|
130
|
-
timeline,
|
|
131
|
-
children
|
|
132
|
-
};
|
|
141
|
+
* alert state a row turns red on is `depth > 0 && liveWorkers.length === 0`. Control-plane-only. */
|
|
142
|
+
async workerHealth() {
|
|
143
|
+
return this.controlPlane().engine.workerHealth();
|
|
144
|
+
}
|
|
145
|
+
getRunDetail(runId) {
|
|
146
|
+
return this.gateway.getRunDetail(runId);
|
|
133
147
|
}
|
|
134
148
|
/** Fix-and-replay a dead/failed run with a corrected input — a fresh linked run. Returns its id. */
|
|
135
149
|
retryWithInput(runId, input) {
|
|
136
|
-
return this.
|
|
150
|
+
return this.gateway.retryWithInput(runId, input);
|
|
137
151
|
}
|
|
138
152
|
/** Re-run a failed/incomplete run; completed steps replay from their checkpoints. */
|
|
139
153
|
retry(runId) {
|
|
140
|
-
return this.
|
|
154
|
+
return this.gateway.retry(runId);
|
|
141
155
|
}
|
|
142
156
|
cancel(runId, opts) {
|
|
143
|
-
return this.
|
|
157
|
+
return this.gateway.cancel(runId, opts);
|
|
144
158
|
}
|
|
145
159
|
/**
|
|
146
160
|
* Apply `retry` or `cancel` to every run matching a filter (status / tag / workflow) — e.g. "retry
|
|
@@ -148,15 +162,15 @@ var DashboardService = class {
|
|
|
148
162
|
* (already terminal) are skipped. Returns how many matched and how many the action applied to.
|
|
149
163
|
*/
|
|
150
164
|
async bulk(action, filter, opts) {
|
|
151
|
-
const runs = await this.
|
|
165
|
+
const runs = await this.gateway.listRuns({
|
|
152
166
|
...filter,
|
|
153
167
|
limit: 500
|
|
154
168
|
});
|
|
155
169
|
let applied = 0;
|
|
156
170
|
for (const r of runs) {
|
|
157
171
|
try {
|
|
158
|
-
if (action === "retry") await this.
|
|
159
|
-
else await this.
|
|
172
|
+
if (action === "retry") await this.gateway.retry(r.id);
|
|
173
|
+
else await this.gateway.cancel(r.id, opts);
|
|
160
174
|
applied += 1;
|
|
161
175
|
} catch {
|
|
162
176
|
}
|
|
@@ -168,45 +182,48 @@ var DashboardService = class {
|
|
|
168
182
|
}
|
|
169
183
|
/** Resume a run paused at a `ctx.breakpoint` (the "continue" button). */
|
|
170
184
|
continue(runId) {
|
|
171
|
-
return this.
|
|
185
|
+
return this.gateway.continue(runId);
|
|
172
186
|
}
|
|
173
187
|
/**
|
|
174
188
|
* Deliver a `ctx.webhook()` callback: turn an inbound POST (token + body) into the signal the
|
|
175
189
|
* waiting run is parked on. Returns the run result, or `null` if no run waits on that token (a
|
|
176
|
-
* stale/duplicate callback) — a safe no-op the controller maps to 404.
|
|
190
|
+
* stale/duplicate callback) — a safe no-op the controller maps to 404. Control-plane-only.
|
|
177
191
|
*/
|
|
178
|
-
deliverWebhook(token, body) {
|
|
179
|
-
return this.engine.signal(token, body);
|
|
192
|
+
async deliverWebhook(token, body) {
|
|
193
|
+
return this.controlPlane().engine.signal(token, body);
|
|
180
194
|
}
|
|
181
|
-
/** Side-effect-free read of a value a run published via `ctx.setEvent` (a live query). */
|
|
182
|
-
getEvent(runId, key) {
|
|
183
|
-
return this.engine.getEvent(runId, key);
|
|
195
|
+
/** Side-effect-free read of a value a run published via `ctx.setEvent` (a live query). Control-plane-only. */
|
|
196
|
+
async getEvent(runId, key) {
|
|
197
|
+
return this.controlPlane().engine.getEvent(runId, key);
|
|
184
198
|
}
|
|
185
|
-
/** Deliver a validated `ctx.onUpdate` to a run; the validator may reject it (see UpdateResult). */
|
|
186
|
-
update(runId, name, arg) {
|
|
187
|
-
return this.engine.update(runId, name, arg);
|
|
199
|
+
/** Deliver a validated `ctx.onUpdate` to a run; the validator may reject it (see UpdateResult). Control-plane-only. */
|
|
200
|
+
async update(runId, name, arg) {
|
|
201
|
+
return this.controlPlane().engine.update(runId, name, arg);
|
|
188
202
|
}
|
|
189
203
|
/**
|
|
190
|
-
* Live stream of a run's lifecycle events for SSE. Backed by `
|
|
191
|
-
*
|
|
192
|
-
*
|
|
204
|
+
* Live stream of a run's lifecycle events for SSE. Backed by `gateway.subscribe`, which — on the
|
|
205
|
+
* control plane — receives events from EVERY instance (so a dashboard-only pod tails a run
|
|
206
|
+
* executing on a worker pod), and on a tenant round-trips over the transport. Either way the
|
|
207
|
+
* gateway already filters to `runId`; the dashboard just wires the events through.
|
|
193
208
|
*/
|
|
194
209
|
streamRun(runId) {
|
|
195
210
|
return new import_rxjs.Observable((subscriber) => {
|
|
196
|
-
const off = this.
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
});
|
|
200
|
-
});
|
|
211
|
+
const off = this.gateway.subscribe(runId, (event) => subscriber.next({
|
|
212
|
+
data: event
|
|
213
|
+
}));
|
|
201
214
|
return () => off();
|
|
202
215
|
});
|
|
203
216
|
}
|
|
204
217
|
};
|
|
205
218
|
DashboardService = _ts_decorate([
|
|
206
219
|
(0, import_common.Injectable)(),
|
|
207
|
-
_ts_param(0, (0, import_common.Inject)(
|
|
220
|
+
_ts_param(0, (0, import_common.Inject)(RUN_GATEWAY)),
|
|
221
|
+
_ts_param(1, (0, import_common.Optional)()),
|
|
222
|
+
_ts_param(1, (0, import_common.Inject)(import_nestjs_durable_core.STATE_STORE_CANONICAL)),
|
|
223
|
+
_ts_param(2, (0, import_common.Optional)()),
|
|
208
224
|
_ts_metadata("design:type", Function),
|
|
209
225
|
_ts_metadata("design:paramtypes", [
|
|
226
|
+
typeof RunGateway === "undefined" ? Object : RunGateway,
|
|
210
227
|
typeof StateStore === "undefined" ? Object : StateStore,
|
|
211
228
|
typeof import_nestjs_durable_core.WorkflowEngine === "undefined" ? Object : import_nestjs_durable_core.WorkflowEngine
|
|
212
229
|
])
|