@mastra/factory 0.29.0-alpha.2 → 0.29.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -5
- package/dist/factory.d.ts +8 -0
- package/dist/factory.d.ts.map +1 -1
- package/dist/factory.js +26 -10
- package/dist/factory.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -2
- package/dist/integrations/dependabot/rules.d.ts.map +1 -1
- package/dist/integrations/dependabot/rules.js +1 -5
- package/dist/integrations/dependabot/rules.js.map +1 -1
- package/dist/integrations/github/commits.d.ts +0 -21
- package/dist/integrations/github/commits.d.ts.map +1 -1
- package/dist/integrations/github/commits.js +1 -24
- package/dist/integrations/github/commits.js.map +1 -1
- package/dist/integrations/github/integration.d.ts +1 -1
- package/dist/integrations/github/integration.d.ts.map +1 -1
- package/dist/integrations/github/integration.js +2 -6
- package/dist/integrations/github/integration.js.map +1 -1
- package/dist/integrations/github/reconcile-worker.d.ts.map +1 -1
- package/dist/integrations/github/reconcile-worker.js +30 -1
- package/dist/integrations/github/reconcile-worker.js.map +1 -1
- package/dist/integrations/github/routes.d.ts +1 -11
- package/dist/integrations/github/routes.d.ts.map +1 -1
- package/dist/integrations/github/routes.js +6 -515
- package/dist/integrations/github/routes.js.map +1 -1
- package/dist/integrations/github/rules.d.ts.map +1 -1
- package/dist/integrations/github/rules.js +76 -13
- package/dist/integrations/github/rules.js.map +1 -1
- package/dist/integrations/github/sandbox.d.ts.map +1 -1
- package/dist/integrations/github/sandbox.js +11 -2
- package/dist/integrations/github/sandbox.js.map +1 -1
- package/dist/integrations/gitlab/discovery.d.ts +1 -1
- package/dist/integrations/gitlab/discovery.js +1 -1
- package/dist/integrations/gitlab/discovery.js.map +1 -1
- package/dist/integrations/gitlab/integration.d.ts.map +1 -1
- package/dist/integrations/gitlab/integration.js +0 -1
- package/dist/integrations/gitlab/integration.js.map +1 -1
- package/dist/integrations/gitlab/routes.d.ts +0 -2
- package/dist/integrations/gitlab/routes.d.ts.map +1 -1
- package/dist/integrations/gitlab/routes.js +49 -165
- package/dist/integrations/gitlab/routes.js.map +1 -1
- package/dist/integrations/gitlab/rules.d.ts.map +1 -1
- package/dist/integrations/gitlab/rules.js +20 -7
- package/dist/integrations/gitlab/rules.js.map +1 -1
- package/dist/integrations/incidentio/routes.d.ts +5 -5
- package/dist/integrations/incidentio/routes.d.ts.map +1 -1
- package/dist/integrations/incidentio/routes.js +3 -59
- package/dist/integrations/incidentio/routes.js.map +1 -1
- package/dist/integrations/incidentio/rules.d.ts.map +1 -1
- package/dist/integrations/incidentio/rules.js +1 -5
- package/dist/integrations/incidentio/rules.js.map +1 -1
- package/dist/integrations/jira/issue-reconciler.d.ts +1 -1
- package/dist/integrations/jira/issue-reconciler.d.ts.map +1 -1
- package/dist/integrations/jira/issue-reconciler.js.map +1 -1
- package/dist/integrations/jira/routes.d.ts +2 -2
- package/dist/integrations/jira/routes.d.ts.map +1 -1
- package/dist/integrations/jira/routes.js +0 -68
- package/dist/integrations/jira/routes.js.map +1 -1
- package/dist/integrations/jira/rules.d.ts.map +1 -1
- package/dist/integrations/jira/rules.js +1 -5
- package/dist/integrations/jira/rules.js.map +1 -1
- package/dist/integrations/linear/routes.d.ts +2 -2
- package/dist/integrations/linear/routes.d.ts.map +1 -1
- package/dist/integrations/linear/routes.js +0 -83
- package/dist/integrations/linear/routes.js.map +1 -1
- package/dist/integrations/linear/rules.d.ts.map +1 -1
- package/dist/integrations/linear/rules.js +12 -5
- package/dist/integrations/linear/rules.js.map +1 -1
- package/dist/integrations/platform/github/integration.d.ts.map +1 -1
- package/dist/integrations/platform/github/integration.js +1 -3
- package/dist/integrations/platform/github/integration.js.map +1 -1
- package/dist/integrations/platform/jira/routes.d.ts +2 -2
- package/dist/integrations/platform/jira/routes.d.ts.map +1 -1
- package/dist/integrations/platform/jira/routes.js +0 -69
- package/dist/integrations/platform/jira/routes.js.map +1 -1
- package/dist/integrations/vanta/rules.d.ts.map +1 -1
- package/dist/integrations/vanta/rules.js +1 -5
- package/dist/integrations/vanta/rules.js.map +1 -1
- package/dist/metrics/collectors/postgres.d.ts +29 -0
- package/dist/metrics/collectors/postgres.d.ts.map +1 -0
- package/dist/metrics/collectors/postgres.js +121 -0
- package/dist/metrics/collectors/postgres.js.map +1 -0
- package/dist/metrics/collectors/probe.d.ts +22 -0
- package/dist/metrics/collectors/probe.d.ts.map +1 -0
- package/dist/metrics/collectors/probe.js +69 -0
- package/dist/metrics/collectors/probe.js.map +1 -0
- package/dist/metrics/collectors/process.d.ts +13 -0
- package/dist/metrics/collectors/process.d.ts.map +1 -0
- package/dist/metrics/collectors/process.js +85 -0
- package/dist/metrics/collectors/process.js.map +1 -0
- package/dist/metrics/collectors/redis.d.ts +29 -0
- package/dist/metrics/collectors/redis.d.ts.map +1 -0
- package/dist/metrics/collectors/redis.js +97 -0
- package/dist/metrics/collectors/redis.js.map +1 -0
- package/dist/metrics/config.d.ts +33 -0
- package/dist/metrics/config.d.ts.map +1 -0
- package/dist/metrics/config.js +65 -0
- package/dist/metrics/config.js.map +1 -0
- package/dist/metrics/index.d.ts +20 -0
- package/dist/metrics/index.d.ts.map +1 -0
- package/dist/metrics/index.js +65 -0
- package/dist/metrics/index.js.map +1 -0
- package/dist/metrics/intake.d.ts +28 -0
- package/dist/metrics/intake.d.ts.map +1 -0
- package/dist/metrics/intake.js +76 -0
- package/dist/metrics/intake.js.map +1 -0
- package/dist/metrics/metrics.d.ts +60 -0
- package/dist/metrics/metrics.d.ts.map +1 -0
- package/dist/metrics/metrics.js +182 -0
- package/dist/metrics/metrics.js.map +1 -0
- package/dist/metrics/sandbox.d.ts +35 -0
- package/dist/metrics/sandbox.d.ts.map +1 -0
- package/dist/metrics/sandbox.js +156 -0
- package/dist/metrics/sandbox.js.map +1 -0
- package/dist/packages/_internals/workspace/dist/index.js.map +1 -1
- package/dist/routes/automation-runs.d.ts +0 -2
- package/dist/routes/automation-runs.d.ts.map +1 -1
- package/dist/routes/automation-runs.js +2 -6
- package/dist/routes/automation-runs.js.map +1 -1
- package/dist/routes/contracts.d.ts +0 -5
- package/dist/routes/contracts.d.ts.map +1 -1
- package/dist/routes/contracts.js +2 -4
- package/dist/routes/contracts.js.map +1 -1
- package/dist/routes/fs.d.ts +10 -0
- package/dist/routes/fs.d.ts.map +1 -1
- package/dist/routes/fs.js +184 -11
- package/dist/routes/fs.js.map +1 -1
- package/dist/routes/projects.d.ts.map +1 -1
- package/dist/routes/projects.js +4 -17
- package/dist/routes/projects.js.map +1 -1
- package/dist/routes/source-control-sessions.d.ts +0 -3
- package/dist/routes/source-control-sessions.d.ts.map +1 -1
- package/dist/routes/source-control-sessions.js +0 -19
- package/dist/routes/source-control-sessions.js.map +1 -1
- package/dist/routes/surface.d.ts +0 -2
- package/dist/routes/surface.d.ts.map +1 -1
- package/dist/routes/surface.js +2 -12
- package/dist/routes/surface.js.map +1 -1
- package/dist/rules/dispatcher.d.ts +1 -0
- package/dist/rules/dispatcher.d.ts.map +1 -1
- package/dist/rules/dispatcher.js +27 -13
- package/dist/rules/dispatcher.js.map +1 -1
- package/dist/rules/processor.d.ts.map +1 -1
- package/dist/rules/processor.js +1 -6
- package/dist/rules/processor.js.map +1 -1
- package/dist/rules/start-coordinator.d.ts +1 -0
- package/dist/rules/start-coordinator.d.ts.map +1 -1
- package/dist/rules/start-coordinator.js +2 -1
- package/dist/rules/start-coordinator.js.map +1 -1
- package/dist/rules/transition-service.d.ts.map +1 -1
- package/dist/rules/transition-service.js +0 -2
- package/dist/rules/transition-service.js.map +1 -1
- package/dist/sandbox/session-retirement.d.ts.map +1 -1
- package/dist/sandbox/session-retirement.js +22 -14
- package/dist/sandbox/session-retirement.js.map +1 -1
- package/dist/sandbox/session-sandbox.d.ts +2 -0
- package/dist/sandbox/session-sandbox.d.ts.map +1 -1
- package/dist/sandbox/session-sandbox.js +7 -1
- package/dist/sandbox/session-sandbox.js.map +1 -1
- package/dist/storage/domains/boards/schemas.d.ts.map +1 -1
- package/dist/storage/domains/boards/schemas.js +0 -4
- package/dist/storage/domains/boards/schemas.js.map +1 -1
- package/dist/storage/domains/intake/base.d.ts +11 -10
- package/dist/storage/domains/intake/base.d.ts.map +1 -1
- package/dist/storage/domains/intake/base.js +30 -65
- package/dist/storage/domains/intake/base.js.map +1 -1
- package/dist/storage/domains/integrations/base.d.ts +0 -2
- package/dist/storage/domains/integrations/base.d.ts.map +1 -1
- package/dist/storage/domains/integrations/base.js +0 -6
- package/dist/storage/domains/integrations/base.js.map +1 -1
- package/dist/storage/domains/projects/base.d.ts.map +1 -1
- package/dist/storage/domains/projects/base.js +0 -12
- package/dist/storage/domains/projects/base.js.map +1 -1
- package/dist/storage/domains/source-control/base.d.ts +0 -10
- package/dist/storage/domains/source-control/base.d.ts.map +1 -1
- package/dist/storage/domains/source-control/base.js +1 -19
- package/dist/storage/domains/source-control/base.js.map +1 -1
- package/dist/storage/domains/source-control/inmemory.d.ts +1 -1
- package/dist/storage/domains/source-control/inmemory.d.ts.map +1 -1
- package/dist/storage/domains/source-control/inmemory.js +1 -6
- package/dist/storage/domains/source-control/inmemory.js.map +1 -1
- package/dist/storage/domains/work-items/base.d.ts +6 -36
- package/dist/storage/domains/work-items/base.d.ts.map +1 -1
- package/dist/storage/domains/work-items/base.js +42 -82
- package/dist/storage/domains/work-items/base.js.map +1 -1
- package/dist/storage/domains/work-items/work-item-schema.d.ts +12 -3
- package/dist/storage/domains/work-items/work-item-schema.d.ts.map +1 -1
- package/dist/storage/domains/work-items/work-item-schema.js +12 -3
- package/dist/storage/domains/work-items/work-item-schema.js.map +1 -1
- package/dist/storage/legacy-collections.d.ts +9 -0
- package/dist/storage/legacy-collections.d.ts.map +1 -0
- package/dist/storage/legacy-collections.js +17 -0
- package/dist/storage/legacy-collections.js.map +1 -0
- package/dist/timing.d.ts +4 -1
- package/dist/timing.d.ts.map +1 -1
- package/dist/timing.js +12 -4
- package/dist/timing.js.map +1 -1
- package/dist/workspace.d.ts.map +1 -1
- package/dist/workspace.js +8 -4
- package/dist/workspace.js.map +1 -1
- package/factory-skills/configure-factory-rules/SKILL.md +2 -2
- package/package.json +10 -10
- package/dist/session/filesystem-capture.d.ts +0 -50
- package/dist/session/filesystem-capture.d.ts.map +0 -1
- package/dist/session/filesystem-capture.js +0 -229
- package/dist/session/filesystem-capture.js.map +0 -1
- package/dist/storage/domains/filesystem/base.d.ts +0 -30
- package/dist/storage/domains/filesystem/base.d.ts.map +0 -1
- package/dist/storage/domains/filesystem/base.js +0 -78
- package/dist/storage/domains/filesystem/base.js.map +0 -1
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { readFileSync } from "fs";
|
|
2
|
+
import { availableParallelism } from "os";
|
|
3
|
+
import { PerformanceObserver, constants, monitorEventLoopDelay } from "perf_hooks";
|
|
4
|
+
import { getHeapStatistics } from "v8";
|
|
5
|
+
// src/metrics/collectors/process.ts
|
|
6
|
+
const LOOP_RESOLUTION_MS = 20;
|
|
7
|
+
function readNumbers(path) {
|
|
8
|
+
try {
|
|
9
|
+
return readFileSync(path, "utf8").trim().split(/\s+/);
|
|
10
|
+
} catch {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Cores this process can use: the container's CPU limit when one is set (cgroup v2 `cpu.max`, or
|
|
16
|
+
* v1 `cpu.cfs_quota_us` / `cpu.cfs_period_us`), otherwise the cores the machine reports.
|
|
17
|
+
*/
|
|
18
|
+
function cpuLimitCores() {
|
|
19
|
+
const parallelism = availableParallelism();
|
|
20
|
+
const v2 = readNumbers("/sys/fs/cgroup/cpu.max");
|
|
21
|
+
const quota = v2 ? Number(v2[0]) : Number(readNumbers("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")?.[0]);
|
|
22
|
+
const period = v2 ? Number(v2[1]) : Number(readNumbers("/sys/fs/cgroup/cpu/cpu.cfs_period_us")?.[0]);
|
|
23
|
+
if (!(quota > 0 && period > 0)) return parallelism;
|
|
24
|
+
return Math.min(quota / period, parallelism);
|
|
25
|
+
}
|
|
26
|
+
const GC_KINDS = {
|
|
27
|
+
[constants.NODE_PERFORMANCE_GC_MAJOR]: "major",
|
|
28
|
+
[constants.NODE_PERFORMANCE_GC_INCREMENTAL]: "incremental"
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Event-loop delay and utilization, memory against the heap limit, CPU against the CPU limit, and
|
|
32
|
+
* uptime of this Node process. Each major or incremental GC pause is passed to `onGcPause` as it
|
|
33
|
+
* happens, to be aggregated per interval.
|
|
34
|
+
*/
|
|
35
|
+
function processCollector(onGcPause) {
|
|
36
|
+
const cpuLimit = cpuLimitCores();
|
|
37
|
+
let lastUtilization = performance.eventLoopUtilization();
|
|
38
|
+
const loop = monitorEventLoopDelay({ resolution: LOOP_RESOLUTION_MS });
|
|
39
|
+
loop.enable();
|
|
40
|
+
let gc;
|
|
41
|
+
if (onGcPause) {
|
|
42
|
+
gc = new PerformanceObserver((list) => {
|
|
43
|
+
for (const entry of list.getEntries()) {
|
|
44
|
+
const kind = GC_KINDS[entry.detail?.kind ?? -1];
|
|
45
|
+
if (kind) onGcPause(kind, entry.duration);
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
gc.observe({ entryTypes: ["gc"] });
|
|
49
|
+
}
|
|
50
|
+
let lastCpu = process.cpuUsage();
|
|
51
|
+
let lastAt = performance.now();
|
|
52
|
+
return {
|
|
53
|
+
collect(sink) {
|
|
54
|
+
const delayMs = (ns) => Math.max(0, ns / 1e6 - LOOP_RESOLUTION_MS);
|
|
55
|
+
sink.emit("mastra_factory_event_loop_delay_ms_count", loop.count);
|
|
56
|
+
sink.emit("mastra_factory_event_loop_delay_ms", delayMs(loop.mean), { stat: "avg" });
|
|
57
|
+
sink.emit("mastra_factory_event_loop_delay_ms", delayMs(loop.percentile(95)), { stat: "p95" });
|
|
58
|
+
sink.emit("mastra_factory_event_loop_delay_ms", delayMs(loop.max), { stat: "max" });
|
|
59
|
+
loop.reset();
|
|
60
|
+
const utilization = performance.eventLoopUtilization(lastUtilization);
|
|
61
|
+
lastUtilization = performance.eventLoopUtilization();
|
|
62
|
+
sink.emit("mastra_factory_event_loop_utilization", utilization.utilization);
|
|
63
|
+
const memory = process.memoryUsage();
|
|
64
|
+
sink.emit("mastra_factory_process_memory_bytes", memory.rss, { kind: "rss" });
|
|
65
|
+
sink.emit("mastra_factory_process_memory_bytes", memory.heapUsed, { kind: "heap_used" });
|
|
66
|
+
const heapLimit = getHeapStatistics().heap_size_limit;
|
|
67
|
+
sink.emit("mastra_factory_process_memory_bytes", heapLimit, { kind: "heap_limit" });
|
|
68
|
+
sink.emit("mastra_factory_process_heap_utilization", memory.heapUsed / heapLimit);
|
|
69
|
+
const cpu = process.cpuUsage(lastCpu);
|
|
70
|
+
const now = performance.now();
|
|
71
|
+
sink.emit("mastra_factory_process_cpu_cores", (cpu.user + cpu.system) / 1e3 / (now - lastAt));
|
|
72
|
+
lastCpu = process.cpuUsage();
|
|
73
|
+
lastAt = now;
|
|
74
|
+
sink.emit("mastra_factory_process_cpu_limit_cores", cpuLimit);
|
|
75
|
+
sink.emit("mastra_factory_process_uptime_ms", process.uptime() * 1e3);
|
|
76
|
+
},
|
|
77
|
+
stop() {
|
|
78
|
+
loop.disable();
|
|
79
|
+
gc?.disconnect();
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export { cpuLimitCores, processCollector };
|
|
84
|
+
|
|
85
|
+
//# sourceMappingURL=process.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"process.js","names":[],"sources":["../../../src/metrics/collectors/process.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { availableParallelism } from 'node:os';\nimport { constants, monitorEventLoopDelay, PerformanceObserver } from 'node:perf_hooks';\nimport type { PerformanceEntry } from 'node:perf_hooks';\nimport { getHeapStatistics } from 'node:v8';\n\nimport type { MetricsCollector } from '../metrics.js';\n\nconst LOOP_RESOLUTION_MS = 20;\n\nfunction readNumbers(path: string): string[] | undefined {\n try {\n return readFileSync(path, 'utf8').trim().split(/\\s+/);\n } catch {\n return undefined;\n }\n}\n\n/**\n * Cores this process can use: the container's CPU limit when one is set (cgroup v2 `cpu.max`, or\n * v1 `cpu.cfs_quota_us` / `cpu.cfs_period_us`), otherwise the cores the machine reports.\n */\nexport function cpuLimitCores(): number {\n const parallelism = availableParallelism();\n const v2 = readNumbers('/sys/fs/cgroup/cpu.max');\n const quota = v2 ? Number(v2[0]) : Number(readNumbers('/sys/fs/cgroup/cpu/cpu.cfs_quota_us')?.[0]);\n const period = v2 ? Number(v2[1]) : Number(readNumbers('/sys/fs/cgroup/cpu/cpu.cfs_period_us')?.[0]);\n if (!(quota > 0 && period > 0)) return parallelism;\n return Math.min(quota / period, parallelism);\n}\n\n// Only the pauses worth a series: minor pauses already show in event-loop delay, and weak-callback\n// passes are noise.\nconst GC_KINDS: Record<number, string> = {\n [constants.NODE_PERFORMANCE_GC_MAJOR]: 'major',\n [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental',\n};\n\n/**\n * Event-loop delay and utilization, memory against the heap limit, CPU against the CPU limit, and\n * uptime of this Node process. Each major or incremental GC pause is passed to `onGcPause` as it\n * happens, to be aggregated per interval.\n */\nexport function processCollector(onGcPause?: (kind: string, ms: number) => void): MetricsCollector {\n const cpuLimit = cpuLimitCores();\n let lastUtilization = performance.eventLoopUtilization();\n const loop = monitorEventLoopDelay({ resolution: LOOP_RESOLUTION_MS });\n loop.enable();\n let gc: PerformanceObserver | undefined;\n if (onGcPause) {\n gc = new PerformanceObserver(list => {\n for (const entry of list.getEntries()) {\n const kind = GC_KINDS[(entry as PerformanceEntry & { detail?: { kind?: number } }).detail?.kind ?? -1];\n if (kind) onGcPause(kind, entry.duration);\n }\n });\n gc.observe({ entryTypes: ['gc'] });\n }\n let lastCpu = process.cpuUsage();\n let lastAt = performance.now();\n return {\n collect(sink) {\n // The histogram records each sampling timer's full wait, so an idle loop reads as the\n // resolution; subtract it to get the delay.\n const delayMs = (ns: number) => Math.max(0, ns / 1e6 - LOOP_RESOLUTION_MS);\n // Same shape as the other timings: `stat=avg|p95|max` plus a `_count` series (delay samples taken).\n sink.emit('mastra_factory_event_loop_delay_ms_count', loop.count);\n sink.emit('mastra_factory_event_loop_delay_ms', delayMs(loop.mean), { stat: 'avg' });\n sink.emit('mastra_factory_event_loop_delay_ms', delayMs(loop.percentile(95)), { stat: 'p95' });\n sink.emit('mastra_factory_event_loop_delay_ms', delayMs(loop.max), { stat: 'max' });\n loop.reset();\n\n // Share of the interval the main thread spent running JavaScript and callbacks, 0 to 1.\n const utilization = performance.eventLoopUtilization(lastUtilization);\n lastUtilization = performance.eventLoopUtilization();\n sink.emit('mastra_factory_event_loop_utilization', utilization.utilization);\n\n const memory = process.memoryUsage();\n sink.emit('mastra_factory_process_memory_bytes', memory.rss, { kind: 'rss' });\n sink.emit('mastra_factory_process_memory_bytes', memory.heapUsed, { kind: 'heap_used' });\n const heapLimit = getHeapStatistics().heap_size_limit;\n sink.emit('mastra_factory_process_memory_bytes', heapLimit, { kind: 'heap_limit' });\n sink.emit('mastra_factory_process_heap_utilization', memory.heapUsed / heapLimit);\n\n // CPU used by every thread of the process (main thread, GC helpers, the libuv pool, workers)\n // since the last sample, in cores: 1.4 means 1.4 cores' worth, like 140% in top. It can go\n // above 1; compare it to the limit, not to 1. The main thread's load is event-loop utilization.\n const cpu = process.cpuUsage(lastCpu);\n const now = performance.now();\n sink.emit('mastra_factory_process_cpu_cores', (cpu.user + cpu.system) / 1000 / (now - lastAt));\n lastCpu = process.cpuUsage();\n lastAt = now;\n sink.emit('mastra_factory_process_cpu_limit_cores', cpuLimit);\n\n sink.emit('mastra_factory_process_uptime_ms', process.uptime() * 1000);\n },\n stop() {\n loop.disable();\n gc?.disconnect();\n },\n };\n}\n"],"mappings":";;;;;AAQA,MAAM,qBAAqB;AAE3B,SAAS,YAAY,MAAoC;CACvD,IAAI;EACF,OAAO,aAAa,MAAM,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK;CACtD,QAAQ;EACN;CACF;AACF;;;;;AAMA,SAAgB,gBAAwB;CACtC,MAAM,cAAc,qBAAqB;CACzC,MAAM,KAAK,YAAY,wBAAwB;CAC/C,MAAM,QAAQ,KAAK,OAAO,GAAG,EAAE,IAAI,OAAO,YAAY,qCAAqC,CAAC,GAAG,EAAE;CACjG,MAAM,SAAS,KAAK,OAAO,GAAG,EAAE,IAAI,OAAO,YAAY,sCAAsC,CAAC,GAAG,EAAE;CACnG,IAAI,EAAE,QAAQ,KAAK,SAAS,IAAI,OAAO;CACvC,OAAO,KAAK,IAAI,QAAQ,QAAQ,WAAW;AAC7C;AAIA,MAAM,WAAmC;EACtC,UAAU,4BAA4B;EACtC,UAAU,kCAAkC;AAC/C;;;;;;AAOA,SAAgB,iBAAiB,WAAkE;CACjG,MAAM,WAAW,cAAc;CAC/B,IAAI,kBAAkB,YAAY,qBAAqB;CACvD,MAAM,OAAO,sBAAsB,EAAE,YAAY,mBAAmB,CAAC;CACrE,KAAK,OAAO;CACZ,IAAI;CACJ,IAAI,WAAW;EACb,KAAK,IAAI,qBAAoB,SAAQ;GACnC,KAAK,MAAM,SAAS,KAAK,WAAW,GAAG;IACrC,MAAM,OAAO,SAAU,MAA4D,QAAQ,QAAQ;IACnG,IAAI,MAAM,UAAU,MAAM,MAAM,QAAQ;GAC1C;EACF,CAAC;EACD,GAAG,QAAQ,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC;CACnC;CACA,IAAI,UAAU,QAAQ,SAAS;CAC/B,IAAI,SAAS,YAAY,IAAI;CAC7B,OAAO;EACL,QAAQ,MAAM;GAGZ,MAAM,WAAW,OAAe,KAAK,IAAI,GAAG,KAAK,MAAM,kBAAkB;GAEzE,KAAK,KAAK,4CAA4C,KAAK,KAAK;GAChE,KAAK,KAAK,sCAAsC,QAAQ,KAAK,IAAI,GAAG,EAAE,MAAM,MAAM,CAAC;GACnF,KAAK,KAAK,sCAAsC,QAAQ,KAAK,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;GAC7F,KAAK,KAAK,sCAAsC,QAAQ,KAAK,GAAG,GAAG,EAAE,MAAM,MAAM,CAAC;GAClF,KAAK,MAAM;GAGX,MAAM,cAAc,YAAY,qBAAqB,eAAe;GACpE,kBAAkB,YAAY,qBAAqB;GACnD,KAAK,KAAK,yCAAyC,YAAY,WAAW;GAE1E,MAAM,SAAS,QAAQ,YAAY;GACnC,KAAK,KAAK,uCAAuC,OAAO,KAAK,EAAE,MAAM,MAAM,CAAC;GAC5E,KAAK,KAAK,uCAAuC,OAAO,UAAU,EAAE,MAAM,YAAY,CAAC;GACvF,MAAM,YAAY,kBAAkB,CAAC,CAAC;GACtC,KAAK,KAAK,uCAAuC,WAAW,EAAE,MAAM,aAAa,CAAC;GAClF,KAAK,KAAK,2CAA2C,OAAO,WAAW,SAAS;GAKhF,MAAM,MAAM,QAAQ,SAAS,OAAO;GACpC,MAAM,MAAM,YAAY,IAAI;GAC5B,KAAK,KAAK,qCAAqC,IAAI,OAAO,IAAI,UAAU,OAAQ,MAAM,OAAO;GAC7F,UAAU,QAAQ,SAAS;GAC3B,SAAS;GACT,KAAK,KAAK,0CAA0C,QAAQ;GAE5D,KAAK,KAAK,oCAAoC,QAAQ,OAAO,IAAI,GAAI;EACvE;EACA,OAAO;GACL,KAAK,QAAQ;GACb,IAAI,WAAW;EACjB;CACF;AACF"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { MetricsCollector, MetricsLogger } from '../metrics.js';
|
|
2
|
+
/** The commands the collector uses, structurally matching a node-redis client. */
|
|
3
|
+
export interface RedisMonitorClient {
|
|
4
|
+
connect(): Promise<unknown>;
|
|
5
|
+
info(): Promise<string>;
|
|
6
|
+
xInfoGroups(key: string): Promise<Array<{
|
|
7
|
+
name: unknown;
|
|
8
|
+
pending: unknown;
|
|
9
|
+
lag?: unknown;
|
|
10
|
+
}>>;
|
|
11
|
+
on(event: 'error', listener: (error: unknown) => void): unknown;
|
|
12
|
+
destroy?(): void;
|
|
13
|
+
}
|
|
14
|
+
/** What `@mastra/redis-streams`' `RedisStreamsPubSub` offers for monitoring. */
|
|
15
|
+
export interface MonitorablePubSub {
|
|
16
|
+
createMonitorClient(): RedisMonitorClient;
|
|
17
|
+
streamKey(topic: string): string;
|
|
18
|
+
}
|
|
19
|
+
export declare function asMonitorablePubSub(pubsub: unknown): MonitorablePubSub | undefined;
|
|
20
|
+
/** Owns a dedicated monitoring connection, so probes never queue behind pub/sub traffic. */
|
|
21
|
+
export declare function connectMonitor(pubsub: MonitorablePubSub, getLogger: () => MetricsLogger | undefined): RedisMonitorClient;
|
|
22
|
+
/** The `INFO` round trip as the Redis probe, run in every process. */
|
|
23
|
+
export declare function redisProbeCollector(client: RedisMonitorClient, timeoutMs: number): MetricsCollector;
|
|
24
|
+
/**
|
|
25
|
+
* Server-wide Redis stats, collected by one process per deployment: memory against `maxmemory`,
|
|
26
|
+
* evictions, clients, and the backlog of the grouped event streams.
|
|
27
|
+
*/
|
|
28
|
+
export declare function redisStatsCollector(pubsub: MonitorablePubSub, client: RedisMonitorClient, timeoutMs: number): MetricsCollector;
|
|
29
|
+
//# sourceMappingURL=redis.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.d.ts","sourceRoot":"","sources":["../../../src/metrics/collectors/redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGrE,kFAAkF;AAClF,MAAM,WAAW,kBAAkB;IACjC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5B,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IACxB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC;QAAC,GAAG,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC,CAAC;IAC7F,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC;IAChE,OAAO,CAAC,IAAI,IAAI,CAAC;CAClB;AAED,gFAAgF;AAChF,MAAM,WAAW,iBAAiB;IAChC,mBAAmB,IAAI,kBAAkB,CAAC;IAC1C,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;CAClC;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,OAAO,GAAG,iBAAiB,GAAG,SAAS,CAMlF;AAiBD,4FAA4F;AAC5F,wBAAgB,cAAc,CAAC,MAAM,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,GAAG,SAAS,sBAcnG;AAED,sEAAsE;AACtE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,GAAG,gBAAgB,CAOnG;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,iBAAiB,EACzB,MAAM,EAAE,kBAAkB,EAC1B,SAAS,EAAE,MAAM,GAChB,gBAAgB,CA8DlB"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { createProbe } from "./probe.js";
|
|
2
|
+
// src/metrics/collectors/redis.ts
|
|
3
|
+
function asMonitorablePubSub(pubsub) {
|
|
4
|
+
if (!pubsub || typeof pubsub !== "object") return void 0;
|
|
5
|
+
const candidate = pubsub;
|
|
6
|
+
return typeof candidate.createMonitorClient === "function" && typeof candidate.streamKey === "function" ? candidate : void 0;
|
|
7
|
+
}
|
|
8
|
+
const GROUPED_TOPICS = ["workflows", "background-tasks"];
|
|
9
|
+
const FANOUT_GROUP_PREFIX = "__fanout-";
|
|
10
|
+
function parseInfo(text) {
|
|
11
|
+
const fields = /* @__PURE__ */ new Map();
|
|
12
|
+
for (const line of text.split("\r\n")) {
|
|
13
|
+
const colon = line.indexOf(":");
|
|
14
|
+
if (colon > 0 && !line.startsWith("#")) fields.set(line.slice(0, colon), line.slice(colon + 1));
|
|
15
|
+
}
|
|
16
|
+
return fields;
|
|
17
|
+
}
|
|
18
|
+
/** Owns a dedicated monitoring connection, so probes never queue behind pub/sub traffic. */
|
|
19
|
+
function connectMonitor(pubsub, getLogger) {
|
|
20
|
+
const client = pubsub.createMonitorClient();
|
|
21
|
+
let lastErrorLogAt = 0;
|
|
22
|
+
client.on("error", (error) => {
|
|
23
|
+
if (Date.now() - lastErrorLogAt < 6e4) return;
|
|
24
|
+
lastErrorLogAt = Date.now();
|
|
25
|
+
getLogger()?.warn("[factory:metrics] Redis monitor client error.", { error: error instanceof Error ? error.message : String(error) });
|
|
26
|
+
});
|
|
27
|
+
client.connect().catch(() => {});
|
|
28
|
+
return client;
|
|
29
|
+
}
|
|
30
|
+
/** The `INFO` round trip as the Redis probe, run in every process. */
|
|
31
|
+
function redisProbeCollector(client, timeoutMs) {
|
|
32
|
+
const probe = createProbe();
|
|
33
|
+
return { async collect(sink) {
|
|
34
|
+
await probe(sink, "redis", timeoutMs, () => client.info());
|
|
35
|
+
} };
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Server-wide Redis stats, collected by one process per deployment: memory against `maxmemory`,
|
|
39
|
+
* evictions, clients, and the backlog of the grouped event streams.
|
|
40
|
+
*/
|
|
41
|
+
function redisStatsCollector(pubsub, client, timeoutMs) {
|
|
42
|
+
const probe = createProbe();
|
|
43
|
+
let lastEvicted;
|
|
44
|
+
return { async collect(sink) {
|
|
45
|
+
let info = "";
|
|
46
|
+
if (!(await probe(sink, "redis_stats", timeoutMs, async () => {
|
|
47
|
+
info = await client.info();
|
|
48
|
+
})).ok) return;
|
|
49
|
+
const fields = parseInfo(info);
|
|
50
|
+
const number = (key) => Number(fields.get(key));
|
|
51
|
+
sink.emit("mastra_factory_redis_memory_bytes", number("used_memory"), { kind: "used" });
|
|
52
|
+
const maxMemory = number("maxmemory");
|
|
53
|
+
if (maxMemory > 0) {
|
|
54
|
+
sink.emit("mastra_factory_redis_memory_bytes", maxMemory, { kind: "max" });
|
|
55
|
+
sink.emit("mastra_factory_redis_memory_utilization", number("used_memory") / maxMemory);
|
|
56
|
+
}
|
|
57
|
+
sink.emit("mastra_factory_redis_clients", number("connected_clients"));
|
|
58
|
+
const evicted = number("evicted_keys");
|
|
59
|
+
if (Number.isFinite(evicted)) {
|
|
60
|
+
if (lastEvicted !== void 0) sink.emit("mastra_factory_redis_evicted_keys_total", evicted >= lastEvicted ? evicted - lastEvicted : evicted);
|
|
61
|
+
lastEvicted = evicted;
|
|
62
|
+
}
|
|
63
|
+
const backlogs = [];
|
|
64
|
+
await probe(sink, "redis_streams", timeoutMs, () => Promise.all(GROUPED_TOPICS.map(async (topic) => {
|
|
65
|
+
const groups = await client.xInfoGroups(pubsub.streamKey(topic)).catch((error) => {
|
|
66
|
+
if (/no such key/i.test(error instanceof Error ? error.message : String(error))) return void 0;
|
|
67
|
+
throw error;
|
|
68
|
+
});
|
|
69
|
+
if (!groups) return;
|
|
70
|
+
let pending = 0;
|
|
71
|
+
let lag = 0;
|
|
72
|
+
for (const group of groups) {
|
|
73
|
+
if (String(group.name).startsWith(FANOUT_GROUP_PREFIX)) continue;
|
|
74
|
+
pending += Number(group.pending);
|
|
75
|
+
lag = group.lag == null || lag === void 0 ? void 0 : lag + Number(group.lag);
|
|
76
|
+
}
|
|
77
|
+
backlogs.push({
|
|
78
|
+
topic,
|
|
79
|
+
pending,
|
|
80
|
+
lag
|
|
81
|
+
});
|
|
82
|
+
})));
|
|
83
|
+
for (const { topic, pending, lag } of backlogs) {
|
|
84
|
+
sink.emit("mastra_factory_redis_stream_backlog", pending, {
|
|
85
|
+
topic,
|
|
86
|
+
kind: "pending"
|
|
87
|
+
});
|
|
88
|
+
if (lag !== void 0) sink.emit("mastra_factory_redis_stream_backlog", lag, {
|
|
89
|
+
topic,
|
|
90
|
+
kind: "lag"
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
} };
|
|
94
|
+
}
|
|
95
|
+
export { asMonitorablePubSub, connectMonitor, redisProbeCollector, redisStatsCollector };
|
|
96
|
+
|
|
97
|
+
//# sourceMappingURL=redis.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.js","names":[],"sources":["../../../src/metrics/collectors/redis.ts"],"sourcesContent":["import type { MetricsCollector, MetricsLogger } from '../metrics.js';\nimport { createProbe } from './probe.js';\n\n/** The commands the collector uses, structurally matching a node-redis client. */\nexport interface RedisMonitorClient {\n connect(): Promise<unknown>;\n info(): Promise<string>;\n xInfoGroups(key: string): Promise<Array<{ name: unknown; pending: unknown; lag?: unknown }>>;\n on(event: 'error', listener: (error: unknown) => void): unknown;\n destroy?(): void;\n}\n\n/** What `@mastra/redis-streams`' `RedisStreamsPubSub` offers for monitoring. */\nexport interface MonitorablePubSub {\n createMonitorClient(): RedisMonitorClient;\n streamKey(topic: string): string;\n}\n\nexport function asMonitorablePubSub(pubsub: unknown): MonitorablePubSub | undefined {\n if (!pubsub || typeof pubsub !== 'object') return undefined;\n const candidate = pubsub as Partial<MonitorablePubSub>;\n return typeof candidate.createMonitorClient === 'function' && typeof candidate.streamKey === 'function'\n ? (candidate as MonitorablePubSub)\n : undefined;\n}\n\n// Topics read through named consumer groups (workflow orchestration and background tasks). Their\n// backlog is work that was published but not processed yet. Fan-out subscriptions use throwaway\n// groups and are skipped.\nconst GROUPED_TOPICS = ['workflows', 'background-tasks'];\nconst FANOUT_GROUP_PREFIX = '__fanout-';\n\nfunction parseInfo(text: string): Map<string, string> {\n const fields = new Map<string, string>();\n for (const line of text.split('\\r\\n')) {\n const colon = line.indexOf(':');\n if (colon > 0 && !line.startsWith('#')) fields.set(line.slice(0, colon), line.slice(colon + 1));\n }\n return fields;\n}\n\n/** Owns a dedicated monitoring connection, so probes never queue behind pub/sub traffic. */\nexport function connectMonitor(pubsub: MonitorablePubSub, getLogger: () => MetricsLogger | undefined) {\n const client = pubsub.createMonitorClient();\n let lastErrorLogAt = 0;\n client.on('error', error => {\n if (Date.now() - lastErrorLogAt < 60_000) return;\n lastErrorLogAt = Date.now();\n getLogger()?.warn('[factory:metrics] Redis monitor client error.', {\n error: error instanceof Error ? error.message : String(error),\n });\n });\n void client.connect().catch(() => {\n // Reported through the 'error' listener; node-redis keeps reconnecting.\n });\n return client;\n}\n\n/** The `INFO` round trip as the Redis probe, run in every process. */\nexport function redisProbeCollector(client: RedisMonitorClient, timeoutMs: number): MetricsCollector {\n const probe = createProbe();\n return {\n async collect(sink) {\n await probe(sink, 'redis', timeoutMs, () => client.info());\n },\n };\n}\n\n/**\n * Server-wide Redis stats, collected by one process per deployment: memory against `maxmemory`,\n * evictions, clients, and the backlog of the grouped event streams.\n */\nexport function redisStatsCollector(\n pubsub: MonitorablePubSub,\n client: RedisMonitorClient,\n timeoutMs: number,\n): MetricsCollector {\n const probe = createProbe();\n let lastEvicted: number | undefined;\n return {\n async collect(sink) {\n let info = '';\n const result = await probe(sink, 'redis_stats', timeoutMs, async () => {\n info = await client.info();\n });\n if (!result.ok) return;\n\n const fields = parseInfo(info);\n const number = (key: string) => Number(fields.get(key));\n sink.emit('mastra_factory_redis_memory_bytes', number('used_memory'), { kind: 'used' });\n const maxMemory = number('maxmemory');\n // `maxmemory` is 0 when Redis has no memory limit; then neither the limit nor the ratio is sent.\n if (maxMemory > 0) {\n sink.emit('mastra_factory_redis_memory_bytes', maxMemory, { kind: 'max' });\n sink.emit('mastra_factory_redis_memory_utilization', number('used_memory') / maxMemory);\n }\n sink.emit('mastra_factory_redis_clients', number('connected_clients'));\n // INFO reports evictions since the Redis server started; send the increase per interval.\n const evicted = number('evicted_keys');\n if (Number.isFinite(evicted)) {\n if (lastEvicted !== undefined) {\n sink.emit(\n 'mastra_factory_redis_evicted_keys_total',\n evicted >= lastEvicted ? evicted - lastEvicted : evicted,\n );\n }\n lastEvicted = evicted;\n }\n\n // Bounded like every other call: a Redis that answers INFO and then stalls must not hold up the tick.\n const backlogs: Array<{ topic: string; pending: number; lag: number | undefined }> = [];\n await probe(sink, 'redis_streams', timeoutMs, () =>\n Promise.all(\n GROUPED_TOPICS.map(async topic => {\n // A missing stream (nothing published yet) is not a failure; anything else is.\n const groups = await client.xInfoGroups(pubsub.streamKey(topic)).catch((error: unknown) => {\n if (/no such key/i.test(error instanceof Error ? error.message : String(error))) return undefined;\n throw error;\n });\n if (!groups) return;\n let pending = 0;\n // Redis reports lag as null when it can't compute it; then the topic gets no lag record.\n let lag: number | undefined = 0;\n for (const group of groups) {\n if (String(group.name).startsWith(FANOUT_GROUP_PREFIX)) continue;\n pending += Number(group.pending);\n lag = group.lag == null || lag === undefined ? undefined : lag + Number(group.lag);\n }\n backlogs.push({ topic, pending, lag });\n }),\n ),\n );\n for (const { topic, pending, lag } of backlogs) {\n sink.emit('mastra_factory_redis_stream_backlog', pending, { topic, kind: 'pending' });\n if (lag !== undefined) sink.emit('mastra_factory_redis_stream_backlog', lag, { topic, kind: 'lag' });\n }\n },\n };\n}\n"],"mappings":";;AAkBA,SAAgB,oBAAoB,QAAgD;CAClF,IAAI,CAAC,UAAU,OAAO,WAAW,UAAU,OAAO,KAAA;CAClD,MAAM,YAAY;CAClB,OAAO,OAAO,UAAU,wBAAwB,cAAc,OAAO,UAAU,cAAc,aACxF,YACD,KAAA;AACN;AAKA,MAAM,iBAAiB,CAAC,aAAa,kBAAkB;AACvD,MAAM,sBAAsB;AAE5B,SAAS,UAAU,MAAmC;CACpD,MAAM,yBAAS,IAAI,IAAoB;CACvC,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM,GAAG;EACrC,MAAM,QAAQ,KAAK,QAAQ,GAAG;EAC9B,IAAI,QAAQ,KAAK,CAAC,KAAK,WAAW,GAAG,GAAG,OAAO,IAAI,KAAK,MAAM,GAAG,KAAK,GAAG,KAAK,MAAM,QAAQ,CAAC,CAAC;CAChG;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,QAA2B,WAA4C;CACpG,MAAM,SAAS,OAAO,oBAAoB;CAC1C,IAAI,iBAAiB;CACrB,OAAO,GAAG,UAAS,UAAS;EAC1B,IAAI,KAAK,IAAI,IAAI,iBAAiB,KAAQ;EAC1C,iBAAiB,KAAK,IAAI;EAC1B,UAAU,CAAC,EAAE,KAAK,iDAAiD,EACjE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAC9D,CAAC;CACH,CAAC;CACD,OAAY,QAAQ,CAAC,CAAC,YAAY,CAElC,CAAC;CACD,OAAO;AACT;;AAGA,SAAgB,oBAAoB,QAA4B,WAAqC;CACnG,MAAM,QAAQ,YAAY;CAC1B,OAAO,EACL,MAAM,QAAQ,MAAM;EAClB,MAAM,MAAM,MAAM,SAAS,iBAAiB,OAAO,KAAK,CAAC;CAC3D,EACF;AACF;;;;;AAMA,SAAgB,oBACd,QACA,QACA,WACkB;CAClB,MAAM,QAAQ,YAAY;CAC1B,IAAI;CACJ,OAAO,EACL,MAAM,QAAQ,MAAM;EAClB,IAAI,OAAO;EAIX,IAAI,EAAC,MAHgB,MAAM,MAAM,eAAe,WAAW,YAAY;GACrE,OAAO,MAAM,OAAO,KAAK;EAC3B,CAAC,EAAA,CACW,IAAI;EAEhB,MAAM,SAAS,UAAU,IAAI;EAC7B,MAAM,UAAU,QAAgB,OAAO,OAAO,IAAI,GAAG,CAAC;EACtD,KAAK,KAAK,qCAAqC,OAAO,aAAa,GAAG,EAAE,MAAM,OAAO,CAAC;EACtF,MAAM,YAAY,OAAO,WAAW;EAEpC,IAAI,YAAY,GAAG;GACjB,KAAK,KAAK,qCAAqC,WAAW,EAAE,MAAM,MAAM,CAAC;GACzE,KAAK,KAAK,2CAA2C,OAAO,aAAa,IAAI,SAAS;EACxF;EACA,KAAK,KAAK,gCAAgC,OAAO,mBAAmB,CAAC;EAErE,MAAM,UAAU,OAAO,cAAc;EACrC,IAAI,OAAO,SAAS,OAAO,GAAG;GAC5B,IAAI,gBAAgB,KAAA,GAClB,KAAK,KACH,2CACA,WAAW,cAAc,UAAU,cAAc,OACnD;GAEF,cAAc;EAChB;EAGA,MAAM,WAA+E,CAAC;EACtF,MAAM,MAAM,MAAM,iBAAiB,iBACjC,QAAQ,IACN,eAAe,IAAI,OAAM,UAAS;GAEhC,MAAM,SAAS,MAAM,OAAO,YAAY,OAAO,UAAU,KAAK,CAAC,CAAC,CAAC,OAAO,UAAmB;IACzF,IAAI,eAAe,KAAK,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC,GAAG,OAAO,KAAA;IACxF,MAAM;GACR,CAAC;GACD,IAAI,CAAC,QAAQ;GACb,IAAI,UAAU;GAEd,IAAI,MAA0B;GAC9B,KAAK,MAAM,SAAS,QAAQ;IAC1B,IAAI,OAAO,MAAM,IAAI,CAAC,CAAC,WAAW,mBAAmB,GAAG;IACxD,WAAW,OAAO,MAAM,OAAO;IAC/B,MAAM,MAAM,OAAO,QAAQ,QAAQ,KAAA,IAAY,KAAA,IAAY,MAAM,OAAO,MAAM,GAAG;GACnF;GACA,SAAS,KAAK;IAAE;IAAO;IAAS;GAAI,CAAC;EACvC,CAAC,CACH,CACF;EACA,KAAK,MAAM,EAAE,OAAO,SAAS,SAAS,UAAU;GAC9C,KAAK,KAAK,uCAAuC,SAAS;IAAE;IAAO,MAAM;GAAU,CAAC;GACpF,IAAI,QAAQ,KAAA,GAAW,KAAK,KAAK,uCAAuC,KAAK;IAAE;IAAO,MAAM;GAAM,CAAC;EACrG;CACF,EACF;AACF"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Metric groups a deployment can turn on, one per component to investigate. */
|
|
2
|
+
export declare const METRIC_GROUPS: readonly ['process', 'postgres', 'redis', 'sandbox', 'intake'];
|
|
3
|
+
export type MetricGroup = (typeof METRIC_GROUPS)[number];
|
|
4
|
+
export interface FactoryMetricsOptions {
|
|
5
|
+
/** Groups to collect, or `'all'`. An empty list turns metrics off. */
|
|
6
|
+
groups: readonly MetricGroup[] | 'all';
|
|
7
|
+
/** How often gauges are sampled and aggregated timings are sent. Default 60s, minimum 15s. */
|
|
8
|
+
intervalMs?: number;
|
|
9
|
+
/** Most records sent per minute per process; the rest are dropped and counted. Default 500. */
|
|
10
|
+
maxRecordsPerMinute?: number;
|
|
11
|
+
}
|
|
12
|
+
export interface ResolvedMetricsConfig {
|
|
13
|
+
groups: ReadonlySet<MetricGroup>;
|
|
14
|
+
intervalMs: number;
|
|
15
|
+
maxPerMinute: number;
|
|
16
|
+
/** Group names that were asked for but don't exist. */
|
|
17
|
+
unknownGroups: string[];
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Resolves which metric groups are on. Every record counts towards the deployment's observability
|
|
21
|
+
* usage, so nothing is collected unless a group is asked for. Env vars override the code option:
|
|
22
|
+
* `FACTORY_METRICS` (`all`, `0`, or a list like `sandbox,intake`), `FACTORY_METRICS_INTERVAL_MS`
|
|
23
|
+
* and `FACTORY_METRICS_MAX_PER_MIN`. Returns `undefined` when no group is on.
|
|
24
|
+
*/
|
|
25
|
+
export declare function resolveMetricsConfig(option: FactoryMetricsOptions | undefined, env?: NodeJS.ProcessEnv): ResolvedMetricsConfig | undefined;
|
|
26
|
+
export type MetricsService = 'api' | 'worker' | 'all';
|
|
27
|
+
/**
|
|
28
|
+
* Which part of a split deployment this process is. Platform deploys a separate worker service
|
|
29
|
+
* when the app has storage and pubsub: the API container runs with `MASTRA_WORKERS=false`, the
|
|
30
|
+
* worker container with it set (empty, or a list of worker names). Unset means one process does both.
|
|
31
|
+
*/
|
|
32
|
+
export declare function metricsService(env?: NodeJS.ProcessEnv): MetricsService;
|
|
33
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/metrics/config.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,eAAO,MAAM,aAAa,YAAI,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAU,CAAC;AAC5F,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzD,MAAM,WAAW,qBAAqB;IACpC,sEAAsE;IACtE,MAAM,EAAE,SAAS,WAAW,EAAE,GAAG,KAAK,CAAC;IACvC,8FAA8F;IAC9F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+FAA+F;IAC/F,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAmBD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,qBAAqB,GAAG,SAAS,EACzC,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,qBAAqB,GAAG,SAAS,CA2BnC;AAED,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,QAAQ,GAAG,KAAK,CAAC;AAEtD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,cAAc,CAInF"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// src/metrics/config.ts
|
|
2
|
+
/** Metric groups a deployment can turn on, one per component to investigate. */
|
|
3
|
+
const METRIC_GROUPS = [
|
|
4
|
+
"process",
|
|
5
|
+
"postgres",
|
|
6
|
+
"redis",
|
|
7
|
+
"sandbox",
|
|
8
|
+
"intake"
|
|
9
|
+
];
|
|
10
|
+
const DEFAULT_INTERVAL_MS = 6e4;
|
|
11
|
+
const MIN_INTERVAL_MS = 15e3;
|
|
12
|
+
const MAX_INTERVAL_MS = 2147483647;
|
|
13
|
+
const DEFAULT_MAX_PER_MINUTE = 500;
|
|
14
|
+
function parseGroups(value) {
|
|
15
|
+
if (value === "all") return {
|
|
16
|
+
groups: new Set(METRIC_GROUPS),
|
|
17
|
+
unknownGroups: []
|
|
18
|
+
};
|
|
19
|
+
const groups = /* @__PURE__ */ new Set();
|
|
20
|
+
const unknownGroups = [];
|
|
21
|
+
for (const name of value) if (METRIC_GROUPS.includes(name)) groups.add(name);
|
|
22
|
+
else unknownGroups.push(name);
|
|
23
|
+
return {
|
|
24
|
+
groups,
|
|
25
|
+
unknownGroups
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Resolves which metric groups are on. Every record counts towards the deployment's observability
|
|
30
|
+
* usage, so nothing is collected unless a group is asked for. Env vars override the code option:
|
|
31
|
+
* `FACTORY_METRICS` (`all`, `0`, or a list like `sandbox,intake`), `FACTORY_METRICS_INTERVAL_MS`
|
|
32
|
+
* and `FACTORY_METRICS_MAX_PER_MIN`. Returns `undefined` when no group is on.
|
|
33
|
+
*/
|
|
34
|
+
function resolveMetricsConfig(option, env = process.env) {
|
|
35
|
+
const envGroups = env.FACTORY_METRICS?.trim();
|
|
36
|
+
let requested = option?.groups;
|
|
37
|
+
if (envGroups) {
|
|
38
|
+
if (/^(0|false|no|off)$/i.test(envGroups)) return void 0;
|
|
39
|
+
requested = /^(all|1|true|yes)$/i.test(envGroups) ? "all" : envGroups.split(",").map((name) => name.trim()).filter(Boolean);
|
|
40
|
+
}
|
|
41
|
+
if (!requested) return void 0;
|
|
42
|
+
const { groups, unknownGroups } = parseGroups(requested);
|
|
43
|
+
if (groups.size === 0 && unknownGroups.length === 0) return void 0;
|
|
44
|
+
const interval = Number(env.FACTORY_METRICS_INTERVAL_MS || option?.intervalMs);
|
|
45
|
+
const maxPerMinute = Number(env.FACTORY_METRICS_MAX_PER_MIN || option?.maxRecordsPerMinute);
|
|
46
|
+
return {
|
|
47
|
+
groups,
|
|
48
|
+
intervalMs: Number.isFinite(interval) && interval > 0 ? Math.min(Math.max(interval, MIN_INTERVAL_MS), MAX_INTERVAL_MS) : DEFAULT_INTERVAL_MS,
|
|
49
|
+
maxPerMinute: Number.isInteger(maxPerMinute) && maxPerMinute > 0 ? maxPerMinute : DEFAULT_MAX_PER_MINUTE,
|
|
50
|
+
unknownGroups
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Which part of a split deployment this process is. Platform deploys a separate worker service
|
|
55
|
+
* when the app has storage and pubsub: the API container runs with `MASTRA_WORKERS=false`, the
|
|
56
|
+
* worker container with it set (empty, or a list of worker names). Unset means one process does both.
|
|
57
|
+
*/
|
|
58
|
+
function metricsService(env = process.env) {
|
|
59
|
+
const workers = env.MASTRA_WORKERS;
|
|
60
|
+
if (workers === void 0) return "all";
|
|
61
|
+
return workers.trim() === "false" ? "api" : "worker";
|
|
62
|
+
}
|
|
63
|
+
export { METRIC_GROUPS, metricsService, resolveMetricsConfig };
|
|
64
|
+
|
|
65
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","names":[],"sources":["../../src/metrics/config.ts"],"sourcesContent":["/** Metric groups a deployment can turn on, one per component to investigate. */\nexport const METRIC_GROUPS = ['process', 'postgres', 'redis', 'sandbox', 'intake'] as const;\nexport type MetricGroup = (typeof METRIC_GROUPS)[number];\n\nexport interface FactoryMetricsOptions {\n /** Groups to collect, or `'all'`. An empty list turns metrics off. */\n groups: readonly MetricGroup[] | 'all';\n /** How often gauges are sampled and aggregated timings are sent. Default 60s, minimum 15s. */\n intervalMs?: number;\n /** Most records sent per minute per process; the rest are dropped and counted. Default 500. */\n maxRecordsPerMinute?: number;\n}\n\nexport interface ResolvedMetricsConfig {\n groups: ReadonlySet<MetricGroup>;\n intervalMs: number;\n maxPerMinute: number;\n /** Group names that were asked for but don't exist. */\n unknownGroups: string[];\n}\n\nconst DEFAULT_INTERVAL_MS = 60_000;\nconst MIN_INTERVAL_MS = 15_000;\n// Longest delay setInterval accepts; Node turns anything above it into 1 ms.\nconst MAX_INTERVAL_MS = 2_147_483_647;\nconst DEFAULT_MAX_PER_MINUTE = 500;\n\nfunction parseGroups(value: readonly string[] | 'all'): { groups: Set<MetricGroup>; unknownGroups: string[] } {\n if (value === 'all') return { groups: new Set(METRIC_GROUPS), unknownGroups: [] };\n const groups = new Set<MetricGroup>();\n const unknownGroups: string[] = [];\n for (const name of value) {\n if ((METRIC_GROUPS as readonly string[]).includes(name)) groups.add(name as MetricGroup);\n else unknownGroups.push(name);\n }\n return { groups, unknownGroups };\n}\n\n/**\n * Resolves which metric groups are on. Every record counts towards the deployment's observability\n * usage, so nothing is collected unless a group is asked for. Env vars override the code option:\n * `FACTORY_METRICS` (`all`, `0`, or a list like `sandbox,intake`), `FACTORY_METRICS_INTERVAL_MS`\n * and `FACTORY_METRICS_MAX_PER_MIN`. Returns `undefined` when no group is on.\n */\nexport function resolveMetricsConfig(\n option: FactoryMetricsOptions | undefined,\n env: NodeJS.ProcessEnv = process.env,\n): ResolvedMetricsConfig | undefined {\n const envGroups = env.FACTORY_METRICS?.trim();\n let requested: readonly string[] | 'all' | undefined = option?.groups;\n if (envGroups) {\n if (/^(0|false|no|off)$/i.test(envGroups)) return undefined;\n requested = /^(all|1|true|yes)$/i.test(envGroups)\n ? 'all'\n : envGroups\n .split(',')\n .map(name => name.trim())\n .filter(Boolean);\n }\n if (!requested) return undefined;\n const { groups, unknownGroups } = parseGroups(requested);\n if (groups.size === 0 && unknownGroups.length === 0) return undefined;\n\n const interval = Number(env.FACTORY_METRICS_INTERVAL_MS || option?.intervalMs);\n const maxPerMinute = Number(env.FACTORY_METRICS_MAX_PER_MIN || option?.maxRecordsPerMinute);\n return {\n groups,\n intervalMs:\n Number.isFinite(interval) && interval > 0\n ? Math.min(Math.max(interval, MIN_INTERVAL_MS), MAX_INTERVAL_MS)\n : DEFAULT_INTERVAL_MS,\n maxPerMinute: Number.isInteger(maxPerMinute) && maxPerMinute > 0 ? maxPerMinute : DEFAULT_MAX_PER_MINUTE,\n unknownGroups,\n };\n}\n\nexport type MetricsService = 'api' | 'worker' | 'all';\n\n/**\n * Which part of a split deployment this process is. Platform deploys a separate worker service\n * when the app has storage and pubsub: the API container runs with `MASTRA_WORKERS=false`, the\n * worker container with it set (empty, or a list of worker names). Unset means one process does both.\n */\nexport function metricsService(env: NodeJS.ProcessEnv = process.env): MetricsService {\n const workers = env.MASTRA_WORKERS;\n if (workers === undefined) return 'all';\n return workers.trim() === 'false' ? 'api' : 'worker';\n}\n"],"mappings":";;AACA,MAAa,gBAAgB;CAAC;CAAW;CAAY;CAAS;CAAW;AAAQ;AAoBjF,MAAM,sBAAsB;AAC5B,MAAM,kBAAkB;AAExB,MAAM,kBAAkB;AACxB,MAAM,yBAAyB;AAE/B,SAAS,YAAY,OAAyF;CAC5G,IAAI,UAAU,OAAO,OAAO;EAAE,QAAQ,IAAI,IAAI,aAAa;EAAG,eAAe,CAAC;CAAE;CAChF,MAAM,yBAAS,IAAI,IAAiB;CACpC,MAAM,gBAA0B,CAAC;CACjC,KAAK,MAAM,QAAQ,OACjB,IAAK,cAAoC,SAAS,IAAI,GAAG,OAAO,IAAI,IAAmB;MAClF,cAAc,KAAK,IAAI;CAE9B,OAAO;EAAE;EAAQ;CAAc;AACjC;;;;;;;AAQA,SAAgB,qBACd,QACA,MAAyB,QAAQ,KACE;CACnC,MAAM,YAAY,IAAI,iBAAiB,KAAK;CAC5C,IAAI,YAAmD,QAAQ;CAC/D,IAAI,WAAW;EACb,IAAI,sBAAsB,KAAK,SAAS,GAAG,OAAO,KAAA;EAClD,YAAY,sBAAsB,KAAK,SAAS,IAC5C,QACA,UACG,MAAM,GAAG,CAAC,CACV,KAAI,SAAQ,KAAK,KAAK,CAAC,CAAC,CACxB,OAAO,OAAO;CACvB;CACA,IAAI,CAAC,WAAW,OAAO,KAAA;CACvB,MAAM,EAAE,QAAQ,kBAAkB,YAAY,SAAS;CACvD,IAAI,OAAO,SAAS,KAAK,cAAc,WAAW,GAAG,OAAO,KAAA;CAE5D,MAAM,WAAW,OAAO,IAAI,+BAA+B,QAAQ,UAAU;CAC7E,MAAM,eAAe,OAAO,IAAI,+BAA+B,QAAQ,mBAAmB;CAC1F,OAAO;EACL;EACA,YACE,OAAO,SAAS,QAAQ,KAAK,WAAW,IACpC,KAAK,IAAI,KAAK,IAAI,UAAU,eAAe,GAAG,eAAe,IAC7D;EACN,cAAc,OAAO,UAAU,YAAY,KAAK,eAAe,IAAI,eAAe;EAClF;CACF;AACF;;;;;;AASA,SAAgB,eAAe,MAAyB,QAAQ,KAAqB;CACnF,MAAM,UAAU,IAAI;CACpB,IAAI,YAAY,KAAA,GAAW,OAAO;CAClC,OAAO,QAAQ,KAAK,MAAM,UAAU,QAAQ;AAC9C"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { FactoryMetrics, MetricsTargets } from './metrics.js';
|
|
2
|
+
export { METRIC_GROUPS, metricsService, resolveMetricsConfig } from './config.js';
|
|
3
|
+
export type { FactoryMetricsOptions, MetricGroup } from './config.js';
|
|
4
|
+
export { FactoryMetrics, factoryMetrics, installFactoryMetrics } from './metrics.js';
|
|
5
|
+
export interface MetricsSources {
|
|
6
|
+
/** The Mastra storage, read for its `pg` pool when it's Postgres. */
|
|
7
|
+
storage: unknown;
|
|
8
|
+
vector: unknown;
|
|
9
|
+
pubsub: unknown;
|
|
10
|
+
/** Ids of the configured integrations; GitHub and Linear intake counts are sent as 0 when quiet. */
|
|
11
|
+
integrationIds?: readonly string[];
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Builds the collectors for the enabled groups and starts the interval timer. Server-wide stats
|
|
15
|
+
* (Postgres activity, Redis memory and stream backlog) only run on the worker service (or a
|
|
16
|
+
* single-process deployment), so a split deployment doesn't report them twice. Returns a stop
|
|
17
|
+
* function that also closes the Redis monitor connection.
|
|
18
|
+
*/
|
|
19
|
+
export declare function startFactoryMetrics(metrics: FactoryMetrics, targets: MetricsTargets, sources: MetricsSources): () => Promise<void>;
|
|
20
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,cAAc,EAAoB,cAAc,EAAE,MAAM,cAAc,CAAC;AAGrF,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAClF,YAAY,EAAE,qBAAqB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAIrF,MAAM,WAAW,cAAc;IAC7B,qEAAqE;IACrE,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;IAChB,oGAAoG;IACpG,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,cAAc,EACvB,OAAO,EAAE,cAAc,EACvB,OAAO,EAAE,cAAc,GACtB,MAAM,OAAO,CAAC,IAAI,CAAC,CAyDrB"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { countRunningSessionSandboxes } from "../sandbox/session-sandbox.js";
|
|
2
|
+
import { asPool, postgresPoolCollector, postgresStatsCollector } from "./collectors/postgres.js";
|
|
3
|
+
import { processCollector } from "./collectors/process.js";
|
|
4
|
+
import { asMonitorablePubSub, connectMonitor, redisProbeCollector, redisStatsCollector } from "./collectors/redis.js";
|
|
5
|
+
import { intakeCollector, keepIntakeSource } from "./intake.js";
|
|
6
|
+
import { sessionStartsCollector } from "./sandbox.js";
|
|
7
|
+
import { METRIC_GROUPS, metricsService, resolveMetricsConfig } from "./config.js";
|
|
8
|
+
import { FactoryMetrics, factoryMetrics, installFactoryMetrics } from "./metrics.js";
|
|
9
|
+
// src/metrics/index.ts
|
|
10
|
+
const SLOW_STATEMENTS_EVERY_MS = 6e5;
|
|
11
|
+
/**
|
|
12
|
+
* Builds the collectors for the enabled groups and starts the interval timer. Server-wide stats
|
|
13
|
+
* (Postgres activity, Redis memory and stream backlog) only run on the worker service (or a
|
|
14
|
+
* single-process deployment), so a split deployment doesn't report them twice. Returns a stop
|
|
15
|
+
* function that also closes the Redis monitor connection.
|
|
16
|
+
*/
|
|
17
|
+
function startFactoryMetrics(metrics, targets, sources) {
|
|
18
|
+
const intervalMs = metrics.intervalMs;
|
|
19
|
+
if (intervalMs === 0) return async () => {};
|
|
20
|
+
const timeoutMs = Math.min(5e3, intervalMs / 2);
|
|
21
|
+
const serverWide = metrics.service !== "api";
|
|
22
|
+
const collectors = [];
|
|
23
|
+
let redisClient;
|
|
24
|
+
if (metrics.enabled("process")) collectors.push(processCollector((kind, ms) => metrics.observe("process", "mastra_factory_gc_ms", ms, { kind })));
|
|
25
|
+
if (metrics.enabled("postgres")) {
|
|
26
|
+
const pools = {};
|
|
27
|
+
const store = asPool(sources.storage?.pool);
|
|
28
|
+
const vector = asPool(sources.vector?.pool);
|
|
29
|
+
if (store) pools.store = store;
|
|
30
|
+
if (vector) pools.vector = vector;
|
|
31
|
+
if (Object.keys(pools).length > 0) collectors.push(postgresPoolCollector(pools, timeoutMs));
|
|
32
|
+
const statsPool = store ?? vector;
|
|
33
|
+
if (statsPool && serverWide) {
|
|
34
|
+
const statementsEveryTicks = Math.max(1, Math.round(SLOW_STATEMENTS_EVERY_MS / intervalMs));
|
|
35
|
+
collectors.push(postgresStatsCollector(statsPool, timeoutMs, statementsEveryTicks, targets.getLogger));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (metrics.enabled("redis")) {
|
|
39
|
+
const pubsub = asMonitorablePubSub(sources.pubsub);
|
|
40
|
+
if (pubsub) {
|
|
41
|
+
redisClient = connectMonitor(pubsub, targets.getLogger);
|
|
42
|
+
collectors.push(redisProbeCollector(redisClient, timeoutMs));
|
|
43
|
+
const cluster = "masters" in redisClient;
|
|
44
|
+
if (serverWide && !cluster) collectors.push(redisStatsCollector(pubsub, redisClient, timeoutMs));
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (metrics.enabled("sandbox")) {
|
|
48
|
+
collectors.push({ collect(sink) {
|
|
49
|
+
sink.emit("mastra_factory_sandboxes_active", countRunningSessionSandboxes());
|
|
50
|
+
} });
|
|
51
|
+
collectors.push(sessionStartsCollector());
|
|
52
|
+
}
|
|
53
|
+
if (metrics.enabled("intake")) {
|
|
54
|
+
for (const id of sources.integrationIds ?? []) if (id === "github" || id === "linear") keepIntakeSource(id);
|
|
55
|
+
collectors.push(intakeCollector());
|
|
56
|
+
}
|
|
57
|
+
metrics.start(targets, collectors);
|
|
58
|
+
return async () => {
|
|
59
|
+
metrics.stop();
|
|
60
|
+
await redisClient?.destroy?.();
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
export { FactoryMetrics, METRIC_GROUPS, factoryMetrics, installFactoryMetrics, metricsService, resolveMetricsConfig, startFactoryMetrics };
|
|
64
|
+
|
|
65
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../../src/metrics/index.ts"],"sourcesContent":["import { countRunningSessionSandboxes } from '../sandbox/session-sandbox.js';\nimport { postgresPoolCollector, postgresStatsCollector, asPool } from './collectors/postgres.js';\nimport type { PoolLike } from './collectors/postgres.js';\nimport { processCollector } from './collectors/process.js';\nimport { asMonitorablePubSub, connectMonitor, redisProbeCollector, redisStatsCollector } from './collectors/redis.js';\nimport type { RedisMonitorClient } from './collectors/redis.js';\nimport { intakeCollector, keepIntakeSource } from './intake.js';\nimport type { FactoryMetrics, MetricsCollector, MetricsTargets } from './metrics.js';\nimport { sessionStartsCollector } from './sandbox.js';\n\nexport { METRIC_GROUPS, metricsService, resolveMetricsConfig } from './config.js';\nexport type { FactoryMetricsOptions, MetricGroup } from './config.js';\nexport { FactoryMetrics, factoryMetrics, installFactoryMetrics } from './metrics.js';\n\nconst SLOW_STATEMENTS_EVERY_MS = 10 * 60_000;\n\nexport interface MetricsSources {\n /** The Mastra storage, read for its `pg` pool when it's Postgres. */\n storage: unknown;\n vector: unknown;\n pubsub: unknown;\n /** Ids of the configured integrations; GitHub and Linear intake counts are sent as 0 when quiet. */\n integrationIds?: readonly string[];\n}\n\n/**\n * Builds the collectors for the enabled groups and starts the interval timer. Server-wide stats\n * (Postgres activity, Redis memory and stream backlog) only run on the worker service (or a\n * single-process deployment), so a split deployment doesn't report them twice. Returns a stop\n * function that also closes the Redis monitor connection.\n */\nexport function startFactoryMetrics(\n metrics: FactoryMetrics,\n targets: MetricsTargets,\n sources: MetricsSources,\n): () => Promise<void> {\n const intervalMs = metrics.intervalMs;\n if (intervalMs === 0) return async () => {};\n const timeoutMs = Math.min(5_000, intervalMs / 2);\n const serverWide = metrics.service !== 'api';\n const collectors: MetricsCollector[] = [];\n let redisClient: RedisMonitorClient | undefined;\n\n if (metrics.enabled('process'))\n collectors.push(processCollector((kind, ms) => metrics.observe('process', 'mastra_factory_gc_ms', ms, { kind })));\n\n if (metrics.enabled('postgres')) {\n const pools: Record<string, PoolLike> = {};\n const store = asPool((sources.storage as { pool?: unknown } | undefined)?.pool);\n const vector = asPool((sources.vector as { pool?: unknown } | undefined)?.pool);\n if (store) pools.store = store;\n if (vector) pools.vector = vector;\n if (Object.keys(pools).length > 0) collectors.push(postgresPoolCollector(pools, timeoutMs));\n const statsPool = store ?? vector;\n if (statsPool && serverWide) {\n const statementsEveryTicks = Math.max(1, Math.round(SLOW_STATEMENTS_EVERY_MS / intervalMs));\n collectors.push(postgresStatsCollector(statsPool, timeoutMs, statementsEveryTicks, targets.getLogger));\n }\n }\n\n if (metrics.enabled('redis')) {\n const pubsub = asMonitorablePubSub(sources.pubsub);\n if (pubsub) {\n redisClient = connectMonitor(pubsub, targets.getLogger);\n collectors.push(redisProbeCollector(redisClient, timeoutMs));\n // On a cluster, INFO reads one node, so server-wide stats would mix nodes. Only the probe runs.\n const cluster = 'masters' in (redisClient as object);\n if (serverWide && !cluster) collectors.push(redisStatsCollector(pubsub, redisClient, timeoutMs));\n }\n }\n\n if (metrics.enabled('sandbox')) {\n collectors.push({\n collect(sink) {\n sink.emit('mastra_factory_sandboxes_active', countRunningSessionSandboxes());\n },\n });\n collectors.push(sessionStartsCollector());\n }\n\n if (metrics.enabled('intake')) {\n for (const id of sources.integrationIds ?? []) {\n if (id === 'github' || id === 'linear') keepIntakeSource(id);\n }\n collectors.push(intakeCollector());\n }\n\n metrics.start(targets, collectors);\n return async () => {\n metrics.stop();\n await redisClient?.destroy?.();\n };\n}\n"],"mappings":";;;;;;;;;AAcA,MAAM,2BAA2B;;;;;;;AAiBjC,SAAgB,oBACd,SACA,SACA,SACqB;CACrB,MAAM,aAAa,QAAQ;CAC3B,IAAI,eAAe,GAAG,OAAO,YAAY,CAAC;CAC1C,MAAM,YAAY,KAAK,IAAI,KAAO,aAAa,CAAC;CAChD,MAAM,aAAa,QAAQ,YAAY;CACvC,MAAM,aAAiC,CAAC;CACxC,IAAI;CAEJ,IAAI,QAAQ,QAAQ,SAAS,GAC3B,WAAW,KAAK,kBAAkB,MAAM,OAAO,QAAQ,QAAQ,WAAW,wBAAwB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;CAElH,IAAI,QAAQ,QAAQ,UAAU,GAAG;EAC/B,MAAM,QAAkC,CAAC;EACzC,MAAM,QAAQ,OAAQ,QAAQ,SAA4C,IAAI;EAC9E,MAAM,SAAS,OAAQ,QAAQ,QAA2C,IAAI;EAC9E,IAAI,OAAO,MAAM,QAAQ;EACzB,IAAI,QAAQ,MAAM,SAAS;EAC3B,IAAI,OAAO,KAAK,KAAK,CAAC,CAAC,SAAS,GAAG,WAAW,KAAK,sBAAsB,OAAO,SAAS,CAAC;EAC1F,MAAM,YAAY,SAAS;EAC3B,IAAI,aAAa,YAAY;GAC3B,MAAM,uBAAuB,KAAK,IAAI,GAAG,KAAK,MAAM,2BAA2B,UAAU,CAAC;GAC1F,WAAW,KAAK,uBAAuB,WAAW,WAAW,sBAAsB,QAAQ,SAAS,CAAC;EACvG;CACF;CAEA,IAAI,QAAQ,QAAQ,OAAO,GAAG;EAC5B,MAAM,SAAS,oBAAoB,QAAQ,MAAM;EACjD,IAAI,QAAQ;GACV,cAAc,eAAe,QAAQ,QAAQ,SAAS;GACtD,WAAW,KAAK,oBAAoB,aAAa,SAAS,CAAC;GAE3D,MAAM,UAAU,aAAc;GAC9B,IAAI,cAAc,CAAC,SAAS,WAAW,KAAK,oBAAoB,QAAQ,aAAa,SAAS,CAAC;EACjG;CACF;CAEA,IAAI,QAAQ,QAAQ,SAAS,GAAG;EAC9B,WAAW,KAAK,EACd,QAAQ,MAAM;GACZ,KAAK,KAAK,mCAAmC,6BAA6B,CAAC;EAC7E,EACF,CAAC;EACD,WAAW,KAAK,uBAAuB,CAAC;CAC1C;CAEA,IAAI,QAAQ,QAAQ,QAAQ,GAAG;EAC7B,KAAK,MAAM,MAAM,QAAQ,kBAAkB,CAAC,GAC1C,IAAI,OAAO,YAAY,OAAO,UAAU,iBAAiB,EAAE;EAE7D,WAAW,KAAK,gBAAgB,CAAC;CACnC;CAEA,QAAQ,MAAM,SAAS,UAAU;CACjC,OAAO,YAAY;EACjB,QAAQ,KAAK;EACb,MAAM,aAAa,UAAU;CAC/B;AACF"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { MetricsCollector } from './metrics.js';
|
|
2
|
+
export type IntakeSource = 'github' | 'linear' | 'gitlab';
|
|
3
|
+
/** One inbound event handed to a source's rules, by how it ended (`committed`, `replayed`, `ignored`, `missing`, `error`). */
|
|
4
|
+
export declare function recordIntakeEvent(source: IntakeSource, outcome: string): void;
|
|
5
|
+
/**
|
|
6
|
+
* Sends intake event counts for `source` every interval, 0 when quiet, so an idle source reads
|
|
7
|
+
* as zero instead of looking unreported.
|
|
8
|
+
*/
|
|
9
|
+
export declare function keepIntakeSource(source: IntakeSource): void;
|
|
10
|
+
/**
|
|
11
|
+
* One reconcile sweep. `replayed` counts changes the sweep found that live events never
|
|
12
|
+
* delivered: each one is a missed webhook.
|
|
13
|
+
*/
|
|
14
|
+
export declare function recordReconcileSweep(source: IntakeSource, kind: string, result: {
|
|
15
|
+
replayed: number;
|
|
16
|
+
ok: boolean;
|
|
17
|
+
}): void;
|
|
18
|
+
/**
|
|
19
|
+
* The reconcile worker for `source` started sweeping `kind`. The age counts from here until the
|
|
20
|
+
* first clean sweep, so a sweep that fails or stalls from boot shows a growing age instead of no series.
|
|
21
|
+
*/
|
|
22
|
+
export declare function recordSweepWorkerStarted(source: IntakeSource, kind: string): void;
|
|
23
|
+
/**
|
|
24
|
+
* Age of each source's last clean reconcile sweep, per kind. Growing without bound means the sweep has
|
|
25
|
+
* stalled, and webhooks it would have caught stay missed.
|
|
26
|
+
*/
|
|
27
|
+
export declare function intakeCollector(): MetricsCollector;
|
|
28
|
+
//# sourceMappingURL=intake.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"intake.d.ts","sourceRoot":"","sources":["../../src/metrics/intake.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGrD,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE1D,8HAA8H;AAC9H,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE7E;AAKD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAK3D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,OAAO,CAAA;CAAE,GACxC,IAAI,CAON;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAMjF;AAED;;;GAGG;AACH,wBAAgB,eAAe,IAAI,gBAAgB,CASlD"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { factoryMetrics } from "./metrics.js";
|
|
2
|
+
// src/metrics/intake.ts
|
|
3
|
+
/** One inbound event handed to a source's rules, by how it ended (`committed`, `replayed`, `ignored`, `missing`, `error`). */
|
|
4
|
+
function recordIntakeEvent(source, outcome) {
|
|
5
|
+
factoryMetrics().count("intake", "mastra_factory_intake_events_total", {
|
|
6
|
+
source,
|
|
7
|
+
outcome
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
const lastSweepAt = /* @__PURE__ */ new Map();
|
|
11
|
+
/**
|
|
12
|
+
* Sends intake event counts for `source` every interval, 0 when quiet, so an idle source reads
|
|
13
|
+
* as zero instead of looking unreported.
|
|
14
|
+
*/
|
|
15
|
+
function keepIntakeSource(source) {
|
|
16
|
+
const metrics = factoryMetrics();
|
|
17
|
+
for (const outcome of ["committed", "error"]) metrics.keepCounter("intake", "mastra_factory_intake_events_total", {
|
|
18
|
+
source,
|
|
19
|
+
outcome
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* One reconcile sweep. `replayed` counts changes the sweep found that live events never
|
|
24
|
+
* delivered: each one is a missed webhook.
|
|
25
|
+
*/
|
|
26
|
+
function recordReconcileSweep(source, kind, result) {
|
|
27
|
+
const metrics = factoryMetrics();
|
|
28
|
+
if (!metrics.enabled("intake")) return;
|
|
29
|
+
if (result.replayed > 0) metrics.count("intake", "mastra_factory_intake_replayed_total", {
|
|
30
|
+
source,
|
|
31
|
+
kind
|
|
32
|
+
}, result.replayed);
|
|
33
|
+
if (result.ok) lastSweepAt.set(`${source}:${kind}`, {
|
|
34
|
+
source,
|
|
35
|
+
kind,
|
|
36
|
+
at: Date.now()
|
|
37
|
+
});
|
|
38
|
+
else metrics.count("intake", "mastra_factory_intake_sweep_failures_total", {
|
|
39
|
+
source,
|
|
40
|
+
kind
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The reconcile worker for `source` started sweeping `kind`. The age counts from here until the
|
|
45
|
+
* first clean sweep, so a sweep that fails or stalls from boot shows a growing age instead of no series.
|
|
46
|
+
*/
|
|
47
|
+
function recordSweepWorkerStarted(source, kind) {
|
|
48
|
+
const metrics = factoryMetrics();
|
|
49
|
+
if (!metrics.enabled("intake")) return;
|
|
50
|
+
metrics.keepCounter("intake", "mastra_factory_intake_sweep_failures_total", {
|
|
51
|
+
source,
|
|
52
|
+
kind
|
|
53
|
+
});
|
|
54
|
+
const key = `${source}:${kind}`;
|
|
55
|
+
if (!lastSweepAt.has(key)) lastSweepAt.set(key, {
|
|
56
|
+
source,
|
|
57
|
+
kind,
|
|
58
|
+
at: Date.now()
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Age of each source's last clean reconcile sweep, per kind. Growing without bound means the sweep has
|
|
63
|
+
* stalled, and webhooks it would have caught stay missed.
|
|
64
|
+
*/
|
|
65
|
+
function intakeCollector() {
|
|
66
|
+
return { collect(sink) {
|
|
67
|
+
const now = Date.now();
|
|
68
|
+
for (const { source, kind, at } of lastSweepAt.values()) sink.emit("mastra_factory_intake_last_sweep_age_ms", now - at, {
|
|
69
|
+
source,
|
|
70
|
+
kind
|
|
71
|
+
});
|
|
72
|
+
} };
|
|
73
|
+
}
|
|
74
|
+
export { intakeCollector, keepIntakeSource, recordIntakeEvent, recordReconcileSweep, recordSweepWorkerStarted };
|
|
75
|
+
|
|
76
|
+
//# sourceMappingURL=intake.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"intake.js","names":[],"sources":["../../src/metrics/intake.ts"],"sourcesContent":["import type { MetricsCollector } from './metrics.js';\nimport { factoryMetrics } from './metrics.js';\n\nexport type IntakeSource = 'github' | 'linear' | 'gitlab';\n\n/** One inbound event handed to a source's rules, by how it ended (`committed`, `replayed`, `ignored`, `missing`, `error`). */\nexport function recordIntakeEvent(source: IntakeSource, outcome: string): void {\n factoryMetrics().count('intake', 'mastra_factory_intake_events_total', { source, outcome });\n}\n\n// When each source's reconcile sweep of each kind last finished cleanly, in this process.\nconst lastSweepAt = new Map<string, { source: IntakeSource; kind: string; at: number }>();\n\n/**\n * Sends intake event counts for `source` every interval, 0 when quiet, so an idle source reads\n * as zero instead of looking unreported.\n */\nexport function keepIntakeSource(source: IntakeSource): void {\n const metrics = factoryMetrics();\n for (const outcome of ['committed', 'error']) {\n metrics.keepCounter('intake', 'mastra_factory_intake_events_total', { source, outcome });\n }\n}\n\n/**\n * One reconcile sweep. `replayed` counts changes the sweep found that live events never\n * delivered: each one is a missed webhook.\n */\nexport function recordReconcileSweep(\n source: IntakeSource,\n kind: string,\n result: { replayed: number; ok: boolean },\n): void {\n const metrics = factoryMetrics();\n if (!metrics.enabled('intake')) return;\n if (result.replayed > 0)\n metrics.count('intake', 'mastra_factory_intake_replayed_total', { source, kind }, result.replayed);\n if (result.ok) lastSweepAt.set(`${source}:${kind}`, { source, kind, at: Date.now() });\n else metrics.count('intake', 'mastra_factory_intake_sweep_failures_total', { source, kind });\n}\n\n/**\n * The reconcile worker for `source` started sweeping `kind`. The age counts from here until the\n * first clean sweep, so a sweep that fails or stalls from boot shows a growing age instead of no series.\n */\nexport function recordSweepWorkerStarted(source: IntakeSource, kind: string): void {\n const metrics = factoryMetrics();\n if (!metrics.enabled('intake')) return;\n metrics.keepCounter('intake', 'mastra_factory_intake_sweep_failures_total', { source, kind });\n const key = `${source}:${kind}`;\n if (!lastSweepAt.has(key)) lastSweepAt.set(key, { source, kind, at: Date.now() });\n}\n\n/**\n * Age of each source's last clean reconcile sweep, per kind. Growing without bound means the sweep has\n * stalled, and webhooks it would have caught stay missed.\n */\nexport function intakeCollector(): MetricsCollector {\n return {\n collect(sink) {\n const now = Date.now();\n for (const { source, kind, at } of lastSweepAt.values()) {\n sink.emit('mastra_factory_intake_last_sweep_age_ms', now - at, { source, kind });\n }\n },\n };\n}\n"],"mappings":";;;AAMA,SAAgB,kBAAkB,QAAsB,SAAuB;CAC7E,eAAe,CAAC,CAAC,MAAM,UAAU,sCAAsC;EAAE;EAAQ;CAAQ,CAAC;AAC5F;AAGA,MAAM,8BAAc,IAAI,IAAgE;;;;;AAMxF,SAAgB,iBAAiB,QAA4B;CAC3D,MAAM,UAAU,eAAe;CAC/B,KAAK,MAAM,WAAW,CAAC,aAAa,OAAO,GACzC,QAAQ,YAAY,UAAU,sCAAsC;EAAE;EAAQ;CAAQ,CAAC;AAE3F;;;;;AAMA,SAAgB,qBACd,QACA,MACA,QACM;CACN,MAAM,UAAU,eAAe;CAC/B,IAAI,CAAC,QAAQ,QAAQ,QAAQ,GAAG;CAChC,IAAI,OAAO,WAAW,GACpB,QAAQ,MAAM,UAAU,wCAAwC;EAAE;EAAQ;CAAK,GAAG,OAAO,QAAQ;CACnG,IAAI,OAAO,IAAI,YAAY,IAAI,GAAG,OAAO,GAAG,QAAQ;EAAE;EAAQ;EAAM,IAAI,KAAK,IAAI;CAAE,CAAC;MAC/E,QAAQ,MAAM,UAAU,8CAA8C;EAAE;EAAQ;CAAK,CAAC;AAC7F;;;;;AAMA,SAAgB,yBAAyB,QAAsB,MAAoB;CACjF,MAAM,UAAU,eAAe;CAC/B,IAAI,CAAC,QAAQ,QAAQ,QAAQ,GAAG;CAChC,QAAQ,YAAY,UAAU,8CAA8C;EAAE;EAAQ;CAAK,CAAC;CAC5F,MAAM,MAAM,GAAG,OAAO,GAAG;CACzB,IAAI,CAAC,YAAY,IAAI,GAAG,GAAG,YAAY,IAAI,KAAK;EAAE;EAAQ;EAAM,IAAI,KAAK,IAAI;CAAE,CAAC;AAClF;;;;;AAMA,SAAgB,kBAAoC;CAClD,OAAO,EACL,QAAQ,MAAM;EACZ,MAAM,MAAM,KAAK,IAAI;EACrB,KAAK,MAAM,EAAE,QAAQ,MAAM,QAAQ,YAAY,OAAO,GACpD,KAAK,KAAK,2CAA2C,MAAM,IAAI;GAAE;GAAQ;EAAK,CAAC;CAEnF,EACF;AACF"}
|