@mastra/factory 0.29.0-alpha.1 → 0.29.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -1
- package/dist/factory.d.ts +8 -0
- package/dist/factory.d.ts.map +1 -1
- package/dist/factory.js +26 -2
- package/dist/factory.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/integrations/github/reconcile-worker.d.ts.map +1 -1
- package/dist/integrations/github/reconcile-worker.js +30 -1
- package/dist/integrations/github/reconcile-worker.js.map +1 -1
- package/dist/integrations/github/rules.d.ts.map +1 -1
- package/dist/integrations/github/rules.js +17 -3
- package/dist/integrations/github/rules.js.map +1 -1
- package/dist/integrations/github/sandbox.d.ts.map +1 -1
- package/dist/integrations/github/sandbox.js +11 -2
- package/dist/integrations/github/sandbox.js.map +1 -1
- package/dist/integrations/linear/rules.d.ts.map +1 -1
- package/dist/integrations/linear/rules.js +11 -0
- package/dist/integrations/linear/rules.js.map +1 -1
- package/dist/metrics/collectors/postgres.d.ts +29 -0
- package/dist/metrics/collectors/postgres.d.ts.map +1 -0
- package/dist/metrics/collectors/postgres.js +121 -0
- package/dist/metrics/collectors/postgres.js.map +1 -0
- package/dist/metrics/collectors/probe.d.ts +22 -0
- package/dist/metrics/collectors/probe.d.ts.map +1 -0
- package/dist/metrics/collectors/probe.js +69 -0
- package/dist/metrics/collectors/probe.js.map +1 -0
- package/dist/metrics/collectors/process.d.ts +13 -0
- package/dist/metrics/collectors/process.d.ts.map +1 -0
- package/dist/metrics/collectors/process.js +85 -0
- package/dist/metrics/collectors/process.js.map +1 -0
- package/dist/metrics/collectors/redis.d.ts +29 -0
- package/dist/metrics/collectors/redis.d.ts.map +1 -0
- package/dist/metrics/collectors/redis.js +97 -0
- package/dist/metrics/collectors/redis.js.map +1 -0
- package/dist/metrics/config.d.ts +33 -0
- package/dist/metrics/config.d.ts.map +1 -0
- package/dist/metrics/config.js +65 -0
- package/dist/metrics/config.js.map +1 -0
- package/dist/metrics/index.d.ts +20 -0
- package/dist/metrics/index.d.ts.map +1 -0
- package/dist/metrics/index.js +65 -0
- package/dist/metrics/index.js.map +1 -0
- package/dist/metrics/intake.d.ts +28 -0
- package/dist/metrics/intake.d.ts.map +1 -0
- package/dist/metrics/intake.js +76 -0
- package/dist/metrics/intake.js.map +1 -0
- package/dist/metrics/metrics.d.ts +60 -0
- package/dist/metrics/metrics.d.ts.map +1 -0
- package/dist/metrics/metrics.js +182 -0
- package/dist/metrics/metrics.js.map +1 -0
- package/dist/metrics/sandbox.d.ts +35 -0
- package/dist/metrics/sandbox.d.ts.map +1 -0
- package/dist/metrics/sandbox.js +156 -0
- package/dist/metrics/sandbox.js.map +1 -0
- package/dist/routes/surface.d.ts.map +1 -1
- package/dist/routes/surface.js +1 -0
- package/dist/routes/surface.js.map +1 -1
- package/dist/rules/dispatcher.d.ts +1 -0
- package/dist/rules/dispatcher.d.ts.map +1 -1
- package/dist/rules/dispatcher.js +26 -8
- package/dist/rules/dispatcher.js.map +1 -1
- package/dist/rules/start-coordinator.d.ts +1 -0
- package/dist/rules/start-coordinator.d.ts.map +1 -1
- package/dist/rules/start-coordinator.js +1 -0
- package/dist/rules/start-coordinator.js.map +1 -1
- package/dist/rules/types.d.ts +2 -2
- package/dist/rules/types.d.ts.map +1 -1
- package/dist/rules/types.js +4 -1
- package/dist/rules/types.js.map +1 -1
- package/dist/rules/validation.d.ts.map +1 -1
- package/dist/rules/validation.js +1 -0
- package/dist/rules/validation.js.map +1 -1
- package/dist/sandbox/session-retirement.d.ts.map +1 -1
- package/dist/sandbox/session-retirement.js +22 -14
- package/dist/sandbox/session-retirement.js.map +1 -1
- package/dist/sandbox/session-sandbox.d.ts +2 -0
- package/dist/sandbox/session-sandbox.d.ts.map +1 -1
- package/dist/sandbox/session-sandbox.js +7 -1
- package/dist/sandbox/session-sandbox.js.map +1 -1
- package/dist/storage/domains/work-items/base.d.ts +5 -0
- package/dist/storage/domains/work-items/base.d.ts.map +1 -1
- package/dist/storage/domains/work-items/base.js +19 -2
- package/dist/storage/domains/work-items/base.js.map +1 -1
- package/dist/timing.d.ts +4 -1
- package/dist/timing.d.ts.map +1 -1
- package/dist/timing.js +12 -4
- package/dist/timing.js.map +1 -1
- package/dist/work-item-branch.d.ts +1 -1
- package/dist/work-item-branch.d.ts.map +1 -1
- package/dist/work-item-branch.js +1 -0
- package/dist/work-item-branch.js.map +1 -1
- package/dist/workspace.d.ts.map +1 -1
- package/dist/workspace.js +7 -1
- package/dist/workspace.js.map +1 -1
- package/package.json +6 -6
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { createProbe, errorText } from "./probe.js";
|
|
2
|
+
// src/metrics/collectors/postgres.ts
|
|
3
|
+
/** `value` when it looks like a `pg.Pool` (e.g. `PostgresStore.pool`, `PgVector.pool`). */
|
|
4
|
+
function asPool(value) {
|
|
5
|
+
if (!value || typeof value !== "object") return void 0;
|
|
6
|
+
const pool = value;
|
|
7
|
+
return typeof pool.totalCount === "number" && typeof pool.idleCount === "number" && typeof pool.waitingCount === "number" && typeof pool.query === "function" ? pool : void 0;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Pool usage plus a `SELECT 1` round trip for each Postgres pool. `waiting` above 0 means queries
|
|
11
|
+
* are queued for a connection: the pool is too small or queries hold connections too long.
|
|
12
|
+
* The gauges are read after the probe, so a pool that had closed its idle connections shows the
|
|
13
|
+
* connection the probe just opened; the probe's duration then includes connecting.
|
|
14
|
+
*/
|
|
15
|
+
function postgresPoolCollector(pools, timeoutMs) {
|
|
16
|
+
const probe = createProbe();
|
|
17
|
+
return { async collect(sink) {
|
|
18
|
+
await Promise.all(Object.entries(pools).map(async ([name, pool]) => {
|
|
19
|
+
await probe(sink, `db_${name}`, timeoutMs, () => pool.query("SELECT 1"));
|
|
20
|
+
sink.emit("mastra_factory_db_pool", pool.totalCount, {
|
|
21
|
+
pool: name,
|
|
22
|
+
state: "total"
|
|
23
|
+
});
|
|
24
|
+
sink.emit("mastra_factory_db_pool", pool.idleCount, {
|
|
25
|
+
pool: name,
|
|
26
|
+
state: "idle"
|
|
27
|
+
});
|
|
28
|
+
sink.emit("mastra_factory_db_pool", pool.waitingCount, {
|
|
29
|
+
pool: name,
|
|
30
|
+
state: "waiting"
|
|
31
|
+
});
|
|
32
|
+
}));
|
|
33
|
+
} };
|
|
34
|
+
}
|
|
35
|
+
const ACTIVITY_SQL = `
|
|
36
|
+
SELECT
|
|
37
|
+
count(*) FILTER (WHERE state = 'active') AS active,
|
|
38
|
+
count(*) FILTER (WHERE state = 'idle in transaction') AS idle_in_transaction,
|
|
39
|
+
count(*) FILTER (WHERE wait_event_type = 'Lock') AS waiting_on_lock,
|
|
40
|
+
coalesce(max(extract(epoch FROM now() - query_start)) FILTER (WHERE state = 'active'), 0) AS longest_query_s,
|
|
41
|
+
coalesce(max(extract(epoch FROM now() - xact_start)) FILTER (WHERE state = 'idle in transaction'), 0) AS longest_idle_transaction_s
|
|
42
|
+
FROM pg_stat_activity
|
|
43
|
+
WHERE datname = current_database() AND backend_type = 'client backend' AND pid <> pg_backend_pid()`;
|
|
44
|
+
const STATEMENTS_SQL = `
|
|
45
|
+
SELECT queryid::text AS id, sum(calls) AS calls, sum(total_exec_time) AS total_exec_time
|
|
46
|
+
FROM pg_stat_statements
|
|
47
|
+
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) AND queryid IS NOT NULL
|
|
48
|
+
GROUP BY queryid`;
|
|
49
|
+
const SLOW_STATEMENTS_LOGGED = 5;
|
|
50
|
+
/**
|
|
51
|
+
* Server-wide Postgres stats, collected by one process per deployment. Every tick:
|
|
52
|
+
* `pg_stat_activity` session counts and the age of the longest running query and longest idle
|
|
53
|
+
* transaction. Every `statementsEveryTicks` ticks: the statements that used the most time since the
|
|
54
|
+
* previous sample, from `pg_stat_statements`, written as one log line with their query IDs and
|
|
55
|
+
* timings. Statement text is left out: pg_stat_statements can keep literal values in it, which
|
|
56
|
+
* mustn't reach application logs. Skipped when the extension isn't installed.
|
|
57
|
+
*/
|
|
58
|
+
function postgresStatsCollector(pool, timeoutMs, statementsEveryTicks, getLogger) {
|
|
59
|
+
const probe = createProbe();
|
|
60
|
+
let tick = 0;
|
|
61
|
+
let previous;
|
|
62
|
+
let failing = false;
|
|
63
|
+
const number = (value) => Number(value ?? 0);
|
|
64
|
+
async function logSlowStatements() {
|
|
65
|
+
const { rows: installed } = await pool.query(`SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'`);
|
|
66
|
+
if (installed.length === 0) return;
|
|
67
|
+
const { rows } = await pool.query(STATEMENTS_SQL);
|
|
68
|
+
const current = new Map(rows.map((row) => [String(row.id), {
|
|
69
|
+
calls: number(row.calls),
|
|
70
|
+
time: number(row.total_exec_time)
|
|
71
|
+
}]));
|
|
72
|
+
if (previous) {
|
|
73
|
+
const before = previous;
|
|
74
|
+
const slowest = [...current].map(([id, now]) => {
|
|
75
|
+
const then = before.get(id);
|
|
76
|
+
if (!then) return {
|
|
77
|
+
id,
|
|
78
|
+
calls: 0,
|
|
79
|
+
time: 0
|
|
80
|
+
};
|
|
81
|
+
const reset = then.calls > now.calls;
|
|
82
|
+
return {
|
|
83
|
+
id,
|
|
84
|
+
calls: reset ? now.calls : now.calls - then.calls,
|
|
85
|
+
time: reset ? now.time : now.time - then.time
|
|
86
|
+
};
|
|
87
|
+
}).filter((statement) => statement.calls > 0).sort((a, b) => b.time - a.time).slice(0, SLOW_STATEMENTS_LOGGED).map((statement) => ({
|
|
88
|
+
queryId: statement.id,
|
|
89
|
+
calls: statement.calls,
|
|
90
|
+
totalMs: Math.round(statement.time),
|
|
91
|
+
meanMs: Math.round(statement.time / statement.calls * 10) / 10
|
|
92
|
+
}));
|
|
93
|
+
if (slowest.length > 0) getLogger()?.info("[factory:metrics] Slowest Postgres statements since last sample.", { statements: slowest });
|
|
94
|
+
}
|
|
95
|
+
previous = current;
|
|
96
|
+
}
|
|
97
|
+
return { async collect(sink) {
|
|
98
|
+
const sampleStatements = tick % statementsEveryTicks === 0;
|
|
99
|
+
tick++;
|
|
100
|
+
let row;
|
|
101
|
+
const result = await probe(sink, "db_stats", timeoutMs, async () => {
|
|
102
|
+
row = (await pool.query(ACTIVITY_SQL)).rows[0] ?? {};
|
|
103
|
+
});
|
|
104
|
+
if (result.ok && row) {
|
|
105
|
+
sink.emit("mastra_factory_db_sessions", number(row.active), { state: "active" });
|
|
106
|
+
sink.emit("mastra_factory_db_sessions", number(row.idle_in_transaction), { state: "idle_in_transaction" });
|
|
107
|
+
sink.emit("mastra_factory_db_sessions", number(row.waiting_on_lock), { state: "waiting_on_lock" });
|
|
108
|
+
sink.emit("mastra_factory_db_longest_ms", number(row.longest_query_s) * 1e3, { kind: "query" });
|
|
109
|
+
sink.emit("mastra_factory_db_longest_ms", number(row.longest_idle_transaction_s) * 1e3, { kind: "idle_transaction" });
|
|
110
|
+
if (failing) getLogger()?.info("[factory:metrics] Postgres stats sampling recovered.");
|
|
111
|
+
failing = false;
|
|
112
|
+
} else if (!result.ok) {
|
|
113
|
+
if (!failing) getLogger()?.warn("[factory:metrics] Postgres stats sampling failed.", { error: errorText(result.error) });
|
|
114
|
+
failing = true;
|
|
115
|
+
}
|
|
116
|
+
if (sampleStatements) await probe(sink, "db_statements", timeoutMs, logSlowStatements);
|
|
117
|
+
} };
|
|
118
|
+
}
|
|
119
|
+
export { asPool, postgresPoolCollector, postgresStatsCollector };
|
|
120
|
+
|
|
121
|
+
//# sourceMappingURL=postgres.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postgres.js","names":[],"sources":["../../../src/metrics/collectors/postgres.ts"],"sourcesContent":["import type { MetricsCollector, MetricsLogger } from '../metrics.js';\nimport { createProbe, errorText } from './probe.js';\n\n/** The subset of `pg.Pool` the collectors read. */\nexport interface PoolLike {\n totalCount: number;\n idleCount: number;\n waitingCount: number;\n query(sql: string): Promise<{ rows: Array<Record<string, unknown>> }>;\n}\n\n/** `value` when it looks like a `pg.Pool` (e.g. `PostgresStore.pool`, `PgVector.pool`). */\nexport function asPool(value: unknown): PoolLike | undefined {\n if (!value || typeof value !== 'object') return undefined;\n const pool = value as Partial<PoolLike>;\n return typeof pool.totalCount === 'number' &&\n typeof pool.idleCount === 'number' &&\n typeof pool.waitingCount === 'number' &&\n typeof pool.query === 'function'\n ? (pool as PoolLike)\n : undefined;\n}\n\n/**\n * Pool usage plus a `SELECT 1` round trip for each Postgres pool. `waiting` above 0 means queries\n * are queued for a connection: the pool is too small or queries hold connections too long.\n * The gauges are read after the probe, so a pool that had closed its idle connections shows the\n * connection the probe just opened; the probe's duration then includes connecting.\n */\nexport function postgresPoolCollector(pools: Record<string, PoolLike>, timeoutMs: number): MetricsCollector {\n const probe = createProbe();\n return {\n async collect(sink) {\n await Promise.all(\n Object.entries(pools).map(async ([name, pool]) => {\n await probe(sink, `db_${name}`, timeoutMs, () => pool.query('SELECT 1'));\n sink.emit('mastra_factory_db_pool', pool.totalCount, { pool: name, state: 'total' });\n sink.emit('mastra_factory_db_pool', pool.idleCount, { pool: name, state: 'idle' });\n sink.emit('mastra_factory_db_pool', pool.waitingCount, { pool: name, state: 'waiting' });\n }),\n );\n },\n };\n}\n\n// Sessions of this database, from the server's point of view. Locks and long-open transactions\n// slow every query, and the pool stats can't see them.\nconst ACTIVITY_SQL = `\nSELECT\n count(*) FILTER (WHERE state = 'active') AS active,\n count(*) FILTER (WHERE state = 'idle in transaction') AS idle_in_transaction,\n count(*) FILTER (WHERE wait_event_type = 'Lock') AS waiting_on_lock,\n coalesce(max(extract(epoch FROM now() - query_start)) FILTER (WHERE state = 'active'), 0) AS longest_query_s,\n coalesce(max(extract(epoch FROM now() - xact_start)) FILTER (WHERE state = 'idle in transaction'), 0) AS longest_idle_transaction_s\nFROM pg_stat_activity\nWHERE datname = current_database() AND backend_type = 'client backend' AND pid <> pg_backend_pid()`;\n\n// Lifetime totals of every statement of this database, one row per query ID. pg_stat_statements\n// keeps a row per user (and top-level flag) for the same query ID, so they are summed. Without\n// `pg_read_all_stats`, other users' query IDs read as null; those rows are skipped rather than lumped together. No limit:\n// the slowest are picked from the change since the previous sample, which a statement with a small\n// lifetime total can still lead. The extension caps its rows (`pg_stat_statements.max`, 5000 by default).\nconst STATEMENTS_SQL = `\nSELECT queryid::text AS id, sum(calls) AS calls, sum(total_exec_time) AS total_exec_time\nFROM pg_stat_statements\nWHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) AND queryid IS NOT NULL\nGROUP BY queryid`;\n\nconst SLOW_STATEMENTS_LOGGED = 5;\n\n/**\n * Server-wide Postgres stats, collected by one process per deployment. Every tick:\n * `pg_stat_activity` session counts and the age of the longest running query and longest idle\n * transaction. Every `statementsEveryTicks` ticks: the statements that used the most time since the\n * previous sample, from `pg_stat_statements`, written as one log line with their query IDs and\n * timings. Statement text is left out: pg_stat_statements can keep literal values in it, which\n * mustn't reach application logs. Skipped when the extension isn't installed.\n */\nexport function postgresStatsCollector(\n pool: PoolLike,\n timeoutMs: number,\n statementsEveryTicks: number,\n getLogger: () => MetricsLogger | undefined,\n): MetricsCollector {\n const probe = createProbe();\n let tick = 0;\n let previous: Map<string, { calls: number; time: number }> | undefined;\n let failing = false;\n const number = (value: unknown) => Number(value ?? 0);\n\n async function logSlowStatements(): Promise<void> {\n const { rows: installed } = await pool.query(`SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'`);\n if (installed.length === 0) return;\n const { rows } = await pool.query(STATEMENTS_SQL);\n const current = new Map(\n rows.map(row => [String(row.id), { calls: number(row.calls), time: number(row.total_exec_time) }]),\n );\n if (previous) {\n const before = previous;\n const slowest = [...current]\n .map(([id, now]) => {\n const then = before.get(id);\n // Not in the previous sample: no baseline yet (the totals are lifetime), so skip it.\n if (!then) return { id, calls: 0, time: 0 };\n // Fewer calls than before means the stats were reset: everything counted is new.\n const reset = then.calls > now.calls;\n return {\n id,\n calls: reset ? now.calls : now.calls - then.calls,\n time: reset ? now.time : now.time - then.time,\n };\n })\n .filter(statement => statement.calls > 0)\n .sort((a, b) => b.time - a.time)\n .slice(0, SLOW_STATEMENTS_LOGGED)\n .map(statement => ({\n queryId: statement.id,\n calls: statement.calls,\n totalMs: Math.round(statement.time),\n meanMs: Math.round((statement.time / statement.calls) * 10) / 10,\n }));\n if (slowest.length > 0) {\n getLogger()?.info('[factory:metrics] Slowest Postgres statements since last sample.', { statements: slowest });\n }\n }\n previous = current;\n }\n\n return {\n async collect(sink) {\n const sampleStatements = tick % statementsEveryTicks === 0;\n tick++;\n let row: Record<string, unknown> | undefined;\n const result = await probe(sink, 'db_stats', timeoutMs, async () => {\n row = (await pool.query(ACTIVITY_SQL)).rows[0] ?? {};\n });\n if (result.ok && row) {\n sink.emit('mastra_factory_db_sessions', number(row.active), { state: 'active' });\n sink.emit('mastra_factory_db_sessions', number(row.idle_in_transaction), { state: 'idle_in_transaction' });\n sink.emit('mastra_factory_db_sessions', number(row.waiting_on_lock), { state: 'waiting_on_lock' });\n sink.emit('mastra_factory_db_longest_ms', number(row.longest_query_s) * 1000, { kind: 'query' });\n sink.emit('mastra_factory_db_longest_ms', number(row.longest_idle_transaction_s) * 1000, {\n kind: 'idle_transaction',\n });\n if (failing) getLogger()?.info('[factory:metrics] Postgres stats sampling recovered.');\n failing = false;\n } else if (!result.ok) {\n // Log on the transition only; the probe failure counter counts each failure.\n if (!failing)\n getLogger()?.warn('[factory:metrics] Postgres stats sampling failed.', { error: errorText(result.error) });\n failing = true;\n }\n // Its own probe, so a slow pg_stat_statements read never costs the activity sample.\n if (sampleStatements) await probe(sink, 'db_statements', timeoutMs, logSlowStatements);\n },\n };\n}\n"],"mappings":";;;AAYA,SAAgB,OAAO,OAAsC;CAC3D,IAAI,CAAC,SAAS,OAAO,UAAU,UAAU,OAAO,KAAA;CAChD,MAAM,OAAO;CACb,OAAO,OAAO,KAAK,eAAe,YAChC,OAAO,KAAK,cAAc,YAC1B,OAAO,KAAK,iBAAiB,YAC7B,OAAO,KAAK,UAAU,aACnB,OACD,KAAA;AACN;;;;;;;AAQA,SAAgB,sBAAsB,OAAiC,WAAqC;CAC1G,MAAM,QAAQ,YAAY;CAC1B,OAAO,EACL,MAAM,QAAQ,MAAM;EAClB,MAAM,QAAQ,IACZ,OAAO,QAAQ,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM,UAAU;GAChD,MAAM,MAAM,MAAM,MAAM,QAAQ,iBAAiB,KAAK,MAAM,UAAU,CAAC;GACvE,KAAK,KAAK,0BAA0B,KAAK,YAAY;IAAE,MAAM;IAAM,OAAO;GAAQ,CAAC;GACnF,KAAK,KAAK,0BAA0B,KAAK,WAAW;IAAE,MAAM;IAAM,OAAO;GAAO,CAAC;GACjF,KAAK,KAAK,0BAA0B,KAAK,cAAc;IAAE,MAAM;IAAM,OAAO;GAAU,CAAC;EACzF,CAAC,CACH;CACF,EACF;AACF;AAIA,MAAM,eAAe;;;;;;;;;AAerB,MAAM,iBAAiB;;;;;AAMvB,MAAM,yBAAyB;;;;;;;;;AAU/B,SAAgB,uBACd,MACA,WACA,sBACA,WACkB;CAClB,MAAM,QAAQ,YAAY;CAC1B,IAAI,OAAO;CACX,IAAI;CACJ,IAAI,UAAU;CACd,MAAM,UAAU,UAAmB,OAAO,SAAS,CAAC;CAEpD,eAAe,oBAAmC;EAChD,MAAM,EAAE,MAAM,cAAc,MAAM,KAAK,MAAM,iEAAiE;EAC9G,IAAI,UAAU,WAAW,GAAG;EAC5B,MAAM,EAAE,SAAS,MAAM,KAAK,MAAM,cAAc;EAChD,MAAM,UAAU,IAAI,IAClB,KAAK,KAAI,QAAO,CAAC,OAAO,IAAI,EAAE,GAAG;GAAE,OAAO,OAAO,IAAI,KAAK;GAAG,MAAM,OAAO,IAAI,eAAe;EAAE,CAAC,CAAC,CACnG;EACA,IAAI,UAAU;GACZ,MAAM,SAAS;GACf,MAAM,UAAU,CAAC,GAAG,OAAO,CAAC,CACzB,KAAK,CAAC,IAAI,SAAS;IAClB,MAAM,OAAO,OAAO,IAAI,EAAE;IAE1B,IAAI,CAAC,MAAM,OAAO;KAAE;KAAI,OAAO;KAAG,MAAM;IAAE;IAE1C,MAAM,QAAQ,KAAK,QAAQ,IAAI;IAC/B,OAAO;KACL;KACA,OAAO,QAAQ,IAAI,QAAQ,IAAI,QAAQ,KAAK;KAC5C,MAAM,QAAQ,IAAI,OAAO,IAAI,OAAO,KAAK;IAC3C;GACF,CAAC,CAAC,CACD,QAAO,cAAa,UAAU,QAAQ,CAAC,CAAC,CACxC,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAC/B,MAAM,GAAG,sBAAsB,CAAC,CAChC,KAAI,eAAc;IACjB,SAAS,UAAU;IACnB,OAAO,UAAU;IACjB,SAAS,KAAK,MAAM,UAAU,IAAI;IAClC,QAAQ,KAAK,MAAO,UAAU,OAAO,UAAU,QAAS,EAAE,IAAI;GAChE,EAAE;GACJ,IAAI,QAAQ,SAAS,GACnB,UAAU,CAAC,EAAE,KAAK,oEAAoE,EAAE,YAAY,QAAQ,CAAC;EAEjH;EACA,WAAW;CACb;CAEA,OAAO,EACL,MAAM,QAAQ,MAAM;EAClB,MAAM,mBAAmB,OAAO,yBAAyB;EACzD;EACA,IAAI;EACJ,MAAM,SAAS,MAAM,MAAM,MAAM,YAAY,WAAW,YAAY;GAClE,OAAO,MAAM,KAAK,MAAM,YAAY,EAAA,CAAG,KAAK,MAAM,CAAC;EACrD,CAAC;EACD,IAAI,OAAO,MAAM,KAAK;GACpB,KAAK,KAAK,8BAA8B,OAAO,IAAI,MAAM,GAAG,EAAE,OAAO,SAAS,CAAC;GAC/E,KAAK,KAAK,8BAA8B,OAAO,IAAI,mBAAmB,GAAG,EAAE,OAAO,sBAAsB,CAAC;GACzG,KAAK,KAAK,8BAA8B,OAAO,IAAI,eAAe,GAAG,EAAE,OAAO,kBAAkB,CAAC;GACjG,KAAK,KAAK,gCAAgC,OAAO,IAAI,eAAe,IAAI,KAAM,EAAE,MAAM,QAAQ,CAAC;GAC/F,KAAK,KAAK,gCAAgC,OAAO,IAAI,0BAA0B,IAAI,KAAM,EACvF,MAAM,mBACR,CAAC;GACD,IAAI,SAAS,UAAU,CAAC,EAAE,KAAK,sDAAsD;GACrF,UAAU;EACZ,OAAO,IAAI,CAAC,OAAO,IAAI;GAErB,IAAI,CAAC,SACH,UAAU,CAAC,EAAE,KAAK,qDAAqD,EAAE,OAAO,UAAU,OAAO,KAAK,EAAE,CAAC;GAC3G,UAAU;EACZ;EAEA,IAAI,kBAAkB,MAAM,MAAM,MAAM,iBAAiB,WAAW,iBAAiB;CACvF,EACF;AACF"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { MetricSink } from '../metrics.js';
|
|
2
|
+
export declare class ProbeTimeout extends Error {
|
|
3
|
+
}
|
|
4
|
+
export declare function withTimeout<T>(timeoutMs: number, run: () => Promise<T>): Promise<T>;
|
|
5
|
+
export type ProbeResult = {
|
|
6
|
+
ok: true;
|
|
7
|
+
} | {
|
|
8
|
+
ok: false;
|
|
9
|
+
error: unknown;
|
|
10
|
+
};
|
|
11
|
+
export type Probe = (sink: MetricSink, component: string, timeoutMs: number, run: () => Promise<unknown>) => Promise<ProbeResult>;
|
|
12
|
+
/**
|
|
13
|
+
* A probe for one collector. It times `run` and emits `mastra_factory_probe_duration_ms{component}`
|
|
14
|
+
* on success (only for the components in {@link TIMED_COMPONENTS}), or
|
|
15
|
+
* `mastra_factory_probe_failures_total{component,reason}` when it throws or exceeds
|
|
16
|
+
* `timeoutMs`. The timeout only stops waiting, it can't cancel the call, so a component whose call
|
|
17
|
+
* hasn't settled isn't checked again until it returns: a hung dependency holds at most one call
|
|
18
|
+
* (and one pooled connection) per component. Each collector has its own guard.
|
|
19
|
+
*/
|
|
20
|
+
export declare function createProbe(): Probe;
|
|
21
|
+
export declare function errorText(error: unknown): string;
|
|
22
|
+
//# sourceMappingURL=probe.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../../src/metrics/collectors/probe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAEhD,qBAAa,YAAa,SAAQ,KAAK;CAAG;AAE1C,wBAAsB,WAAW,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAYzF;AAED,MAAM,MAAM,WAAW,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAQvE,MAAM,MAAM,KAAK,GAAG,CAClB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,KACxB,OAAO,CAAC,WAAW,CAAC,CAAC;AAE1B;;;;;;;GAOG;AACH,wBAAgB,WAAW,IAAI,KAAK,CA0BnC;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAGhD"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// src/metrics/collectors/probe.ts
|
|
2
|
+
var ProbeTimeout = class extends Error {};
|
|
3
|
+
async function withTimeout(timeoutMs, run) {
|
|
4
|
+
let timer;
|
|
5
|
+
try {
|
|
6
|
+
return await Promise.race([run(), new Promise((_, reject) => {
|
|
7
|
+
timer = setTimeout(() => reject(new ProbeTimeout()), timeoutMs);
|
|
8
|
+
})]);
|
|
9
|
+
} finally {
|
|
10
|
+
clearTimeout(timer);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Components whose round trip is its own signal. The stats probes run over the same pool or
|
|
15
|
+
* Redis client, so their durations would repeat these; only their failures are sent.
|
|
16
|
+
*/
|
|
17
|
+
const TIMED_COMPONENTS = /* @__PURE__ */ new Set([
|
|
18
|
+
"db_store",
|
|
19
|
+
"db_vector",
|
|
20
|
+
"redis"
|
|
21
|
+
]);
|
|
22
|
+
/**
|
|
23
|
+
* A probe for one collector. It times `run` and emits `mastra_factory_probe_duration_ms{component}`
|
|
24
|
+
* on success (only for the components in {@link TIMED_COMPONENTS}), or
|
|
25
|
+
* `mastra_factory_probe_failures_total{component,reason}` when it throws or exceeds
|
|
26
|
+
* `timeoutMs`. The timeout only stops waiting, it can't cancel the call, so a component whose call
|
|
27
|
+
* hasn't settled isn't checked again until it returns: a hung dependency holds at most one call
|
|
28
|
+
* (and one pooled connection) per component. Each collector has its own guard.
|
|
29
|
+
*/
|
|
30
|
+
function createProbe() {
|
|
31
|
+
const stillRunning = /* @__PURE__ */ new Set();
|
|
32
|
+
return async (sink, component, timeoutMs, run) => {
|
|
33
|
+
const timed = TIMED_COMPONENTS.has(component);
|
|
34
|
+
if (stillRunning.has(component)) {
|
|
35
|
+
sink.emit("mastra_factory_probe_failures_total", 1, {
|
|
36
|
+
component,
|
|
37
|
+
reason: "timeout"
|
|
38
|
+
});
|
|
39
|
+
return {
|
|
40
|
+
ok: false,
|
|
41
|
+
error: new ProbeTimeout()
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
const start = performance.now();
|
|
45
|
+
stillRunning.add(component);
|
|
46
|
+
const call = Promise.resolve().then(run).finally(() => stillRunning.delete(component));
|
|
47
|
+
try {
|
|
48
|
+
await withTimeout(timeoutMs, () => call);
|
|
49
|
+
if (timed) sink.emit("mastra_factory_probe_duration_ms", performance.now() - start, { component });
|
|
50
|
+
return { ok: true };
|
|
51
|
+
} catch (error) {
|
|
52
|
+
sink.emit("mastra_factory_probe_failures_total", 1, {
|
|
53
|
+
component,
|
|
54
|
+
reason: error instanceof ProbeTimeout ? "timeout" : "error"
|
|
55
|
+
});
|
|
56
|
+
return {
|
|
57
|
+
ok: false,
|
|
58
|
+
error
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
function errorText(error) {
|
|
64
|
+
if (error instanceof ProbeTimeout) return "timeout";
|
|
65
|
+
return error instanceof Error ? error.message : String(error);
|
|
66
|
+
}
|
|
67
|
+
export { ProbeTimeout, createProbe, errorText, withTimeout };
|
|
68
|
+
|
|
69
|
+
//# sourceMappingURL=probe.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probe.js","names":[],"sources":["../../../src/metrics/collectors/probe.ts"],"sourcesContent":["import type { MetricSink } from '../metrics.js';\n\nexport class ProbeTimeout extends Error {}\n\nexport async function withTimeout<T>(timeoutMs: number, run: () => Promise<T>): Promise<T> {\n let timer: ReturnType<typeof setTimeout> | undefined;\n try {\n return await Promise.race([\n run(),\n new Promise<never>((_, reject) => {\n timer = setTimeout(() => reject(new ProbeTimeout()), timeoutMs);\n }),\n ]);\n } finally {\n clearTimeout(timer);\n }\n}\n\nexport type ProbeResult = { ok: true } | { ok: false; error: unknown };\n\n/**\n * Components whose round trip is its own signal. The stats probes run over the same pool or\n * Redis client, so their durations would repeat these; only their failures are sent.\n */\nconst TIMED_COMPONENTS = new Set(['db_store', 'db_vector', 'redis']);\n\nexport type Probe = (\n sink: MetricSink,\n component: string,\n timeoutMs: number,\n run: () => Promise<unknown>,\n) => Promise<ProbeResult>;\n\n/**\n * A probe for one collector. It times `run` and emits `mastra_factory_probe_duration_ms{component}`\n * on success (only for the components in {@link TIMED_COMPONENTS}), or\n * `mastra_factory_probe_failures_total{component,reason}` when it throws or exceeds\n * `timeoutMs`. The timeout only stops waiting, it can't cancel the call, so a component whose call\n * hasn't settled isn't checked again until it returns: a hung dependency holds at most one call\n * (and one pooled connection) per component. Each collector has its own guard.\n */\nexport function createProbe(): Probe {\n const stillRunning = new Set<string>();\n return async (sink, component, timeoutMs, run) => {\n const timed = TIMED_COMPONENTS.has(component);\n if (stillRunning.has(component)) {\n sink.emit('mastra_factory_probe_failures_total', 1, { component, reason: 'timeout' });\n return { ok: false, error: new ProbeTimeout() };\n }\n const start = performance.now();\n stillRunning.add(component);\n // A synchronous throw becomes a rejection, so the component is always released.\n const call = Promise.resolve()\n .then(run)\n .finally(() => stillRunning.delete(component));\n try {\n await withTimeout(timeoutMs, () => call);\n if (timed) sink.emit('mastra_factory_probe_duration_ms', performance.now() - start, { component });\n return { ok: true };\n } catch (error) {\n sink.emit('mastra_factory_probe_failures_total', 1, {\n component,\n reason: error instanceof ProbeTimeout ? 'timeout' : 'error',\n });\n return { ok: false, error };\n }\n };\n}\n\nexport function errorText(error: unknown): string {\n if (error instanceof ProbeTimeout) return 'timeout';\n return error instanceof Error ? error.message : String(error);\n}\n"],"mappings":";AAEA,IAAa,eAAb,cAAkC,MAAM,CAAC;AAEzC,eAAsB,YAAe,WAAmB,KAAmC;CACzF,IAAI;CACJ,IAAI;EACF,OAAO,MAAM,QAAQ,KAAK,CACxB,IAAI,GACJ,IAAI,SAAgB,GAAG,WAAW;GAChC,QAAQ,iBAAiB,OAAO,IAAI,aAAa,CAAC,GAAG,SAAS;EAChE,CAAC,CACH,CAAC;CACH,UAAU;EACR,aAAa,KAAK;CACpB;AACF;;;;;AAQA,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAY;CAAa;AAAO,CAAC;;;;;;;;;AAiBnE,SAAgB,cAAqB;CACnC,MAAM,+BAAe,IAAI,IAAY;CACrC,OAAO,OAAO,MAAM,WAAW,WAAW,QAAQ;EAChD,MAAM,QAAQ,iBAAiB,IAAI,SAAS;EAC5C,IAAI,aAAa,IAAI,SAAS,GAAG;GAC/B,KAAK,KAAK,uCAAuC,GAAG;IAAE;IAAW,QAAQ;GAAU,CAAC;GACpF,OAAO;IAAE,IAAI;IAAO,OAAO,IAAI,aAAa;GAAE;EAChD;EACA,MAAM,QAAQ,YAAY,IAAI;EAC9B,aAAa,IAAI,SAAS;EAE1B,MAAM,OAAO,QAAQ,QAAQ,CAAC,CAC3B,KAAK,GAAG,CAAC,CACT,cAAc,aAAa,OAAO,SAAS,CAAC;EAC/C,IAAI;GACF,MAAM,YAAY,iBAAiB,IAAI;GACvC,IAAI,OAAO,KAAK,KAAK,oCAAoC,YAAY,IAAI,IAAI,OAAO,EAAE,UAAU,CAAC;GACjG,OAAO,EAAE,IAAI,KAAK;EACpB,SAAS,OAAO;GACd,KAAK,KAAK,uCAAuC,GAAG;IAClD;IACA,QAAQ,iBAAiB,eAAe,YAAY;GACtD,CAAC;GACD,OAAO;IAAE,IAAI;IAAO;GAAM;EAC5B;CACF;AACF;AAEA,SAAgB,UAAU,OAAwB;CAChD,IAAI,iBAAiB,cAAc,OAAO;CAC1C,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { MetricsCollector } from '../metrics.js';
|
|
2
|
+
/**
|
|
3
|
+
* Cores this process can use: the container's CPU limit when one is set (cgroup v2 `cpu.max`, or
|
|
4
|
+
* v1 `cpu.cfs_quota_us` / `cpu.cfs_period_us`), otherwise the cores the machine reports.
|
|
5
|
+
*/
|
|
6
|
+
export declare function cpuLimitCores(): number;
|
|
7
|
+
/**
|
|
8
|
+
* Event-loop delay and utilization, memory against the heap limit, CPU against the CPU limit, and
|
|
9
|
+
* uptime of this Node process. Each major or incremental GC pause is passed to `onGcPause` as it
|
|
10
|
+
* happens, to be aggregated per interval.
|
|
11
|
+
*/
|
|
12
|
+
export declare function processCollector(onGcPause?: (kind: string, ms: number) => void): MetricsCollector;
|
|
13
|
+
//# sourceMappingURL=process.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"process.d.ts","sourceRoot":"","sources":["../../../src/metrics/collectors/process.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAYtD;;;GAGG;AACH,wBAAgB,aAAa,IAAI,MAAM,CAOtC;AASD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,IAAI,GAAG,gBAAgB,CA0DjG"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { readFileSync } from "fs";
|
|
2
|
+
import { availableParallelism } from "os";
|
|
3
|
+
import { PerformanceObserver, constants, monitorEventLoopDelay } from "perf_hooks";
|
|
4
|
+
import { getHeapStatistics } from "v8";
|
|
5
|
+
// src/metrics/collectors/process.ts
|
|
6
|
+
const LOOP_RESOLUTION_MS = 20;
|
|
7
|
+
function readNumbers(path) {
|
|
8
|
+
try {
|
|
9
|
+
return readFileSync(path, "utf8").trim().split(/\s+/);
|
|
10
|
+
} catch {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Cores this process can use: the container's CPU limit when one is set (cgroup v2 `cpu.max`, or
|
|
16
|
+
* v1 `cpu.cfs_quota_us` / `cpu.cfs_period_us`), otherwise the cores the machine reports.
|
|
17
|
+
*/
|
|
18
|
+
function cpuLimitCores() {
|
|
19
|
+
const parallelism = availableParallelism();
|
|
20
|
+
const v2 = readNumbers("/sys/fs/cgroup/cpu.max");
|
|
21
|
+
const quota = v2 ? Number(v2[0]) : Number(readNumbers("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")?.[0]);
|
|
22
|
+
const period = v2 ? Number(v2[1]) : Number(readNumbers("/sys/fs/cgroup/cpu/cpu.cfs_period_us")?.[0]);
|
|
23
|
+
if (!(quota > 0 && period > 0)) return parallelism;
|
|
24
|
+
return Math.min(quota / period, parallelism);
|
|
25
|
+
}
|
|
26
|
+
const GC_KINDS = {
|
|
27
|
+
[constants.NODE_PERFORMANCE_GC_MAJOR]: "major",
|
|
28
|
+
[constants.NODE_PERFORMANCE_GC_INCREMENTAL]: "incremental"
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Event-loop delay and utilization, memory against the heap limit, CPU against the CPU limit, and
|
|
32
|
+
* uptime of this Node process. Each major or incremental GC pause is passed to `onGcPause` as it
|
|
33
|
+
* happens, to be aggregated per interval.
|
|
34
|
+
*/
|
|
35
|
+
function processCollector(onGcPause) {
|
|
36
|
+
const cpuLimit = cpuLimitCores();
|
|
37
|
+
let lastUtilization = performance.eventLoopUtilization();
|
|
38
|
+
const loop = monitorEventLoopDelay({ resolution: LOOP_RESOLUTION_MS });
|
|
39
|
+
loop.enable();
|
|
40
|
+
let gc;
|
|
41
|
+
if (onGcPause) {
|
|
42
|
+
gc = new PerformanceObserver((list) => {
|
|
43
|
+
for (const entry of list.getEntries()) {
|
|
44
|
+
const kind = GC_KINDS[entry.detail?.kind ?? -1];
|
|
45
|
+
if (kind) onGcPause(kind, entry.duration);
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
gc.observe({ entryTypes: ["gc"] });
|
|
49
|
+
}
|
|
50
|
+
let lastCpu = process.cpuUsage();
|
|
51
|
+
let lastAt = performance.now();
|
|
52
|
+
return {
|
|
53
|
+
collect(sink) {
|
|
54
|
+
const delayMs = (ns) => Math.max(0, ns / 1e6 - LOOP_RESOLUTION_MS);
|
|
55
|
+
sink.emit("mastra_factory_event_loop_delay_ms_count", loop.count);
|
|
56
|
+
sink.emit("mastra_factory_event_loop_delay_ms", delayMs(loop.mean), { stat: "avg" });
|
|
57
|
+
sink.emit("mastra_factory_event_loop_delay_ms", delayMs(loop.percentile(95)), { stat: "p95" });
|
|
58
|
+
sink.emit("mastra_factory_event_loop_delay_ms", delayMs(loop.max), { stat: "max" });
|
|
59
|
+
loop.reset();
|
|
60
|
+
const utilization = performance.eventLoopUtilization(lastUtilization);
|
|
61
|
+
lastUtilization = performance.eventLoopUtilization();
|
|
62
|
+
sink.emit("mastra_factory_event_loop_utilization", utilization.utilization);
|
|
63
|
+
const memory = process.memoryUsage();
|
|
64
|
+
sink.emit("mastra_factory_process_memory_bytes", memory.rss, { kind: "rss" });
|
|
65
|
+
sink.emit("mastra_factory_process_memory_bytes", memory.heapUsed, { kind: "heap_used" });
|
|
66
|
+
const heapLimit = getHeapStatistics().heap_size_limit;
|
|
67
|
+
sink.emit("mastra_factory_process_memory_bytes", heapLimit, { kind: "heap_limit" });
|
|
68
|
+
sink.emit("mastra_factory_process_heap_utilization", memory.heapUsed / heapLimit);
|
|
69
|
+
const cpu = process.cpuUsage(lastCpu);
|
|
70
|
+
const now = performance.now();
|
|
71
|
+
sink.emit("mastra_factory_process_cpu_cores", (cpu.user + cpu.system) / 1e3 / (now - lastAt));
|
|
72
|
+
lastCpu = process.cpuUsage();
|
|
73
|
+
lastAt = now;
|
|
74
|
+
sink.emit("mastra_factory_process_cpu_limit_cores", cpuLimit);
|
|
75
|
+
sink.emit("mastra_factory_process_uptime_ms", process.uptime() * 1e3);
|
|
76
|
+
},
|
|
77
|
+
stop() {
|
|
78
|
+
loop.disable();
|
|
79
|
+
gc?.disconnect();
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export { cpuLimitCores, processCollector };
|
|
84
|
+
|
|
85
|
+
//# sourceMappingURL=process.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"process.js","names":[],"sources":["../../../src/metrics/collectors/process.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { availableParallelism } from 'node:os';\nimport { constants, monitorEventLoopDelay, PerformanceObserver } from 'node:perf_hooks';\nimport type { PerformanceEntry } from 'node:perf_hooks';\nimport { getHeapStatistics } from 'node:v8';\n\nimport type { MetricsCollector } from '../metrics.js';\n\nconst LOOP_RESOLUTION_MS = 20;\n\nfunction readNumbers(path: string): string[] | undefined {\n try {\n return readFileSync(path, 'utf8').trim().split(/\\s+/);\n } catch {\n return undefined;\n }\n}\n\n/**\n * Cores this process can use: the container's CPU limit when one is set (cgroup v2 `cpu.max`, or\n * v1 `cpu.cfs_quota_us` / `cpu.cfs_period_us`), otherwise the cores the machine reports.\n */\nexport function cpuLimitCores(): number {\n const parallelism = availableParallelism();\n const v2 = readNumbers('/sys/fs/cgroup/cpu.max');\n const quota = v2 ? Number(v2[0]) : Number(readNumbers('/sys/fs/cgroup/cpu/cpu.cfs_quota_us')?.[0]);\n const period = v2 ? Number(v2[1]) : Number(readNumbers('/sys/fs/cgroup/cpu/cpu.cfs_period_us')?.[0]);\n if (!(quota > 0 && period > 0)) return parallelism;\n return Math.min(quota / period, parallelism);\n}\n\n// Only the pauses worth a series: minor pauses already show in event-loop delay, and weak-callback\n// passes are noise.\nconst GC_KINDS: Record<number, string> = {\n [constants.NODE_PERFORMANCE_GC_MAJOR]: 'major',\n [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental',\n};\n\n/**\n * Event-loop delay and utilization, memory against the heap limit, CPU against the CPU limit, and\n * uptime of this Node process. Each major or incremental GC pause is passed to `onGcPause` as it\n * happens, to be aggregated per interval.\n */\nexport function processCollector(onGcPause?: (kind: string, ms: number) => void): MetricsCollector {\n const cpuLimit = cpuLimitCores();\n let lastUtilization = performance.eventLoopUtilization();\n const loop = monitorEventLoopDelay({ resolution: LOOP_RESOLUTION_MS });\n loop.enable();\n let gc: PerformanceObserver | undefined;\n if (onGcPause) {\n gc = new PerformanceObserver(list => {\n for (const entry of list.getEntries()) {\n const kind = GC_KINDS[(entry as PerformanceEntry & { detail?: { kind?: number } }).detail?.kind ?? -1];\n if (kind) onGcPause(kind, entry.duration);\n }\n });\n gc.observe({ entryTypes: ['gc'] });\n }\n let lastCpu = process.cpuUsage();\n let lastAt = performance.now();\n return {\n collect(sink) {\n // The histogram records each sampling timer's full wait, so an idle loop reads as the\n // resolution; subtract it to get the delay.\n const delayMs = (ns: number) => Math.max(0, ns / 1e6 - LOOP_RESOLUTION_MS);\n // Same shape as the other timings: `stat=avg|p95|max` plus a `_count` series (delay samples taken).\n sink.emit('mastra_factory_event_loop_delay_ms_count', loop.count);\n sink.emit('mastra_factory_event_loop_delay_ms', delayMs(loop.mean), { stat: 'avg' });\n sink.emit('mastra_factory_event_loop_delay_ms', delayMs(loop.percentile(95)), { stat: 'p95' });\n sink.emit('mastra_factory_event_loop_delay_ms', delayMs(loop.max), { stat: 'max' });\n loop.reset();\n\n // Share of the interval the main thread spent running JavaScript and callbacks, 0 to 1.\n const utilization = performance.eventLoopUtilization(lastUtilization);\n lastUtilization = performance.eventLoopUtilization();\n sink.emit('mastra_factory_event_loop_utilization', utilization.utilization);\n\n const memory = process.memoryUsage();\n sink.emit('mastra_factory_process_memory_bytes', memory.rss, { kind: 'rss' });\n sink.emit('mastra_factory_process_memory_bytes', memory.heapUsed, { kind: 'heap_used' });\n const heapLimit = getHeapStatistics().heap_size_limit;\n sink.emit('mastra_factory_process_memory_bytes', heapLimit, { kind: 'heap_limit' });\n sink.emit('mastra_factory_process_heap_utilization', memory.heapUsed / heapLimit);\n\n // CPU used by every thread of the process (main thread, GC helpers, the libuv pool, workers)\n // since the last sample, in cores: 1.4 means 1.4 cores' worth, like 140% in top. It can go\n // above 1; compare it to the limit, not to 1. The main thread's load is event-loop utilization.\n const cpu = process.cpuUsage(lastCpu);\n const now = performance.now();\n sink.emit('mastra_factory_process_cpu_cores', (cpu.user + cpu.system) / 1000 / (now - lastAt));\n lastCpu = process.cpuUsage();\n lastAt = now;\n sink.emit('mastra_factory_process_cpu_limit_cores', cpuLimit);\n\n sink.emit('mastra_factory_process_uptime_ms', process.uptime() * 1000);\n },\n stop() {\n loop.disable();\n gc?.disconnect();\n },\n };\n}\n"],"mappings":";;;;;AAQA,MAAM,qBAAqB;AAE3B,SAAS,YAAY,MAAoC;CACvD,IAAI;EACF,OAAO,aAAa,MAAM,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK;CACtD,QAAQ;EACN;CACF;AACF;;;;;AAMA,SAAgB,gBAAwB;CACtC,MAAM,cAAc,qBAAqB;CACzC,MAAM,KAAK,YAAY,wBAAwB;CAC/C,MAAM,QAAQ,KAAK,OAAO,GAAG,EAAE,IAAI,OAAO,YAAY,qCAAqC,CAAC,GAAG,EAAE;CACjG,MAAM,SAAS,KAAK,OAAO,GAAG,EAAE,IAAI,OAAO,YAAY,sCAAsC,CAAC,GAAG,EAAE;CACnG,IAAI,EAAE,QAAQ,KAAK,SAAS,IAAI,OAAO;CACvC,OAAO,KAAK,IAAI,QAAQ,QAAQ,WAAW;AAC7C;AAIA,MAAM,WAAmC;EACtC,UAAU,4BAA4B;EACtC,UAAU,kCAAkC;AAC/C;;;;;;AAOA,SAAgB,iBAAiB,WAAkE;CACjG,MAAM,WAAW,cAAc;CAC/B,IAAI,kBAAkB,YAAY,qBAAqB;CACvD,MAAM,OAAO,sBAAsB,EAAE,YAAY,mBAAmB,CAAC;CACrE,KAAK,OAAO;CACZ,IAAI;CACJ,IAAI,WAAW;EACb,KAAK,IAAI,qBAAoB,SAAQ;GACnC,KAAK,MAAM,SAAS,KAAK,WAAW,GAAG;IACrC,MAAM,OAAO,SAAU,MAA4D,QAAQ,QAAQ;IACnG,IAAI,MAAM,UAAU,MAAM,MAAM,QAAQ;GAC1C;EACF,CAAC;EACD,GAAG,QAAQ,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC;CACnC;CACA,IAAI,UAAU,QAAQ,SAAS;CAC/B,IAAI,SAAS,YAAY,IAAI;CAC7B,OAAO;EACL,QAAQ,MAAM;GAGZ,MAAM,WAAW,OAAe,KAAK,IAAI,GAAG,KAAK,MAAM,kBAAkB;GAEzE,KAAK,KAAK,4CAA4C,KAAK,KAAK;GAChE,KAAK,KAAK,sCAAsC,QAAQ,KAAK,IAAI,GAAG,EAAE,MAAM,MAAM,CAAC;GACnF,KAAK,KAAK,sCAAsC,QAAQ,KAAK,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;GAC7F,KAAK,KAAK,sCAAsC,QAAQ,KAAK,GAAG,GAAG,EAAE,MAAM,MAAM,CAAC;GAClF,KAAK,MAAM;GAGX,MAAM,cAAc,YAAY,qBAAqB,eAAe;GACpE,kBAAkB,YAAY,qBAAqB;GACnD,KAAK,KAAK,yCAAyC,YAAY,WAAW;GAE1E,MAAM,SAAS,QAAQ,YAAY;GACnC,KAAK,KAAK,uCAAuC,OAAO,KAAK,EAAE,MAAM,MAAM,CAAC;GAC5E,KAAK,KAAK,uCAAuC,OAAO,UAAU,EAAE,MAAM,YAAY,CAAC;GACvF,MAAM,YAAY,kBAAkB,CAAC,CAAC;GACtC,KAAK,KAAK,uCAAuC,WAAW,EAAE,MAAM,aAAa,CAAC;GAClF,KAAK,KAAK,2CAA2C,OAAO,WAAW,SAAS;GAKhF,MAAM,MAAM,QAAQ,SAAS,OAAO;GACpC,MAAM,MAAM,YAAY,IAAI;GAC5B,KAAK,KAAK,qCAAqC,IAAI,OAAO,IAAI,UAAU,OAAQ,MAAM,OAAO;GAC7F,UAAU,QAAQ,SAAS;GAC3B,SAAS;GACT,KAAK,KAAK,0CAA0C,QAAQ;GAE5D,KAAK,KAAK,oCAAoC,QAAQ,OAAO,IAAI,GAAI;EACvE;EACA,OAAO;GACL,KAAK,QAAQ;GACb,IAAI,WAAW;EACjB;CACF;AACF"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { MetricsCollector, MetricsLogger } from '../metrics.js';
|
|
2
|
+
/** The commands the collector uses, structurally matching a node-redis client. */
|
|
3
|
+
export interface RedisMonitorClient {
|
|
4
|
+
connect(): Promise<unknown>;
|
|
5
|
+
info(): Promise<string>;
|
|
6
|
+
xInfoGroups(key: string): Promise<Array<{
|
|
7
|
+
name: unknown;
|
|
8
|
+
pending: unknown;
|
|
9
|
+
lag?: unknown;
|
|
10
|
+
}>>;
|
|
11
|
+
on(event: 'error', listener: (error: unknown) => void): unknown;
|
|
12
|
+
destroy?(): void;
|
|
13
|
+
}
|
|
14
|
+
/** What `@mastra/redis-streams`' `RedisStreamsPubSub` offers for monitoring. */
|
|
15
|
+
export interface MonitorablePubSub {
|
|
16
|
+
createMonitorClient(): RedisMonitorClient;
|
|
17
|
+
streamKey(topic: string): string;
|
|
18
|
+
}
|
|
19
|
+
export declare function asMonitorablePubSub(pubsub: unknown): MonitorablePubSub | undefined;
|
|
20
|
+
/** Owns a dedicated monitoring connection, so probes never queue behind pub/sub traffic. */
|
|
21
|
+
export declare function connectMonitor(pubsub: MonitorablePubSub, getLogger: () => MetricsLogger | undefined): RedisMonitorClient;
|
|
22
|
+
/** The `INFO` round trip as the Redis probe, run in every process. */
|
|
23
|
+
export declare function redisProbeCollector(client: RedisMonitorClient, timeoutMs: number): MetricsCollector;
|
|
24
|
+
/**
|
|
25
|
+
* Server-wide Redis stats, collected by one process per deployment: memory against `maxmemory`,
|
|
26
|
+
* evictions, clients, and the backlog of the grouped event streams.
|
|
27
|
+
*/
|
|
28
|
+
export declare function redisStatsCollector(pubsub: MonitorablePubSub, client: RedisMonitorClient, timeoutMs: number): MetricsCollector;
|
|
29
|
+
//# sourceMappingURL=redis.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.d.ts","sourceRoot":"","sources":["../../../src/metrics/collectors/redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGrE,kFAAkF;AAClF,MAAM,WAAW,kBAAkB;IACjC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5B,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IACxB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC;QAAC,GAAG,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC,CAAC;IAC7F,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC;IAChE,OAAO,CAAC,IAAI,IAAI,CAAC;CAClB;AAED,gFAAgF;AAChF,MAAM,WAAW,iBAAiB;IAChC,mBAAmB,IAAI,kBAAkB,CAAC;IAC1C,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;CAClC;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,OAAO,GAAG,iBAAiB,GAAG,SAAS,CAMlF;AAiBD,4FAA4F;AAC5F,wBAAgB,cAAc,CAAC,MAAM,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,GAAG,SAAS,sBAcnG;AAED,sEAAsE;AACtE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,GAAG,gBAAgB,CAOnG;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,iBAAiB,EACzB,MAAM,EAAE,kBAAkB,EAC1B,SAAS,EAAE,MAAM,GAChB,gBAAgB,CA8DlB"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { createProbe } from "./probe.js";
|
|
2
|
+
// src/metrics/collectors/redis.ts
|
|
3
|
+
function asMonitorablePubSub(pubsub) {
|
|
4
|
+
if (!pubsub || typeof pubsub !== "object") return void 0;
|
|
5
|
+
const candidate = pubsub;
|
|
6
|
+
return typeof candidate.createMonitorClient === "function" && typeof candidate.streamKey === "function" ? candidate : void 0;
|
|
7
|
+
}
|
|
8
|
+
const GROUPED_TOPICS = ["workflows", "background-tasks"];
|
|
9
|
+
const FANOUT_GROUP_PREFIX = "__fanout-";
|
|
10
|
+
function parseInfo(text) {
|
|
11
|
+
const fields = /* @__PURE__ */ new Map();
|
|
12
|
+
for (const line of text.split("\r\n")) {
|
|
13
|
+
const colon = line.indexOf(":");
|
|
14
|
+
if (colon > 0 && !line.startsWith("#")) fields.set(line.slice(0, colon), line.slice(colon + 1));
|
|
15
|
+
}
|
|
16
|
+
return fields;
|
|
17
|
+
}
|
|
18
|
+
/** Owns a dedicated monitoring connection, so probes never queue behind pub/sub traffic. */
|
|
19
|
+
function connectMonitor(pubsub, getLogger) {
|
|
20
|
+
const client = pubsub.createMonitorClient();
|
|
21
|
+
let lastErrorLogAt = 0;
|
|
22
|
+
client.on("error", (error) => {
|
|
23
|
+
if (Date.now() - lastErrorLogAt < 6e4) return;
|
|
24
|
+
lastErrorLogAt = Date.now();
|
|
25
|
+
getLogger()?.warn("[factory:metrics] Redis monitor client error.", { error: error instanceof Error ? error.message : String(error) });
|
|
26
|
+
});
|
|
27
|
+
client.connect().catch(() => {});
|
|
28
|
+
return client;
|
|
29
|
+
}
|
|
30
|
+
/** The `INFO` round trip as the Redis probe, run in every process. */
|
|
31
|
+
function redisProbeCollector(client, timeoutMs) {
|
|
32
|
+
const probe = createProbe();
|
|
33
|
+
return { async collect(sink) {
|
|
34
|
+
await probe(sink, "redis", timeoutMs, () => client.info());
|
|
35
|
+
} };
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Server-wide Redis stats, collected by one process per deployment: memory against `maxmemory`,
|
|
39
|
+
* evictions, clients, and the backlog of the grouped event streams.
|
|
40
|
+
*/
|
|
41
|
+
function redisStatsCollector(pubsub, client, timeoutMs) {
|
|
42
|
+
const probe = createProbe();
|
|
43
|
+
let lastEvicted;
|
|
44
|
+
return { async collect(sink) {
|
|
45
|
+
let info = "";
|
|
46
|
+
if (!(await probe(sink, "redis_stats", timeoutMs, async () => {
|
|
47
|
+
info = await client.info();
|
|
48
|
+
})).ok) return;
|
|
49
|
+
const fields = parseInfo(info);
|
|
50
|
+
const number = (key) => Number(fields.get(key));
|
|
51
|
+
sink.emit("mastra_factory_redis_memory_bytes", number("used_memory"), { kind: "used" });
|
|
52
|
+
const maxMemory = number("maxmemory");
|
|
53
|
+
if (maxMemory > 0) {
|
|
54
|
+
sink.emit("mastra_factory_redis_memory_bytes", maxMemory, { kind: "max" });
|
|
55
|
+
sink.emit("mastra_factory_redis_memory_utilization", number("used_memory") / maxMemory);
|
|
56
|
+
}
|
|
57
|
+
sink.emit("mastra_factory_redis_clients", number("connected_clients"));
|
|
58
|
+
const evicted = number("evicted_keys");
|
|
59
|
+
if (Number.isFinite(evicted)) {
|
|
60
|
+
if (lastEvicted !== void 0) sink.emit("mastra_factory_redis_evicted_keys_total", evicted >= lastEvicted ? evicted - lastEvicted : evicted);
|
|
61
|
+
lastEvicted = evicted;
|
|
62
|
+
}
|
|
63
|
+
const backlogs = [];
|
|
64
|
+
await probe(sink, "redis_streams", timeoutMs, () => Promise.all(GROUPED_TOPICS.map(async (topic) => {
|
|
65
|
+
const groups = await client.xInfoGroups(pubsub.streamKey(topic)).catch((error) => {
|
|
66
|
+
if (/no such key/i.test(error instanceof Error ? error.message : String(error))) return void 0;
|
|
67
|
+
throw error;
|
|
68
|
+
});
|
|
69
|
+
if (!groups) return;
|
|
70
|
+
let pending = 0;
|
|
71
|
+
let lag = 0;
|
|
72
|
+
for (const group of groups) {
|
|
73
|
+
if (String(group.name).startsWith(FANOUT_GROUP_PREFIX)) continue;
|
|
74
|
+
pending += Number(group.pending);
|
|
75
|
+
lag = group.lag == null || lag === void 0 ? void 0 : lag + Number(group.lag);
|
|
76
|
+
}
|
|
77
|
+
backlogs.push({
|
|
78
|
+
topic,
|
|
79
|
+
pending,
|
|
80
|
+
lag
|
|
81
|
+
});
|
|
82
|
+
})));
|
|
83
|
+
for (const { topic, pending, lag } of backlogs) {
|
|
84
|
+
sink.emit("mastra_factory_redis_stream_backlog", pending, {
|
|
85
|
+
topic,
|
|
86
|
+
kind: "pending"
|
|
87
|
+
});
|
|
88
|
+
if (lag !== void 0) sink.emit("mastra_factory_redis_stream_backlog", lag, {
|
|
89
|
+
topic,
|
|
90
|
+
kind: "lag"
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
} };
|
|
94
|
+
}
|
|
95
|
+
export { asMonitorablePubSub, connectMonitor, redisProbeCollector, redisStatsCollector };
|
|
96
|
+
|
|
97
|
+
//# sourceMappingURL=redis.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.js","names":[],"sources":["../../../src/metrics/collectors/redis.ts"],"sourcesContent":["import type { MetricsCollector, MetricsLogger } from '../metrics.js';\nimport { createProbe } from './probe.js';\n\n/** The commands the collector uses, structurally matching a node-redis client. */\nexport interface RedisMonitorClient {\n connect(): Promise<unknown>;\n info(): Promise<string>;\n xInfoGroups(key: string): Promise<Array<{ name: unknown; pending: unknown; lag?: unknown }>>;\n on(event: 'error', listener: (error: unknown) => void): unknown;\n destroy?(): void;\n}\n\n/** What `@mastra/redis-streams`' `RedisStreamsPubSub` offers for monitoring. */\nexport interface MonitorablePubSub {\n createMonitorClient(): RedisMonitorClient;\n streamKey(topic: string): string;\n}\n\nexport function asMonitorablePubSub(pubsub: unknown): MonitorablePubSub | undefined {\n if (!pubsub || typeof pubsub !== 'object') return undefined;\n const candidate = pubsub as Partial<MonitorablePubSub>;\n return typeof candidate.createMonitorClient === 'function' && typeof candidate.streamKey === 'function'\n ? (candidate as MonitorablePubSub)\n : undefined;\n}\n\n// Topics read through named consumer groups (workflow orchestration and background tasks). Their\n// backlog is work that was published but not processed yet. Fan-out subscriptions use throwaway\n// groups and are skipped.\nconst GROUPED_TOPICS = ['workflows', 'background-tasks'];\nconst FANOUT_GROUP_PREFIX = '__fanout-';\n\nfunction parseInfo(text: string): Map<string, string> {\n const fields = new Map<string, string>();\n for (const line of text.split('\\r\\n')) {\n const colon = line.indexOf(':');\n if (colon > 0 && !line.startsWith('#')) fields.set(line.slice(0, colon), line.slice(colon + 1));\n }\n return fields;\n}\n\n/** Owns a dedicated monitoring connection, so probes never queue behind pub/sub traffic. */\nexport function connectMonitor(pubsub: MonitorablePubSub, getLogger: () => MetricsLogger | undefined) {\n const client = pubsub.createMonitorClient();\n let lastErrorLogAt = 0;\n client.on('error', error => {\n if (Date.now() - lastErrorLogAt < 60_000) return;\n lastErrorLogAt = Date.now();\n getLogger()?.warn('[factory:metrics] Redis monitor client error.', {\n error: error instanceof Error ? error.message : String(error),\n });\n });\n void client.connect().catch(() => {\n // Reported through the 'error' listener; node-redis keeps reconnecting.\n });\n return client;\n}\n\n/** The `INFO` round trip as the Redis probe, run in every process. */\nexport function redisProbeCollector(client: RedisMonitorClient, timeoutMs: number): MetricsCollector {\n const probe = createProbe();\n return {\n async collect(sink) {\n await probe(sink, 'redis', timeoutMs, () => client.info());\n },\n };\n}\n\n/**\n * Server-wide Redis stats, collected by one process per deployment: memory against `maxmemory`,\n * evictions, clients, and the backlog of the grouped event streams.\n */\nexport function redisStatsCollector(\n pubsub: MonitorablePubSub,\n client: RedisMonitorClient,\n timeoutMs: number,\n): MetricsCollector {\n const probe = createProbe();\n let lastEvicted: number | undefined;\n return {\n async collect(sink) {\n let info = '';\n const result = await probe(sink, 'redis_stats', timeoutMs, async () => {\n info = await client.info();\n });\n if (!result.ok) return;\n\n const fields = parseInfo(info);\n const number = (key: string) => Number(fields.get(key));\n sink.emit('mastra_factory_redis_memory_bytes', number('used_memory'), { kind: 'used' });\n const maxMemory = number('maxmemory');\n // `maxmemory` is 0 when Redis has no memory limit; then neither the limit nor the ratio is sent.\n if (maxMemory > 0) {\n sink.emit('mastra_factory_redis_memory_bytes', maxMemory, { kind: 'max' });\n sink.emit('mastra_factory_redis_memory_utilization', number('used_memory') / maxMemory);\n }\n sink.emit('mastra_factory_redis_clients', number('connected_clients'));\n // INFO reports evictions since the Redis server started; send the increase per interval.\n const evicted = number('evicted_keys');\n if (Number.isFinite(evicted)) {\n if (lastEvicted !== undefined) {\n sink.emit(\n 'mastra_factory_redis_evicted_keys_total',\n evicted >= lastEvicted ? evicted - lastEvicted : evicted,\n );\n }\n lastEvicted = evicted;\n }\n\n // Bounded like every other call: a Redis that answers INFO and then stalls must not hold up the tick.\n const backlogs: Array<{ topic: string; pending: number; lag: number | undefined }> = [];\n await probe(sink, 'redis_streams', timeoutMs, () =>\n Promise.all(\n GROUPED_TOPICS.map(async topic => {\n // A missing stream (nothing published yet) is not a failure; anything else is.\n const groups = await client.xInfoGroups(pubsub.streamKey(topic)).catch((error: unknown) => {\n if (/no such key/i.test(error instanceof Error ? error.message : String(error))) return undefined;\n throw error;\n });\n if (!groups) return;\n let pending = 0;\n // Redis reports lag as null when it can't compute it; then the topic gets no lag record.\n let lag: number | undefined = 0;\n for (const group of groups) {\n if (String(group.name).startsWith(FANOUT_GROUP_PREFIX)) continue;\n pending += Number(group.pending);\n lag = group.lag == null || lag === undefined ? undefined : lag + Number(group.lag);\n }\n backlogs.push({ topic, pending, lag });\n }),\n ),\n );\n for (const { topic, pending, lag } of backlogs) {\n sink.emit('mastra_factory_redis_stream_backlog', pending, { topic, kind: 'pending' });\n if (lag !== undefined) sink.emit('mastra_factory_redis_stream_backlog', lag, { topic, kind: 'lag' });\n }\n },\n };\n}\n"],"mappings":";;AAkBA,SAAgB,oBAAoB,QAAgD;CAClF,IAAI,CAAC,UAAU,OAAO,WAAW,UAAU,OAAO,KAAA;CAClD,MAAM,YAAY;CAClB,OAAO,OAAO,UAAU,wBAAwB,cAAc,OAAO,UAAU,cAAc,aACxF,YACD,KAAA;AACN;AAKA,MAAM,iBAAiB,CAAC,aAAa,kBAAkB;AACvD,MAAM,sBAAsB;AAE5B,SAAS,UAAU,MAAmC;CACpD,MAAM,yBAAS,IAAI,IAAoB;CACvC,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM,GAAG;EACrC,MAAM,QAAQ,KAAK,QAAQ,GAAG;EAC9B,IAAI,QAAQ,KAAK,CAAC,KAAK,WAAW,GAAG,GAAG,OAAO,IAAI,KAAK,MAAM,GAAG,KAAK,GAAG,KAAK,MAAM,QAAQ,CAAC,CAAC;CAChG;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,QAA2B,WAA4C;CACpG,MAAM,SAAS,OAAO,oBAAoB;CAC1C,IAAI,iBAAiB;CACrB,OAAO,GAAG,UAAS,UAAS;EAC1B,IAAI,KAAK,IAAI,IAAI,iBAAiB,KAAQ;EAC1C,iBAAiB,KAAK,IAAI;EAC1B,UAAU,CAAC,EAAE,KAAK,iDAAiD,EACjE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAC9D,CAAC;CACH,CAAC;CACD,OAAY,QAAQ,CAAC,CAAC,YAAY,CAElC,CAAC;CACD,OAAO;AACT;;AAGA,SAAgB,oBAAoB,QAA4B,WAAqC;CACnG,MAAM,QAAQ,YAAY;CAC1B,OAAO,EACL,MAAM,QAAQ,MAAM;EAClB,MAAM,MAAM,MAAM,SAAS,iBAAiB,OAAO,KAAK,CAAC;CAC3D,EACF;AACF;;;;;AAMA,SAAgB,oBACd,QACA,QACA,WACkB;CAClB,MAAM,QAAQ,YAAY;CAC1B,IAAI;CACJ,OAAO,EACL,MAAM,QAAQ,MAAM;EAClB,IAAI,OAAO;EAIX,IAAI,EAAC,MAHgB,MAAM,MAAM,eAAe,WAAW,YAAY;GACrE,OAAO,MAAM,OAAO,KAAK;EAC3B,CAAC,EAAA,CACW,IAAI;EAEhB,MAAM,SAAS,UAAU,IAAI;EAC7B,MAAM,UAAU,QAAgB,OAAO,OAAO,IAAI,GAAG,CAAC;EACtD,KAAK,KAAK,qCAAqC,OAAO,aAAa,GAAG,EAAE,MAAM,OAAO,CAAC;EACtF,MAAM,YAAY,OAAO,WAAW;EAEpC,IAAI,YAAY,GAAG;GACjB,KAAK,KAAK,qCAAqC,WAAW,EAAE,MAAM,MAAM,CAAC;GACzE,KAAK,KAAK,2CAA2C,OAAO,aAAa,IAAI,SAAS;EACxF;EACA,KAAK,KAAK,gCAAgC,OAAO,mBAAmB,CAAC;EAErE,MAAM,UAAU,OAAO,cAAc;EACrC,IAAI,OAAO,SAAS,OAAO,GAAG;GAC5B,IAAI,gBAAgB,KAAA,GAClB,KAAK,KACH,2CACA,WAAW,cAAc,UAAU,cAAc,OACnD;GAEF,cAAc;EAChB;EAGA,MAAM,WAA+E,CAAC;EACtF,MAAM,MAAM,MAAM,iBAAiB,iBACjC,QAAQ,IACN,eAAe,IAAI,OAAM,UAAS;GAEhC,MAAM,SAAS,MAAM,OAAO,YAAY,OAAO,UAAU,KAAK,CAAC,CAAC,CAAC,OAAO,UAAmB;IACzF,IAAI,eAAe,KAAK,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC,GAAG,OAAO,KAAA;IACxF,MAAM;GACR,CAAC;GACD,IAAI,CAAC,QAAQ;GACb,IAAI,UAAU;GAEd,IAAI,MAA0B;GAC9B,KAAK,MAAM,SAAS,QAAQ;IAC1B,IAAI,OAAO,MAAM,IAAI,CAAC,CAAC,WAAW,mBAAmB,GAAG;IACxD,WAAW,OAAO,MAAM,OAAO;IAC/B,MAAM,MAAM,OAAO,QAAQ,QAAQ,KAAA,IAAY,KAAA,IAAY,MAAM,OAAO,MAAM,GAAG;GACnF;GACA,SAAS,KAAK;IAAE;IAAO;IAAS;GAAI,CAAC;EACvC,CAAC,CACH,CACF;EACA,KAAK,MAAM,EAAE,OAAO,SAAS,SAAS,UAAU;GAC9C,KAAK,KAAK,uCAAuC,SAAS;IAAE;IAAO,MAAM;GAAU,CAAC;GACpF,IAAI,QAAQ,KAAA,GAAW,KAAK,KAAK,uCAAuC,KAAK;IAAE;IAAO,MAAM;GAAM,CAAC;EACrG;CACF,EACF;AACF"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Metric groups a deployment can turn on, one per component to investigate. */
|
|
2
|
+
export declare const METRIC_GROUPS: readonly ['process', 'postgres', 'redis', 'sandbox', 'intake'];
|
|
3
|
+
export type MetricGroup = (typeof METRIC_GROUPS)[number];
|
|
4
|
+
export interface FactoryMetricsOptions {
|
|
5
|
+
/** Groups to collect, or `'all'`. An empty list turns metrics off. */
|
|
6
|
+
groups: readonly MetricGroup[] | 'all';
|
|
7
|
+
/** How often gauges are sampled and aggregated timings are sent. Default 60s, minimum 15s. */
|
|
8
|
+
intervalMs?: number;
|
|
9
|
+
/** Most records sent per minute per process; the rest are dropped and counted. Default 500. */
|
|
10
|
+
maxRecordsPerMinute?: number;
|
|
11
|
+
}
|
|
12
|
+
export interface ResolvedMetricsConfig {
|
|
13
|
+
groups: ReadonlySet<MetricGroup>;
|
|
14
|
+
intervalMs: number;
|
|
15
|
+
maxPerMinute: number;
|
|
16
|
+
/** Group names that were asked for but don't exist. */
|
|
17
|
+
unknownGroups: string[];
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Resolves which metric groups are on. Every record counts towards the deployment's observability
|
|
21
|
+
* usage, so nothing is collected unless a group is asked for. Env vars override the code option:
|
|
22
|
+
* `FACTORY_METRICS` (`all`, `0`, or a list like `sandbox,intake`), `FACTORY_METRICS_INTERVAL_MS`
|
|
23
|
+
* and `FACTORY_METRICS_MAX_PER_MIN`. Returns `undefined` when no group is on.
|
|
24
|
+
*/
|
|
25
|
+
export declare function resolveMetricsConfig(option: FactoryMetricsOptions | undefined, env?: NodeJS.ProcessEnv): ResolvedMetricsConfig | undefined;
|
|
26
|
+
export type MetricsService = 'api' | 'worker' | 'all';
|
|
27
|
+
/**
|
|
28
|
+
* Which part of a split deployment this process is. Platform deploys a separate worker service
|
|
29
|
+
* when the app has storage and pubsub: the API container runs with `MASTRA_WORKERS=false`, the
|
|
30
|
+
* worker container with it set (empty, or a list of worker names). Unset means one process does both.
|
|
31
|
+
*/
|
|
32
|
+
export declare function metricsService(env?: NodeJS.ProcessEnv): MetricsService;
|
|
33
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/metrics/config.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,eAAO,MAAM,aAAa,YAAI,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAU,CAAC;AAC5F,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzD,MAAM,WAAW,qBAAqB;IACpC,sEAAsE;IACtE,MAAM,EAAE,SAAS,WAAW,EAAE,GAAG,KAAK,CAAC;IACvC,8FAA8F;IAC9F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+FAA+F;IAC/F,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAmBD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,qBAAqB,GAAG,SAAS,EACzC,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,qBAAqB,GAAG,SAAS,CA2BnC;AAED,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,QAAQ,GAAG,KAAK,CAAC;AAEtD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,cAAc,CAInF"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// src/metrics/config.ts
|
|
2
|
+
/** Metric groups a deployment can turn on, one per component to investigate. */
|
|
3
|
+
const METRIC_GROUPS = [
|
|
4
|
+
"process",
|
|
5
|
+
"postgres",
|
|
6
|
+
"redis",
|
|
7
|
+
"sandbox",
|
|
8
|
+
"intake"
|
|
9
|
+
];
|
|
10
|
+
const DEFAULT_INTERVAL_MS = 6e4;
|
|
11
|
+
const MIN_INTERVAL_MS = 15e3;
|
|
12
|
+
const MAX_INTERVAL_MS = 2147483647;
|
|
13
|
+
const DEFAULT_MAX_PER_MINUTE = 500;
|
|
14
|
+
function parseGroups(value) {
|
|
15
|
+
if (value === "all") return {
|
|
16
|
+
groups: new Set(METRIC_GROUPS),
|
|
17
|
+
unknownGroups: []
|
|
18
|
+
};
|
|
19
|
+
const groups = /* @__PURE__ */ new Set();
|
|
20
|
+
const unknownGroups = [];
|
|
21
|
+
for (const name of value) if (METRIC_GROUPS.includes(name)) groups.add(name);
|
|
22
|
+
else unknownGroups.push(name);
|
|
23
|
+
return {
|
|
24
|
+
groups,
|
|
25
|
+
unknownGroups
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Resolves which metric groups are on. Every record counts towards the deployment's observability
|
|
30
|
+
* usage, so nothing is collected unless a group is asked for. Env vars override the code option:
|
|
31
|
+
* `FACTORY_METRICS` (`all`, `0`, or a list like `sandbox,intake`), `FACTORY_METRICS_INTERVAL_MS`
|
|
32
|
+
* and `FACTORY_METRICS_MAX_PER_MIN`. Returns `undefined` when no group is on.
|
|
33
|
+
*/
|
|
34
|
+
function resolveMetricsConfig(option, env = process.env) {
|
|
35
|
+
const envGroups = env.FACTORY_METRICS?.trim();
|
|
36
|
+
let requested = option?.groups;
|
|
37
|
+
if (envGroups) {
|
|
38
|
+
if (/^(0|false|no|off)$/i.test(envGroups)) return void 0;
|
|
39
|
+
requested = /^(all|1|true|yes)$/i.test(envGroups) ? "all" : envGroups.split(",").map((name) => name.trim()).filter(Boolean);
|
|
40
|
+
}
|
|
41
|
+
if (!requested) return void 0;
|
|
42
|
+
const { groups, unknownGroups } = parseGroups(requested);
|
|
43
|
+
if (groups.size === 0 && unknownGroups.length === 0) return void 0;
|
|
44
|
+
const interval = Number(env.FACTORY_METRICS_INTERVAL_MS || option?.intervalMs);
|
|
45
|
+
const maxPerMinute = Number(env.FACTORY_METRICS_MAX_PER_MIN || option?.maxRecordsPerMinute);
|
|
46
|
+
return {
|
|
47
|
+
groups,
|
|
48
|
+
intervalMs: Number.isFinite(interval) && interval > 0 ? Math.min(Math.max(interval, MIN_INTERVAL_MS), MAX_INTERVAL_MS) : DEFAULT_INTERVAL_MS,
|
|
49
|
+
maxPerMinute: Number.isInteger(maxPerMinute) && maxPerMinute > 0 ? maxPerMinute : DEFAULT_MAX_PER_MINUTE,
|
|
50
|
+
unknownGroups
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Which part of a split deployment this process is. Platform deploys a separate worker service
|
|
55
|
+
* when the app has storage and pubsub: the API container runs with `MASTRA_WORKERS=false`, the
|
|
56
|
+
* worker container with it set (empty, or a list of worker names). Unset means one process does both.
|
|
57
|
+
*/
|
|
58
|
+
function metricsService(env = process.env) {
|
|
59
|
+
const workers = env.MASTRA_WORKERS;
|
|
60
|
+
if (workers === void 0) return "all";
|
|
61
|
+
return workers.trim() === "false" ? "api" : "worker";
|
|
62
|
+
}
|
|
63
|
+
export { METRIC_GROUPS, metricsService, resolveMetricsConfig };
|
|
64
|
+
|
|
65
|
+
//# sourceMappingURL=config.js.map
|