@getstrata/core 2.2.6 → 2.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/core/database/types.d.ts +2 -0
- package/dist/core/events/outbox/index.d.ts +2 -0
- package/dist/core/events/outbox/metrics.d.ts +27 -0
- package/dist/core/queue/queueMetrics.d.ts +28 -0
- package/dist/entries/database/query.js +12 -2
- package/dist/entries/database/repositoryQuery.js +12 -2
- package/dist/entries/queue/queueMetrics.js +128 -1
- package/dist/framework/public-api.d.ts +2 -2
- package/dist/index.js +170 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# @getstrata/core changelog
|
|
2
2
|
|
|
3
|
+
## 2.2.8
|
|
4
|
+
|
|
5
|
+
- Add bounded read-only Postgres outbox metrics
|
|
6
|
+
|
|
7
|
+
## 2.2.7
|
|
8
|
+
|
|
9
|
+
- Where an ordinary operation needs SQL because the framework cannot express it
|
|
10
|
+
|
|
3
11
|
## 2.2.6
|
|
4
12
|
|
|
5
13
|
- fix(cli): relay shutdown to children and simplify container launcher
|
|
@@ -40,6 +40,8 @@ type ModelWriteValues<TEntity> = Partial<TEntity>;
|
|
|
40
40
|
type QueryOrder<TEntity> = {
|
|
41
41
|
column: keyof TEntity & string;
|
|
42
42
|
direction?: "ASC" | "DESC" | "asc" | "desc";
|
|
43
|
+
/** Explicit placement; omitted retains the database's default. */
|
|
44
|
+
nulls?: "first" | "last";
|
|
43
45
|
};
|
|
44
46
|
type QueryOrderShorthand<TEntity> = Partial<Record<keyof TEntity & string, "ASC" | "DESC" | "asc" | "desc">>;
|
|
45
47
|
type QueryJoinOn = {
|
|
@@ -50,6 +50,8 @@ declare class SqlOutbox {
|
|
|
50
50
|
/** Operator operation: retains the event and atomically makes one failed delivery eligible. */
|
|
51
51
|
replay(eventId: string, listenerName: string): Promise<void>;
|
|
52
52
|
}
|
|
53
|
+
export type { OutboxMetricState, OutboxMetricsCollector, OutboxMetricsOptions, OutboxMetricsSnapshot, } from "./metrics";
|
|
54
|
+
export { createOutboxMetricsCollector, renderOutboxMetrics } from "./metrics";
|
|
53
55
|
export { createOutboxMigration } from "./migration";
|
|
54
56
|
export type { DurableEvent, DurableListener, SqlOutboxOptions };
|
|
55
57
|
export { SqlOutbox };
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
type OutboxMetricState = "pending_due" | "pending_waiting" | "processing_active" | "processing_expired" | "processing_unleased" | "failed";
|
|
2
|
+
interface OutboxMetricsSnapshot {
|
|
3
|
+
sampleLimit: number;
|
|
4
|
+
states: readonly {
|
|
5
|
+
state: OutboxMetricState;
|
|
6
|
+
count: number;
|
|
7
|
+
capped: boolean;
|
|
8
|
+
actionableLatenessSeconds: number | null;
|
|
9
|
+
}[];
|
|
10
|
+
}
|
|
11
|
+
interface OutboxMetricsOptions {
|
|
12
|
+
timeoutMs?: number;
|
|
13
|
+
sampleLimit?: number;
|
|
14
|
+
}
|
|
15
|
+
interface OutboxMetricsCollector {
|
|
16
|
+
collect(): Promise<OutboxMetricsSnapshot>;
|
|
17
|
+
/** Stop new collection and wait for the underlying transaction to settle. */
|
|
18
|
+
close(): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
declare const snapshotQuery: string;
|
|
21
|
+
/** Explicit platform observation, using the same transaction-local RLS bypass as coordination.
|
|
22
|
+
* Initially Postgres-only: other engines fail rather than receiving a false empty snapshot.
|
|
23
|
+
*/
|
|
24
|
+
declare function createOutboxMetricsCollector(options?: OutboxMetricsOptions): OutboxMetricsCollector;
|
|
25
|
+
declare function renderOutboxMetrics(snapshot: OutboxMetricsSnapshot): string;
|
|
26
|
+
export type { OutboxMetricState, OutboxMetricsCollector, OutboxMetricsOptions, OutboxMetricsSnapshot, };
|
|
27
|
+
export { createOutboxMetricsCollector, renderOutboxMetrics, snapshotQuery };
|
|
@@ -13,3 +13,31 @@ declare function readRedisQueueDepth(redisUrl: string): Promise<QueueDepthMetric
|
|
|
13
13
|
declare function collectQueueMetrics(): Promise<QueueMetricsSnapshot>;
|
|
14
14
|
export type { QueueDepthMetrics, QueueMetricsSnapshot };
|
|
15
15
|
export { collectQueueMetrics, readRedisQueueDepth };
|
|
16
|
+
interface RedisQueueSnapshotOptions {
|
|
17
|
+
transport?: "lists" | "streams";
|
|
18
|
+
/** One deadline for all three priorities, including connection establishment. */
|
|
19
|
+
timeoutMs?: number;
|
|
20
|
+
}
|
|
21
|
+
interface QueuePrioritySnapshot {
|
|
22
|
+
priority: "high" | "default" | "low";
|
|
23
|
+
unfinished: number;
|
|
24
|
+
ready: number | null;
|
|
25
|
+
inflight: number;
|
|
26
|
+
inflightCapped: boolean;
|
|
27
|
+
retryDue: number;
|
|
28
|
+
retryWaiting: number;
|
|
29
|
+
quarantined: number;
|
|
30
|
+
/** Residence age of the oldest current stream entry, not original job age across retries. */
|
|
31
|
+
oldestStreamEntryAgeSeconds: number | null;
|
|
32
|
+
}
|
|
33
|
+
interface RedisQueueSnapshot {
|
|
34
|
+
transport: "lists" | "streams";
|
|
35
|
+
priorities: readonly QueuePrioritySnapshot[];
|
|
36
|
+
}
|
|
37
|
+
declare const INFLIGHT_SAMPLE_LIMIT = 500;
|
|
38
|
+
/** Shared state, not per-process counters. Throws on missing configuration, failure or timeout. */
|
|
39
|
+
declare function readRedisQueueSnapshot(redisUrl: string, options?: RedisQueueSnapshotOptions): Promise<RedisQueueSnapshot>;
|
|
40
|
+
/** Fixed metric/label vocabulary; no job IDs, tenant IDs, payloads or credentials. */
|
|
41
|
+
declare function renderRedisQueueMetrics(snapshot: RedisQueueSnapshot): string;
|
|
42
|
+
export type { QueuePrioritySnapshot, RedisQueueSnapshot, RedisQueueSnapshotOptions };
|
|
43
|
+
export { INFLIGHT_SAMPLE_LIMIT, readRedisQueueSnapshot, renderRedisQueueMetrics };
|
|
@@ -254,8 +254,18 @@ function normalizeOrderBy(orderBy) {
|
|
|
254
254
|
}));
|
|
255
255
|
}
|
|
256
256
|
function buildOrderByClause(tableName, orderBy) {
|
|
257
|
-
const parts = normalizeOrderBy(orderBy).map(({ column, direction }) => {
|
|
258
|
-
|
|
257
|
+
const parts = normalizeOrderBy(orderBy).map(({ column, direction, nulls }) => {
|
|
258
|
+
if (nulls !== undefined && nulls !== "first" && nulls !== "last") {
|
|
259
|
+
throw new TypeError("Invalid null ordering: expected first or last.");
|
|
260
|
+
}
|
|
261
|
+
const reference = resolveQualifiedColumn(tableName, column);
|
|
262
|
+
const ordered = `${reference} ${normalizeDirection(direction)}`;
|
|
263
|
+
if (nulls === undefined)
|
|
264
|
+
return ordered;
|
|
265
|
+
if (currentSqlDialect().driver === "mysql") {
|
|
266
|
+
return `(${reference} IS NULL) ${nulls === "first" ? "DESC" : "ASC"}, ${ordered}`;
|
|
267
|
+
}
|
|
268
|
+
return `${ordered} NULLS ${nulls.toUpperCase()}`;
|
|
259
269
|
});
|
|
260
270
|
return parts.length > 0 ? ` ORDER BY ${parts.join(", ")}` : "";
|
|
261
271
|
}
|
|
@@ -278,8 +278,18 @@ function normalizeOrderBy(orderBy) {
|
|
|
278
278
|
}));
|
|
279
279
|
}
|
|
280
280
|
function buildOrderByClause(tableName, orderBy) {
|
|
281
|
-
const parts = normalizeOrderBy(orderBy).map(({ column, direction }) => {
|
|
282
|
-
|
|
281
|
+
const parts = normalizeOrderBy(orderBy).map(({ column, direction, nulls }) => {
|
|
282
|
+
if (nulls !== undefined && nulls !== "first" && nulls !== "last") {
|
|
283
|
+
throw new TypeError("Invalid null ordering: expected first or last.");
|
|
284
|
+
}
|
|
285
|
+
const reference = resolveQualifiedColumn(tableName, column);
|
|
286
|
+
const ordered = `${reference} ${normalizeDirection(direction)}`;
|
|
287
|
+
if (nulls === undefined)
|
|
288
|
+
return ordered;
|
|
289
|
+
if (currentSqlDialect().driver === "mysql") {
|
|
290
|
+
return `(${reference} IS NULL) ${nulls === "first" ? "DESC" : "ASC"}, ${ordered}`;
|
|
291
|
+
}
|
|
292
|
+
return `${ordered} NULLS ${nulls.toUpperCase()}`;
|
|
283
293
|
});
|
|
284
294
|
return parts.length > 0 ? ` ORDER BY ${parts.join(", ")}` : "";
|
|
285
295
|
}
|
|
@@ -1377,7 +1377,134 @@ async function collectQueueMetrics() {
|
|
|
1377
1377
|
failedCount
|
|
1378
1378
|
};
|
|
1379
1379
|
}
|
|
1380
|
+
var INFLIGHT_SAMPLE_LIMIT = 500;
|
|
1381
|
+
var STREAM_SNAPSHOT = `
|
|
1382
|
+
local now = redis.call('TIME')
|
|
1383
|
+
local ms = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000)
|
|
1384
|
+
local size = redis.call('XLEN', KEYS[1])
|
|
1385
|
+
local pending = redis.pcall('XPENDING', KEYS[1], ARGV[1], '-', '+', tonumber(ARGV[2])+1)
|
|
1386
|
+
if pending.err then
|
|
1387
|
+
if not string.find(pending.err, 'NOGROUP', 1, true) then return redis.error_reply(pending.err) end
|
|
1388
|
+
pending = {}
|
|
1389
|
+
end
|
|
1390
|
+
local capped = #pending > tonumber(ARGV[2]) and 1 or 0
|
|
1391
|
+
local inflight = math.min(#pending, tonumber(ARGV[2]))
|
|
1392
|
+
local ready = capped == 1 and -1 or size-inflight
|
|
1393
|
+
if ready < 0 and capped == 0 then return redis.error_reply('Inconsistent stream pending state') end
|
|
1394
|
+
local retries = redis.call('ZCARD', KEYS[2])
|
|
1395
|
+
local due = redis.call('ZCOUNT', KEYS[2], '-inf', ms)
|
|
1396
|
+
local first = redis.call('XRANGE', KEYS[1], '-', '+', 'COUNT', 1)
|
|
1397
|
+
local age = 0
|
|
1398
|
+
if #first > 0 then
|
|
1399
|
+
local timestamp = tonumber(string.match(first[1][1], '^(%d+)-'))
|
|
1400
|
+
age = math.max(0, ms-timestamp)/1000
|
|
1401
|
+
end
|
|
1402
|
+
return {size+retries, ready, inflight, capped, due, retries-due, redis.call('LLEN', KEYS[3]), tostring(age)}
|
|
1403
|
+
`;
|
|
1404
|
+
var LIST_SNAPSHOT = `
|
|
1405
|
+
local ready = redis.call('LLEN', KEYS[1])
|
|
1406
|
+
local inflight = redis.call('LLEN', KEYS[2])
|
|
1407
|
+
return {ready+inflight, ready, inflight, 0, 0, 0, redis.call('LLEN', KEYS[3]), '-1'}
|
|
1408
|
+
`;
|
|
1409
|
+
async function readRedisQueueSnapshot(redisUrl, options = {}) {
|
|
1410
|
+
if (!redisUrl.trim())
|
|
1411
|
+
throw new TypeError("Queue metrics require a Redis URL.");
|
|
1412
|
+
const transport = options.transport ?? resolveRedisQueueTransport();
|
|
1413
|
+
if (transport !== "lists" && transport !== "streams")
|
|
1414
|
+
throw new TypeError("Invalid queue metrics transport.");
|
|
1415
|
+
const timeoutMs = options.timeoutMs ?? 1000;
|
|
1416
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 5000)
|
|
1417
|
+
throw new TypeError("Queue metrics timeout must be between 1 and 5000 milliseconds.");
|
|
1418
|
+
const client = new RedisClient3(redisUrl, { autoReconnect: false });
|
|
1419
|
+
let timer;
|
|
1420
|
+
try {
|
|
1421
|
+
const work = Promise.all(defaultQueueKeys().map(async (key, index) => {
|
|
1422
|
+
const keys = transport === "streams" ? [streamQueueKey(key), streamRetryKey(key), streamDeadLetterKey(key)] : [key, queueProcessingKey(key), queueInvalidKey(key)];
|
|
1423
|
+
const raw = await client.send("EVAL", [
|
|
1424
|
+
transport === "streams" ? STREAM_SNAPSHOT : LIST_SNAPSHOT,
|
|
1425
|
+
"3",
|
|
1426
|
+
...keys,
|
|
1427
|
+
STREAM_GROUP,
|
|
1428
|
+
String(INFLIGHT_SAMPLE_LIMIT)
|
|
1429
|
+
]);
|
|
1430
|
+
if (!Array.isArray(raw) || raw.length !== 8)
|
|
1431
|
+
throw new Error("Invalid queue metrics response.");
|
|
1432
|
+
const values = raw.map(Number);
|
|
1433
|
+
if (values.some((value) => !Number.isFinite(value)) || values.slice(0, 7).some((value, i) => !Number.isSafeInteger(value) || value < (i === 1 ? -1 : 0)))
|
|
1434
|
+
throw new Error("Invalid queue metrics values.");
|
|
1435
|
+
const [unfinished, ready, inflight, capped, retryDue, retryWaiting, quarantined, age] = values;
|
|
1436
|
+
if (unfinished === undefined || ready === undefined || inflight === undefined || capped === undefined || retryDue === undefined || retryWaiting === undefined || quarantined === undefined || age === undefined || capped !== 0 && capped !== 1 || age < (transport === "lists" ? -1 : 0))
|
|
1437
|
+
throw new Error("Invalid queue metrics values.");
|
|
1438
|
+
return {
|
|
1439
|
+
priority: index === 0 ? "high" : index === 1 ? "default" : "low",
|
|
1440
|
+
unfinished,
|
|
1441
|
+
ready: ready === -1 ? null : ready,
|
|
1442
|
+
inflight,
|
|
1443
|
+
inflightCapped: capped === 1,
|
|
1444
|
+
retryDue,
|
|
1445
|
+
retryWaiting,
|
|
1446
|
+
quarantined,
|
|
1447
|
+
oldestStreamEntryAgeSeconds: transport === "streams" ? age : null
|
|
1448
|
+
};
|
|
1449
|
+
}));
|
|
1450
|
+
const priorities = await Promise.race([
|
|
1451
|
+
work,
|
|
1452
|
+
new Promise((_, reject) => {
|
|
1453
|
+
timer = setTimeout(() => reject(new Error("Queue metrics collection timed out.")), timeoutMs);
|
|
1454
|
+
})
|
|
1455
|
+
]);
|
|
1456
|
+
return { transport, priorities };
|
|
1457
|
+
} finally {
|
|
1458
|
+
if (timer !== undefined)
|
|
1459
|
+
clearTimeout(timer);
|
|
1460
|
+
client.close();
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
function renderRedisQueueMetrics(snapshot) {
|
|
1464
|
+
if (snapshot.transport !== "streams" && snapshot.transport !== "lists" || snapshot.priorities.length !== 3 || new Set(snapshot.priorities.map((row) => row.priority)).size !== 3)
|
|
1465
|
+
throw new TypeError("Invalid queue metrics snapshot.");
|
|
1466
|
+
for (const row of snapshot.priorities) {
|
|
1467
|
+
if (!["high", "default", "low"].includes(row.priority) || typeof row.inflightCapped !== "boolean" || [row.unfinished, row.inflight, row.retryDue, row.retryWaiting, row.quarantined].some((value) => !Number.isSafeInteger(value) || value < 0) || row.ready !== null && (!Number.isSafeInteger(row.ready) || row.ready < 0) || row.oldestStreamEntryAgeSeconds !== null && (!Number.isFinite(row.oldestStreamEntryAgeSeconds) || row.oldestStreamEntryAgeSeconds < 0))
|
|
1468
|
+
throw new TypeError("Invalid queue metrics snapshot.");
|
|
1469
|
+
}
|
|
1470
|
+
const lines = [
|
|
1471
|
+
"# HELP strata_queue_collector_success Whether the configured queue collection succeeded.",
|
|
1472
|
+
"# TYPE strata_queue_collector_success gauge",
|
|
1473
|
+
"strata_queue_collector_success 1",
|
|
1474
|
+
"# HELP strata_queue_unfinished_jobs Stream/list entries plus scheduled retries, excluding quarantine and persisted failures.",
|
|
1475
|
+
"# TYPE strata_queue_unfinished_jobs gauge",
|
|
1476
|
+
"# HELP strata_queue_jobs Queue state; inflight_sample is a lower bound when capped.",
|
|
1477
|
+
"# TYPE strata_queue_jobs gauge",
|
|
1478
|
+
"# HELP strata_queue_inflight_sample_capped Whether the inflight sample was truncated; ready is omitted when truncated.",
|
|
1479
|
+
"# TYPE strata_queue_inflight_sample_capped gauge",
|
|
1480
|
+
"# HELP strata_queue_oldest_stream_entry_age_seconds Oldest current stream entry residence age; resets when a retry is promoted.",
|
|
1481
|
+
"# TYPE strata_queue_oldest_stream_entry_age_seconds gauge"
|
|
1482
|
+
];
|
|
1483
|
+
for (const priority of snapshot.priorities) {
|
|
1484
|
+
const label = `priority="${priority.priority}"`;
|
|
1485
|
+
lines.push(`strata_queue_unfinished_jobs{${label}} ${priority.unfinished}`);
|
|
1486
|
+
const states = {
|
|
1487
|
+
ready: priority.ready,
|
|
1488
|
+
inflight_sample: priority.inflight,
|
|
1489
|
+
retry_due: snapshot.transport === "streams" ? priority.retryDue : null,
|
|
1490
|
+
retry_waiting: snapshot.transport === "streams" ? priority.retryWaiting : null,
|
|
1491
|
+
quarantined: priority.quarantined
|
|
1492
|
+
};
|
|
1493
|
+
for (const [state, value] of Object.entries(states))
|
|
1494
|
+
if (value !== null)
|
|
1495
|
+
lines.push(`strata_queue_jobs{${label},state="${state}"} ${value}`);
|
|
1496
|
+
lines.push(`strata_queue_inflight_sample_capped{${label}} ${Number(priority.inflightCapped)}`);
|
|
1497
|
+
if (priority.oldestStreamEntryAgeSeconds !== null)
|
|
1498
|
+
lines.push(`strata_queue_oldest_stream_entry_age_seconds{${label}} ${priority.oldestStreamEntryAgeSeconds}`);
|
|
1499
|
+
}
|
|
1500
|
+
return `${lines.join(`
|
|
1501
|
+
`)}
|
|
1502
|
+
`;
|
|
1503
|
+
}
|
|
1380
1504
|
export {
|
|
1505
|
+
INFLIGHT_SAMPLE_LIMIT,
|
|
1381
1506
|
collectQueueMetrics,
|
|
1382
|
-
readRedisQueueDepth
|
|
1507
|
+
readRedisQueueDepth,
|
|
1508
|
+
readRedisQueueSnapshot,
|
|
1509
|
+
renderRedisQueueMetrics
|
|
1383
1510
|
};
|
|
@@ -72,8 +72,8 @@ export { BadRequestError, ConflictError, ForbiddenError, HttpError, InternalServ
|
|
|
72
72
|
export type { EventListener } from "../core/events/eventBus.ts";
|
|
73
73
|
export { EventBus, eventBus, readSharedEventBus } from "../core/events/eventBus.ts";
|
|
74
74
|
export { dispatchModelEvent, modelEventName, runWithDeferredModelEvents, } from "../core/events/index.ts";
|
|
75
|
-
export type { DurableEvent, DurableListener, SqlOutboxOptions, } from "../core/events/outbox/index.ts";
|
|
76
|
-
export { createOutboxMigration, SqlOutbox } from "../core/events/outbox/index.ts";
|
|
75
|
+
export type { DurableEvent, DurableListener, OutboxMetricState, OutboxMetricsCollector, OutboxMetricsOptions, OutboxMetricsSnapshot, SqlOutboxOptions, } from "../core/events/outbox/index.ts";
|
|
76
|
+
export { createOutboxMetricsCollector, createOutboxMigration, renderOutboxMetrics, SqlOutbox, } from "../core/events/outbox/index.ts";
|
|
77
77
|
export { auth, cache, config, events, log, mail, policyGate, queue, storage, } from "../core/facades/index.ts";
|
|
78
78
|
export { createBodySizeLimitMiddleware } from "../core/http/bodySizeLimitMiddleware.ts";
|
|
79
79
|
export { readClientIp, trustForwardedFor } from "../core/http/clientIp.ts";
|
package/dist/index.js
CHANGED
|
@@ -4438,8 +4438,18 @@ function normalizeOrderBy(orderBy) {
|
|
|
4438
4438
|
}));
|
|
4439
4439
|
}
|
|
4440
4440
|
function buildOrderByClause(tableName, orderBy) {
|
|
4441
|
-
const parts = normalizeOrderBy(orderBy).map(({ column, direction }) => {
|
|
4442
|
-
|
|
4441
|
+
const parts = normalizeOrderBy(orderBy).map(({ column, direction, nulls }) => {
|
|
4442
|
+
if (nulls !== undefined && nulls !== "first" && nulls !== "last") {
|
|
4443
|
+
throw new TypeError("Invalid null ordering: expected first or last.");
|
|
4444
|
+
}
|
|
4445
|
+
const reference = resolveQualifiedColumn(tableName, column);
|
|
4446
|
+
const ordered = `${reference} ${normalizeDirection(direction)}`;
|
|
4447
|
+
if (nulls === undefined)
|
|
4448
|
+
return ordered;
|
|
4449
|
+
if (currentSqlDialect().driver === "mysql") {
|
|
4450
|
+
return `(${reference} IS NULL) ${nulls === "first" ? "DESC" : "ASC"}, ${ordered}`;
|
|
4451
|
+
}
|
|
4452
|
+
return `${ordered} NULLS ${nulls.toUpperCase()}`;
|
|
4443
4453
|
});
|
|
4444
4454
|
return parts.length > 0 ? ` ORDER BY ${parts.join(", ")}` : "";
|
|
4445
4455
|
}
|
|
@@ -9154,6 +9164,162 @@ init_databaseTenantContext();
|
|
|
9154
9164
|
init_tenantContext();
|
|
9155
9165
|
init_tenantDatabaseScope();
|
|
9156
9166
|
|
|
9167
|
+
// ../../src/core/events/outbox/metrics.ts
|
|
9168
|
+
init_dialect();
|
|
9169
|
+
init_repositoryConnection();
|
|
9170
|
+
init_transaction();
|
|
9171
|
+
init_databaseTenantContext();
|
|
9172
|
+
var specifications = [
|
|
9173
|
+
{ state: "pending_due", status: "pending", column: "available_at", predicate: "<=", age: true },
|
|
9174
|
+
{
|
|
9175
|
+
state: "pending_waiting",
|
|
9176
|
+
status: "pending",
|
|
9177
|
+
column: "available_at",
|
|
9178
|
+
predicate: ">",
|
|
9179
|
+
age: false
|
|
9180
|
+
},
|
|
9181
|
+
{
|
|
9182
|
+
state: "processing_active",
|
|
9183
|
+
status: "processing",
|
|
9184
|
+
column: "lease_until",
|
|
9185
|
+
predicate: ">",
|
|
9186
|
+
age: false
|
|
9187
|
+
},
|
|
9188
|
+
{
|
|
9189
|
+
state: "processing_expired",
|
|
9190
|
+
status: "processing",
|
|
9191
|
+
column: "lease_until",
|
|
9192
|
+
predicate: "<=",
|
|
9193
|
+
age: true
|
|
9194
|
+
},
|
|
9195
|
+
{
|
|
9196
|
+
state: "processing_unleased",
|
|
9197
|
+
status: "processing",
|
|
9198
|
+
column: "lease_until",
|
|
9199
|
+
predicate: "IS NULL",
|
|
9200
|
+
age: false
|
|
9201
|
+
},
|
|
9202
|
+
{ state: "failed", status: "failed", column: "available_at", predicate: "", age: false }
|
|
9203
|
+
];
|
|
9204
|
+
var snapshotQuery = `WITH snapshot_clock AS MATERIALIZED (
|
|
9205
|
+
SELECT CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT) AS now_ms
|
|
9206
|
+
), ${specifications.map((spec, index) => `sample_${index} AS (
|
|
9207
|
+
SELECT ${spec.column} AS timestamp_ms FROM strata_outbox_delivery
|
|
9208
|
+
WHERE status = '${spec.status}'${spec.predicate ? ` AND ${spec.column} ${spec.predicate}${spec.predicate === "IS NULL" ? "" : " (SELECT now_ms FROM snapshot_clock)"}` : ""}
|
|
9209
|
+
ORDER BY ${spec.column}, event_id, listener_name LIMIT $1
|
|
9210
|
+
)`).join(`,
|
|
9211
|
+
`)}
|
|
9212
|
+
${specifications.map((spec, index) => `SELECT '${spec.state}' AS state, COUNT(*) AS count,
|
|
9213
|
+
${spec.age ? `COALESCE(GREATEST(0, (SELECT now_ms FROM snapshot_clock) - MIN(timestamp_ms)), 0)` : "NULL::BIGINT"} AS lateness_ms
|
|
9214
|
+
FROM sample_${index}`).join(`
|
|
9215
|
+
UNION ALL
|
|
9216
|
+
`)}`;
|
|
9217
|
+
function createOutboxMetricsCollector(options = {}) {
|
|
9218
|
+
const timeoutMs = options.timeoutMs ?? 1000;
|
|
9219
|
+
const sampleLimit = options.sampleLimit ?? 500;
|
|
9220
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 5000)
|
|
9221
|
+
throw new TypeError("Outbox metrics timeout must be between 1 and 5000 milliseconds.");
|
|
9222
|
+
if (!Number.isSafeInteger(sampleLimit) || sampleLimit < 1 || sampleLimit > 1000)
|
|
9223
|
+
throw new TypeError("Outbox metrics sample limit must be between 1 and 1000.");
|
|
9224
|
+
let active;
|
|
9225
|
+
let closed = false;
|
|
9226
|
+
return {
|
|
9227
|
+
async collect() {
|
|
9228
|
+
if (closed)
|
|
9229
|
+
throw new Error("Outbox metrics collector is closed.");
|
|
9230
|
+
if (hasActiveTransaction())
|
|
9231
|
+
throw new Error("Outbox metrics require an independent transaction.");
|
|
9232
|
+
if (currentSqlDialect().driver !== "pgsql")
|
|
9233
|
+
throw new Error("Outbox metrics currently require Postgres.");
|
|
9234
|
+
if (active)
|
|
9235
|
+
throw new Error("Outbox metrics collection is still settling.");
|
|
9236
|
+
const deadline = performance.now() + timeoutMs;
|
|
9237
|
+
let expired = false;
|
|
9238
|
+
let timer;
|
|
9239
|
+
const remaining = () => {
|
|
9240
|
+
const value = Math.floor(deadline - performance.now());
|
|
9241
|
+
if (expired || value < 1)
|
|
9242
|
+
throw new Error("Outbox metrics collection timed out.");
|
|
9243
|
+
return value;
|
|
9244
|
+
};
|
|
9245
|
+
const work = runInTransaction(async () => {
|
|
9246
|
+
await repositoryConnection.unsafe("SELECT set_config('statement_timeout', $1, true)", [`${remaining()}ms`]);
|
|
9247
|
+
await repositoryConnection.unsafe("SET TRANSACTION READ ONLY");
|
|
9248
|
+
return runWithMigrationBypass(async () => {
|
|
9249
|
+
remaining();
|
|
9250
|
+
const rows = await repositoryConnection.unsafe(snapshotQuery, [sampleLimit + 1]);
|
|
9251
|
+
remaining();
|
|
9252
|
+
if (rows.length !== specifications.length)
|
|
9253
|
+
throw new Error("Invalid outbox metrics response.");
|
|
9254
|
+
const states = specifications.map((spec) => {
|
|
9255
|
+
const row = rows.find((value) => value.state === spec.state);
|
|
9256
|
+
const count = Number(row?.count);
|
|
9257
|
+
const lateness = row?.lateness_ms == null ? null : Number(row.lateness_ms);
|
|
9258
|
+
if (!Number.isSafeInteger(count) || count < 0 || count > sampleLimit + 1 || spec.age && (lateness === null || !Number.isFinite(lateness) || lateness < 0))
|
|
9259
|
+
throw new Error("Invalid outbox metrics values.");
|
|
9260
|
+
return {
|
|
9261
|
+
state: spec.state,
|
|
9262
|
+
count: Math.min(count, sampleLimit),
|
|
9263
|
+
capped: count > sampleLimit,
|
|
9264
|
+
actionableLatenessSeconds: lateness === null ? null : lateness / 1000
|
|
9265
|
+
};
|
|
9266
|
+
});
|
|
9267
|
+
return { sampleLimit, states };
|
|
9268
|
+
});
|
|
9269
|
+
});
|
|
9270
|
+
active = work;
|
|
9271
|
+
work.finally(() => {
|
|
9272
|
+
if (active === work)
|
|
9273
|
+
active = undefined;
|
|
9274
|
+
}).catch(() => {});
|
|
9275
|
+
try {
|
|
9276
|
+
return await Promise.race([
|
|
9277
|
+
work,
|
|
9278
|
+
new Promise((_, reject) => {
|
|
9279
|
+
timer = setTimeout(() => {
|
|
9280
|
+
expired = true;
|
|
9281
|
+
reject(new Error("Outbox metrics collection timed out."));
|
|
9282
|
+
}, timeoutMs);
|
|
9283
|
+
})
|
|
9284
|
+
]);
|
|
9285
|
+
} finally {
|
|
9286
|
+
if (timer !== undefined)
|
|
9287
|
+
clearTimeout(timer);
|
|
9288
|
+
}
|
|
9289
|
+
},
|
|
9290
|
+
async close() {
|
|
9291
|
+
closed = true;
|
|
9292
|
+
await active?.catch(() => {});
|
|
9293
|
+
}
|
|
9294
|
+
};
|
|
9295
|
+
}
|
|
9296
|
+
function renderOutboxMetrics(snapshot) {
|
|
9297
|
+
if (!Number.isSafeInteger(snapshot.sampleLimit) || snapshot.sampleLimit < 1 || snapshot.sampleLimit > 1000 || snapshot.states.length !== specifications.length)
|
|
9298
|
+
throw new TypeError("Invalid outbox metrics snapshot.");
|
|
9299
|
+
const lines = [
|
|
9300
|
+
"# HELP strata_outbox_collector_success Whether outbox collection succeeded.",
|
|
9301
|
+
"# TYPE strata_outbox_collector_success gauge",
|
|
9302
|
+
"strata_outbox_collector_success 1",
|
|
9303
|
+
"# HELP strata_outbox_deliveries_sample Bounded per-state delivery count; lower bound when capped.",
|
|
9304
|
+
"# TYPE strata_outbox_deliveries_sample gauge",
|
|
9305
|
+
"# HELP strata_outbox_sample_capped Whether the delivery sample is truncated.",
|
|
9306
|
+
"# TYPE strata_outbox_sample_capped gauge",
|
|
9307
|
+
"# HELP strata_outbox_actionable_lateness_seconds Oldest due-time or expired-lease lateness, not original event age.",
|
|
9308
|
+
"# TYPE strata_outbox_actionable_lateness_seconds gauge"
|
|
9309
|
+
];
|
|
9310
|
+
for (const spec of specifications) {
|
|
9311
|
+
const row = snapshot.states.find((value) => value.state === spec.state);
|
|
9312
|
+
if (!row || !Number.isSafeInteger(row.count) || row.count < 0 || row.count > snapshot.sampleLimit || typeof row.capped !== "boolean" || row.actionableLatenessSeconds !== null && (!Number.isFinite(row.actionableLatenessSeconds) || row.actionableLatenessSeconds < 0))
|
|
9313
|
+
throw new TypeError("Invalid outbox metrics snapshot.");
|
|
9314
|
+
lines.push(`strata_outbox_deliveries_sample{state="${spec.state}"} ${row.count}`);
|
|
9315
|
+
lines.push(`strata_outbox_sample_capped{state="${spec.state}"} ${Number(row.capped)}`);
|
|
9316
|
+
if (row.actionableLatenessSeconds !== null)
|
|
9317
|
+
lines.push(`strata_outbox_actionable_lateness_seconds{state="${spec.state}"} ${row.actionableLatenessSeconds}`);
|
|
9318
|
+
}
|
|
9319
|
+
return `${lines.join(`
|
|
9320
|
+
`)}
|
|
9321
|
+
`;
|
|
9322
|
+
}
|
|
9157
9323
|
// ../../src/core/events/outbox/migration.ts
|
|
9158
9324
|
function createOutboxMigration(name, driver, options = {}) {
|
|
9159
9325
|
const identity = driver === "mysql" ? " CHARACTER SET ascii COLLATE ascii_bin" : "";
|
|
@@ -15234,6 +15400,7 @@ export {
|
|
|
15234
15400
|
createMysqlPool,
|
|
15235
15401
|
createNotificationDispatcher,
|
|
15236
15402
|
createOidcHandshake,
|
|
15403
|
+
createOutboxMetricsCollector,
|
|
15237
15404
|
createOutboxMigration,
|
|
15238
15405
|
createProductionQueue,
|
|
15239
15406
|
createQueue,
|
|
@@ -15407,6 +15574,7 @@ export {
|
|
|
15407
15574
|
registerShutdownHandler,
|
|
15408
15575
|
renderKernelErrorChrome,
|
|
15409
15576
|
renderMarkdownMail,
|
|
15577
|
+
renderOutboxMetrics,
|
|
15410
15578
|
renderWebErrorHtml,
|
|
15411
15579
|
repositoryConnection,
|
|
15412
15580
|
requestIdMiddleware,
|