@getstrata/core 2.2.7 → 2.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/core/database/baseRepository.d.ts +6 -0
- package/dist/core/database/transaction.d.ts +6 -1
- package/dist/core/events/outbox/index.d.ts +2 -0
- package/dist/core/events/outbox/metrics.d.ts +27 -0
- package/dist/core/queue/queueMetrics.d.ts +28 -0
- package/dist/entries/audit/exportAuditLogs.js +22 -5
- package/dist/entries/auth/scimAuthMiddleware.js +22 -5
- package/dist/entries/jobs/exportAuditLogsJob.js +22 -5
- package/dist/entries/queue/queueMetrics.js +128 -1
- package/dist/entries/tenant/databaseTenantContext.js +22 -5
- package/dist/entries/tenant/tenantDatabaseScope.js +22 -5
- package/dist/framework/public-api.d.ts +3 -2
- package/dist/index.js +183 -5
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# @getstrata/core changelog
|
|
2
2
|
|
|
3
|
+
## 2.2.9
|
|
4
|
+
|
|
5
|
+
- Support native cancellable transaction acquisition
|
|
6
|
+
|
|
7
|
+
## 2.2.8
|
|
8
|
+
|
|
9
|
+
- Add bounded read-only Postgres outbox metrics
|
|
10
|
+
|
|
3
11
|
## 2.2.7
|
|
4
12
|
|
|
5
13
|
- Where an ordinary operation needs SQL because the framework cannot express it
|
|
@@ -11,6 +11,12 @@ type ExtendedQueryOptions<TEntity extends object> = QueryOptions<TEntity> & {
|
|
|
11
11
|
interface DatabaseConnection {
|
|
12
12
|
unsafe<T>(query: string, params?: readonly unknown[]): Promise<T[]>;
|
|
13
13
|
begin?<T>(callback: (transaction: DatabaseConnection) => Promise<T>): Promise<T>;
|
|
14
|
+
/** Optional native cancellable pool checkout; absence must not be silently emulated. */
|
|
15
|
+
reserve?(options: {
|
|
16
|
+
signal?: AbortSignal;
|
|
17
|
+
}): Promise<DatabaseConnection & {
|
|
18
|
+
release(): void | Promise<void>;
|
|
19
|
+
}>;
|
|
14
20
|
/** SQLite closes synchronously; pooled drivers return a promise. */
|
|
15
21
|
close?(): void | Promise<void>;
|
|
16
22
|
}
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import type { DatabaseConnection } from "./baseRepository.ts";
|
|
2
2
|
import { hasActiveTransaction, requestTransactionRollback } from "./transactionControl.ts";
|
|
3
|
-
|
|
3
|
+
interface TransactionOptions {
|
|
4
|
+
/** Cancel admission/pool checkout only. Does not cancel SQL or an admitted business callback. */
|
|
5
|
+
acquisitionSignal?: AbortSignal;
|
|
6
|
+
}
|
|
7
|
+
declare function runInTransaction<TValue>(operation: (connection: DatabaseConnection) => Promise<TValue>, options?: TransactionOptions): Promise<TValue>;
|
|
8
|
+
export type { TransactionOptions };
|
|
4
9
|
export { hasActiveTransaction, requestTransactionRollback, runInTransaction };
|
|
@@ -50,6 +50,8 @@ declare class SqlOutbox {
|
|
|
50
50
|
/** Operator operation: retains the event and atomically makes one failed delivery eligible. */
|
|
51
51
|
replay(eventId: string, listenerName: string): Promise<void>;
|
|
52
52
|
}
|
|
53
|
+
export type { OutboxMetricState, OutboxMetricsCollector, OutboxMetricsOptions, OutboxMetricsSnapshot, } from "./metrics";
|
|
54
|
+
export { createOutboxMetricsCollector, renderOutboxMetrics } from "./metrics";
|
|
53
55
|
export { createOutboxMigration } from "./migration";
|
|
54
56
|
export type { DurableEvent, DurableListener, SqlOutboxOptions };
|
|
55
57
|
export { SqlOutbox };
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
type OutboxMetricState = "pending_due" | "pending_waiting" | "processing_active" | "processing_expired" | "processing_unleased" | "failed";
|
|
2
|
+
interface OutboxMetricsSnapshot {
|
|
3
|
+
sampleLimit: number;
|
|
4
|
+
states: readonly {
|
|
5
|
+
state: OutboxMetricState;
|
|
6
|
+
count: number;
|
|
7
|
+
capped: boolean;
|
|
8
|
+
actionableLatenessSeconds: number | null;
|
|
9
|
+
}[];
|
|
10
|
+
}
|
|
11
|
+
interface OutboxMetricsOptions {
|
|
12
|
+
timeoutMs?: number;
|
|
13
|
+
sampleLimit?: number;
|
|
14
|
+
}
|
|
15
|
+
interface OutboxMetricsCollector {
|
|
16
|
+
collect(): Promise<OutboxMetricsSnapshot>;
|
|
17
|
+
/** Stop new collection and wait for the underlying transaction to settle. */
|
|
18
|
+
close(): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
declare const snapshotQuery: string;
|
|
21
|
+
/** Explicit platform observation, using the same transaction-local RLS bypass as coordination.
|
|
22
|
+
* Initially Postgres-only: other engines fail rather than receiving a false empty snapshot.
|
|
23
|
+
*/
|
|
24
|
+
declare function createOutboxMetricsCollector(options?: OutboxMetricsOptions): OutboxMetricsCollector;
|
|
25
|
+
declare function renderOutboxMetrics(snapshot: OutboxMetricsSnapshot): string;
|
|
26
|
+
export type { OutboxMetricState, OutboxMetricsCollector, OutboxMetricsOptions, OutboxMetricsSnapshot, };
|
|
27
|
+
export { createOutboxMetricsCollector, renderOutboxMetrics, snapshotQuery };
|
|
@@ -13,3 +13,31 @@ declare function readRedisQueueDepth(redisUrl: string): Promise<QueueDepthMetric
|
|
|
13
13
|
declare function collectQueueMetrics(): Promise<QueueMetricsSnapshot>;
|
|
14
14
|
export type { QueueDepthMetrics, QueueMetricsSnapshot };
|
|
15
15
|
export { collectQueueMetrics, readRedisQueueDepth };
|
|
16
|
+
interface RedisQueueSnapshotOptions {
|
|
17
|
+
transport?: "lists" | "streams";
|
|
18
|
+
/** One deadline for all three priorities, including connection establishment. */
|
|
19
|
+
timeoutMs?: number;
|
|
20
|
+
}
|
|
21
|
+
interface QueuePrioritySnapshot {
|
|
22
|
+
priority: "high" | "default" | "low";
|
|
23
|
+
unfinished: number;
|
|
24
|
+
ready: number | null;
|
|
25
|
+
inflight: number;
|
|
26
|
+
inflightCapped: boolean;
|
|
27
|
+
retryDue: number;
|
|
28
|
+
retryWaiting: number;
|
|
29
|
+
quarantined: number;
|
|
30
|
+
/** Residence age of the oldest current stream entry, not original job age across retries. */
|
|
31
|
+
oldestStreamEntryAgeSeconds: number | null;
|
|
32
|
+
}
|
|
33
|
+
interface RedisQueueSnapshot {
|
|
34
|
+
transport: "lists" | "streams";
|
|
35
|
+
priorities: readonly QueuePrioritySnapshot[];
|
|
36
|
+
}
|
|
37
|
+
declare const INFLIGHT_SAMPLE_LIMIT = 500;
|
|
38
|
+
/** Shared state, not per-process counters. Throws on missing configuration, failure or timeout. */
|
|
39
|
+
declare function readRedisQueueSnapshot(redisUrl: string, options?: RedisQueueSnapshotOptions): Promise<RedisQueueSnapshot>;
|
|
40
|
+
/** Fixed metric/label vocabulary; no job IDs, tenant IDs, payloads or credentials. */
|
|
41
|
+
declare function renderRedisQueueMetrics(snapshot: RedisQueueSnapshot): string;
|
|
42
|
+
export type { QueuePrioritySnapshot, RedisQueueSnapshot, RedisQueueSnapshotOptions };
|
|
43
|
+
export { INFLIGHT_SAMPLE_LIMIT, readRedisQueueSnapshot, renderRedisQueueMetrics };
|
|
@@ -697,18 +697,35 @@ async function runInSavepoint(connection, operation) {
|
|
|
697
697
|
function supportsTransactions(connection) {
|
|
698
698
|
return typeof connection.begin === "function";
|
|
699
699
|
}
|
|
700
|
-
async function runInTransaction(operation) {
|
|
700
|
+
async function runInTransaction(operation, options = {}) {
|
|
701
|
+
const signal = options.acquisitionSignal;
|
|
702
|
+
signal?.throwIfAborted();
|
|
703
|
+
const admittedOperation = (connection) => {
|
|
704
|
+
signal?.throwIfAborted();
|
|
705
|
+
return operation(connection);
|
|
706
|
+
};
|
|
701
707
|
const pool = resolveRepositoryConnection();
|
|
702
708
|
if (hasActiveDatabaseConnection()) {
|
|
703
|
-
return await runInSavepoint(getActiveDatabaseConnection(pool),
|
|
709
|
+
return await runInSavepoint(getActiveDatabaseConnection(pool), admittedOperation);
|
|
704
710
|
}
|
|
705
711
|
if (!supportsTransactions(pool)) {
|
|
706
712
|
throw new Error("Active database connection does not support transactions. Bind a client with begin() via bindDatabaseConnection.");
|
|
707
713
|
}
|
|
708
714
|
return await settleTransaction(() => runWithDeferredModelEvents(async () => {
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
});
|
|
715
|
+
if (signal && !pool.reserve)
|
|
716
|
+
throw new Error("Active database connection does not support cancellable transaction acquisition.");
|
|
717
|
+
const reserved = signal ? await pool.reserve?.({ signal }) : undefined;
|
|
718
|
+
const connection = reserved ?? pool;
|
|
719
|
+
try {
|
|
720
|
+
signal?.throwIfAborted();
|
|
721
|
+
if (!supportsTransactions(connection))
|
|
722
|
+
throw new Error("Reserved database connection does not support transactions.");
|
|
723
|
+
return await connection.begin(async (transaction) => {
|
|
724
|
+
return await runWithDatabaseConnection(transaction, () => transactionFrame.run({ connection: transaction, childActive: false }, () => runWithTransactionScope(async () => commitOrRollbackScope(await admittedOperation(createDatabaseConnection(transaction))))));
|
|
725
|
+
});
|
|
726
|
+
} finally {
|
|
727
|
+
await reserved?.release();
|
|
728
|
+
}
|
|
712
729
|
}));
|
|
713
730
|
}
|
|
714
731
|
var transactionFrame, outerFrames;
|
|
@@ -653,18 +653,35 @@ async function runInSavepoint(connection, operation) {
|
|
|
653
653
|
function supportsTransactions(connection) {
|
|
654
654
|
return typeof connection.begin === "function";
|
|
655
655
|
}
|
|
656
|
-
async function runInTransaction(operation) {
|
|
656
|
+
async function runInTransaction(operation, options = {}) {
|
|
657
|
+
const signal = options.acquisitionSignal;
|
|
658
|
+
signal?.throwIfAborted();
|
|
659
|
+
const admittedOperation = (connection) => {
|
|
660
|
+
signal?.throwIfAborted();
|
|
661
|
+
return operation(connection);
|
|
662
|
+
};
|
|
657
663
|
const pool = resolveRepositoryConnection();
|
|
658
664
|
if (hasActiveDatabaseConnection()) {
|
|
659
|
-
return await runInSavepoint(getActiveDatabaseConnection(pool),
|
|
665
|
+
return await runInSavepoint(getActiveDatabaseConnection(pool), admittedOperation);
|
|
660
666
|
}
|
|
661
667
|
if (!supportsTransactions(pool)) {
|
|
662
668
|
throw new Error("Active database connection does not support transactions. Bind a client with begin() via bindDatabaseConnection.");
|
|
663
669
|
}
|
|
664
670
|
return await settleTransaction(() => runWithDeferredModelEvents(async () => {
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
});
|
|
671
|
+
if (signal && !pool.reserve)
|
|
672
|
+
throw new Error("Active database connection does not support cancellable transaction acquisition.");
|
|
673
|
+
const reserved = signal ? await pool.reserve?.({ signal }) : undefined;
|
|
674
|
+
const connection = reserved ?? pool;
|
|
675
|
+
try {
|
|
676
|
+
signal?.throwIfAborted();
|
|
677
|
+
if (!supportsTransactions(connection))
|
|
678
|
+
throw new Error("Reserved database connection does not support transactions.");
|
|
679
|
+
return await connection.begin(async (transaction) => {
|
|
680
|
+
return await runWithDatabaseConnection(transaction, () => transactionFrame.run({ connection: transaction, childActive: false }, () => runWithTransactionScope(async () => commitOrRollbackScope(await admittedOperation(createDatabaseConnection(transaction))))));
|
|
681
|
+
});
|
|
682
|
+
} finally {
|
|
683
|
+
await reserved?.release();
|
|
684
|
+
}
|
|
668
685
|
}));
|
|
669
686
|
}
|
|
670
687
|
var transactionFrame, outerFrames;
|
|
@@ -697,18 +697,35 @@ async function runInSavepoint(connection, operation) {
|
|
|
697
697
|
function supportsTransactions(connection) {
|
|
698
698
|
return typeof connection.begin === "function";
|
|
699
699
|
}
|
|
700
|
-
async function runInTransaction(operation) {
|
|
700
|
+
async function runInTransaction(operation, options = {}) {
|
|
701
|
+
const signal = options.acquisitionSignal;
|
|
702
|
+
signal?.throwIfAborted();
|
|
703
|
+
const admittedOperation = (connection) => {
|
|
704
|
+
signal?.throwIfAborted();
|
|
705
|
+
return operation(connection);
|
|
706
|
+
};
|
|
701
707
|
const pool = resolveRepositoryConnection();
|
|
702
708
|
if (hasActiveDatabaseConnection()) {
|
|
703
|
-
return await runInSavepoint(getActiveDatabaseConnection(pool),
|
|
709
|
+
return await runInSavepoint(getActiveDatabaseConnection(pool), admittedOperation);
|
|
704
710
|
}
|
|
705
711
|
if (!supportsTransactions(pool)) {
|
|
706
712
|
throw new Error("Active database connection does not support transactions. Bind a client with begin() via bindDatabaseConnection.");
|
|
707
713
|
}
|
|
708
714
|
return await settleTransaction(() => runWithDeferredModelEvents(async () => {
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
});
|
|
715
|
+
if (signal && !pool.reserve)
|
|
716
|
+
throw new Error("Active database connection does not support cancellable transaction acquisition.");
|
|
717
|
+
const reserved = signal ? await pool.reserve?.({ signal }) : undefined;
|
|
718
|
+
const connection = reserved ?? pool;
|
|
719
|
+
try {
|
|
720
|
+
signal?.throwIfAborted();
|
|
721
|
+
if (!supportsTransactions(connection))
|
|
722
|
+
throw new Error("Reserved database connection does not support transactions.");
|
|
723
|
+
return await connection.begin(async (transaction) => {
|
|
724
|
+
return await runWithDatabaseConnection(transaction, () => transactionFrame.run({ connection: transaction, childActive: false }, () => runWithTransactionScope(async () => commitOrRollbackScope(await admittedOperation(createDatabaseConnection(transaction))))));
|
|
725
|
+
});
|
|
726
|
+
} finally {
|
|
727
|
+
await reserved?.release();
|
|
728
|
+
}
|
|
712
729
|
}));
|
|
713
730
|
}
|
|
714
731
|
var transactionFrame, outerFrames;
|
|
@@ -1377,7 +1377,134 @@ async function collectQueueMetrics() {
|
|
|
1377
1377
|
failedCount
|
|
1378
1378
|
};
|
|
1379
1379
|
}
|
|
1380
|
+
var INFLIGHT_SAMPLE_LIMIT = 500;
|
|
1381
|
+
var STREAM_SNAPSHOT = `
|
|
1382
|
+
local now = redis.call('TIME')
|
|
1383
|
+
local ms = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000)
|
|
1384
|
+
local size = redis.call('XLEN', KEYS[1])
|
|
1385
|
+
local pending = redis.pcall('XPENDING', KEYS[1], ARGV[1], '-', '+', tonumber(ARGV[2])+1)
|
|
1386
|
+
if pending.err then
|
|
1387
|
+
if not string.find(pending.err, 'NOGROUP', 1, true) then return redis.error_reply(pending.err) end
|
|
1388
|
+
pending = {}
|
|
1389
|
+
end
|
|
1390
|
+
local capped = #pending > tonumber(ARGV[2]) and 1 or 0
|
|
1391
|
+
local inflight = math.min(#pending, tonumber(ARGV[2]))
|
|
1392
|
+
local ready = capped == 1 and -1 or size-inflight
|
|
1393
|
+
if ready < 0 and capped == 0 then return redis.error_reply('Inconsistent stream pending state') end
|
|
1394
|
+
local retries = redis.call('ZCARD', KEYS[2])
|
|
1395
|
+
local due = redis.call('ZCOUNT', KEYS[2], '-inf', ms)
|
|
1396
|
+
local first = redis.call('XRANGE', KEYS[1], '-', '+', 'COUNT', 1)
|
|
1397
|
+
local age = 0
|
|
1398
|
+
if #first > 0 then
|
|
1399
|
+
local timestamp = tonumber(string.match(first[1][1], '^(%d+)-'))
|
|
1400
|
+
age = math.max(0, ms-timestamp)/1000
|
|
1401
|
+
end
|
|
1402
|
+
return {size+retries, ready, inflight, capped, due, retries-due, redis.call('LLEN', KEYS[3]), tostring(age)}
|
|
1403
|
+
`;
|
|
1404
|
+
var LIST_SNAPSHOT = `
|
|
1405
|
+
local ready = redis.call('LLEN', KEYS[1])
|
|
1406
|
+
local inflight = redis.call('LLEN', KEYS[2])
|
|
1407
|
+
return {ready+inflight, ready, inflight, 0, 0, 0, redis.call('LLEN', KEYS[3]), '-1'}
|
|
1408
|
+
`;
|
|
1409
|
+
async function readRedisQueueSnapshot(redisUrl, options = {}) {
|
|
1410
|
+
if (!redisUrl.trim())
|
|
1411
|
+
throw new TypeError("Queue metrics require a Redis URL.");
|
|
1412
|
+
const transport = options.transport ?? resolveRedisQueueTransport();
|
|
1413
|
+
if (transport !== "lists" && transport !== "streams")
|
|
1414
|
+
throw new TypeError("Invalid queue metrics transport.");
|
|
1415
|
+
const timeoutMs = options.timeoutMs ?? 1000;
|
|
1416
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 5000)
|
|
1417
|
+
throw new TypeError("Queue metrics timeout must be between 1 and 5000 milliseconds.");
|
|
1418
|
+
const client = new RedisClient3(redisUrl, { autoReconnect: false });
|
|
1419
|
+
let timer;
|
|
1420
|
+
try {
|
|
1421
|
+
const work = Promise.all(defaultQueueKeys().map(async (key, index) => {
|
|
1422
|
+
const keys = transport === "streams" ? [streamQueueKey(key), streamRetryKey(key), streamDeadLetterKey(key)] : [key, queueProcessingKey(key), queueInvalidKey(key)];
|
|
1423
|
+
const raw = await client.send("EVAL", [
|
|
1424
|
+
transport === "streams" ? STREAM_SNAPSHOT : LIST_SNAPSHOT,
|
|
1425
|
+
"3",
|
|
1426
|
+
...keys,
|
|
1427
|
+
STREAM_GROUP,
|
|
1428
|
+
String(INFLIGHT_SAMPLE_LIMIT)
|
|
1429
|
+
]);
|
|
1430
|
+
if (!Array.isArray(raw) || raw.length !== 8)
|
|
1431
|
+
throw new Error("Invalid queue metrics response.");
|
|
1432
|
+
const values = raw.map(Number);
|
|
1433
|
+
if (values.some((value) => !Number.isFinite(value)) || values.slice(0, 7).some((value, i) => !Number.isSafeInteger(value) || value < (i === 1 ? -1 : 0)))
|
|
1434
|
+
throw new Error("Invalid queue metrics values.");
|
|
1435
|
+
const [unfinished, ready, inflight, capped, retryDue, retryWaiting, quarantined, age] = values;
|
|
1436
|
+
if (unfinished === undefined || ready === undefined || inflight === undefined || capped === undefined || retryDue === undefined || retryWaiting === undefined || quarantined === undefined || age === undefined || capped !== 0 && capped !== 1 || age < (transport === "lists" ? -1 : 0))
|
|
1437
|
+
throw new Error("Invalid queue metrics values.");
|
|
1438
|
+
return {
|
|
1439
|
+
priority: index === 0 ? "high" : index === 1 ? "default" : "low",
|
|
1440
|
+
unfinished,
|
|
1441
|
+
ready: ready === -1 ? null : ready,
|
|
1442
|
+
inflight,
|
|
1443
|
+
inflightCapped: capped === 1,
|
|
1444
|
+
retryDue,
|
|
1445
|
+
retryWaiting,
|
|
1446
|
+
quarantined,
|
|
1447
|
+
oldestStreamEntryAgeSeconds: transport === "streams" ? age : null
|
|
1448
|
+
};
|
|
1449
|
+
}));
|
|
1450
|
+
const priorities = await Promise.race([
|
|
1451
|
+
work,
|
|
1452
|
+
new Promise((_, reject) => {
|
|
1453
|
+
timer = setTimeout(() => reject(new Error("Queue metrics collection timed out.")), timeoutMs);
|
|
1454
|
+
})
|
|
1455
|
+
]);
|
|
1456
|
+
return { transport, priorities };
|
|
1457
|
+
} finally {
|
|
1458
|
+
if (timer !== undefined)
|
|
1459
|
+
clearTimeout(timer);
|
|
1460
|
+
client.close();
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
function renderRedisQueueMetrics(snapshot) {
|
|
1464
|
+
if (snapshot.transport !== "streams" && snapshot.transport !== "lists" || snapshot.priorities.length !== 3 || new Set(snapshot.priorities.map((row) => row.priority)).size !== 3)
|
|
1465
|
+
throw new TypeError("Invalid queue metrics snapshot.");
|
|
1466
|
+
for (const row of snapshot.priorities) {
|
|
1467
|
+
if (!["high", "default", "low"].includes(row.priority) || typeof row.inflightCapped !== "boolean" || [row.unfinished, row.inflight, row.retryDue, row.retryWaiting, row.quarantined].some((value) => !Number.isSafeInteger(value) || value < 0) || row.ready !== null && (!Number.isSafeInteger(row.ready) || row.ready < 0) || row.oldestStreamEntryAgeSeconds !== null && (!Number.isFinite(row.oldestStreamEntryAgeSeconds) || row.oldestStreamEntryAgeSeconds < 0))
|
|
1468
|
+
throw new TypeError("Invalid queue metrics snapshot.");
|
|
1469
|
+
}
|
|
1470
|
+
const lines = [
|
|
1471
|
+
"# HELP strata_queue_collector_success Whether the configured queue collection succeeded.",
|
|
1472
|
+
"# TYPE strata_queue_collector_success gauge",
|
|
1473
|
+
"strata_queue_collector_success 1",
|
|
1474
|
+
"# HELP strata_queue_unfinished_jobs Stream/list entries plus scheduled retries, excluding quarantine and persisted failures.",
|
|
1475
|
+
"# TYPE strata_queue_unfinished_jobs gauge",
|
|
1476
|
+
"# HELP strata_queue_jobs Queue state; inflight_sample is a lower bound when capped.",
|
|
1477
|
+
"# TYPE strata_queue_jobs gauge",
|
|
1478
|
+
"# HELP strata_queue_inflight_sample_capped Whether the inflight sample was truncated; ready is omitted when truncated.",
|
|
1479
|
+
"# TYPE strata_queue_inflight_sample_capped gauge",
|
|
1480
|
+
"# HELP strata_queue_oldest_stream_entry_age_seconds Oldest current stream entry residence age; resets when a retry is promoted.",
|
|
1481
|
+
"# TYPE strata_queue_oldest_stream_entry_age_seconds gauge"
|
|
1482
|
+
];
|
|
1483
|
+
for (const priority of snapshot.priorities) {
|
|
1484
|
+
const label = `priority="${priority.priority}"`;
|
|
1485
|
+
lines.push(`strata_queue_unfinished_jobs{${label}} ${priority.unfinished}`);
|
|
1486
|
+
const states = {
|
|
1487
|
+
ready: priority.ready,
|
|
1488
|
+
inflight_sample: priority.inflight,
|
|
1489
|
+
retry_due: snapshot.transport === "streams" ? priority.retryDue : null,
|
|
1490
|
+
retry_waiting: snapshot.transport === "streams" ? priority.retryWaiting : null,
|
|
1491
|
+
quarantined: priority.quarantined
|
|
1492
|
+
};
|
|
1493
|
+
for (const [state, value] of Object.entries(states))
|
|
1494
|
+
if (value !== null)
|
|
1495
|
+
lines.push(`strata_queue_jobs{${label},state="${state}"} ${value}`);
|
|
1496
|
+
lines.push(`strata_queue_inflight_sample_capped{${label}} ${Number(priority.inflightCapped)}`);
|
|
1497
|
+
if (priority.oldestStreamEntryAgeSeconds !== null)
|
|
1498
|
+
lines.push(`strata_queue_oldest_stream_entry_age_seconds{${label}} ${priority.oldestStreamEntryAgeSeconds}`);
|
|
1499
|
+
}
|
|
1500
|
+
return `${lines.join(`
|
|
1501
|
+
`)}
|
|
1502
|
+
`;
|
|
1503
|
+
}
|
|
1380
1504
|
export {
|
|
1505
|
+
INFLIGHT_SAMPLE_LIMIT,
|
|
1381
1506
|
collectQueueMetrics,
|
|
1382
|
-
readRedisQueueDepth
|
|
1507
|
+
readRedisQueueDepth,
|
|
1508
|
+
readRedisQueueSnapshot,
|
|
1509
|
+
renderRedisQueueMetrics
|
|
1383
1510
|
};
|
|
@@ -697,18 +697,35 @@ async function runInSavepoint(connection, operation) {
|
|
|
697
697
|
function supportsTransactions(connection) {
|
|
698
698
|
return typeof connection.begin === "function";
|
|
699
699
|
}
|
|
700
|
-
async function runInTransaction(operation) {
|
|
700
|
+
async function runInTransaction(operation, options = {}) {
|
|
701
|
+
const signal = options.acquisitionSignal;
|
|
702
|
+
signal?.throwIfAborted();
|
|
703
|
+
const admittedOperation = (connection) => {
|
|
704
|
+
signal?.throwIfAborted();
|
|
705
|
+
return operation(connection);
|
|
706
|
+
};
|
|
701
707
|
const pool = resolveRepositoryConnection();
|
|
702
708
|
if (hasActiveDatabaseConnection()) {
|
|
703
|
-
return await runInSavepoint(getActiveDatabaseConnection(pool),
|
|
709
|
+
return await runInSavepoint(getActiveDatabaseConnection(pool), admittedOperation);
|
|
704
710
|
}
|
|
705
711
|
if (!supportsTransactions(pool)) {
|
|
706
712
|
throw new Error("Active database connection does not support transactions. Bind a client with begin() via bindDatabaseConnection.");
|
|
707
713
|
}
|
|
708
714
|
return await settleTransaction(() => runWithDeferredModelEvents(async () => {
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
});
|
|
715
|
+
if (signal && !pool.reserve)
|
|
716
|
+
throw new Error("Active database connection does not support cancellable transaction acquisition.");
|
|
717
|
+
const reserved = signal ? await pool.reserve?.({ signal }) : undefined;
|
|
718
|
+
const connection = reserved ?? pool;
|
|
719
|
+
try {
|
|
720
|
+
signal?.throwIfAborted();
|
|
721
|
+
if (!supportsTransactions(connection))
|
|
722
|
+
throw new Error("Reserved database connection does not support transactions.");
|
|
723
|
+
return await connection.begin(async (transaction) => {
|
|
724
|
+
return await runWithDatabaseConnection(transaction, () => transactionFrame.run({ connection: transaction, childActive: false }, () => runWithTransactionScope(async () => commitOrRollbackScope(await admittedOperation(createDatabaseConnection(transaction))))));
|
|
725
|
+
});
|
|
726
|
+
} finally {
|
|
727
|
+
await reserved?.release();
|
|
728
|
+
}
|
|
712
729
|
}));
|
|
713
730
|
}
|
|
714
731
|
var transactionFrame, outerFrames;
|
|
@@ -631,18 +631,35 @@ async function runInSavepoint(connection, operation) {
|
|
|
631
631
|
function supportsTransactions(connection) {
|
|
632
632
|
return typeof connection.begin === "function";
|
|
633
633
|
}
|
|
634
|
-
async function runInTransaction(operation) {
|
|
634
|
+
async function runInTransaction(operation, options = {}) {
|
|
635
|
+
const signal = options.acquisitionSignal;
|
|
636
|
+
signal?.throwIfAborted();
|
|
637
|
+
const admittedOperation = (connection) => {
|
|
638
|
+
signal?.throwIfAborted();
|
|
639
|
+
return operation(connection);
|
|
640
|
+
};
|
|
635
641
|
const pool = resolveRepositoryConnection();
|
|
636
642
|
if (hasActiveDatabaseConnection()) {
|
|
637
|
-
return await runInSavepoint(getActiveDatabaseConnection(pool),
|
|
643
|
+
return await runInSavepoint(getActiveDatabaseConnection(pool), admittedOperation);
|
|
638
644
|
}
|
|
639
645
|
if (!supportsTransactions(pool)) {
|
|
640
646
|
throw new Error("Active database connection does not support transactions. Bind a client with begin() via bindDatabaseConnection.");
|
|
641
647
|
}
|
|
642
648
|
return await settleTransaction(() => runWithDeferredModelEvents(async () => {
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
});
|
|
649
|
+
if (signal && !pool.reserve)
|
|
650
|
+
throw new Error("Active database connection does not support cancellable transaction acquisition.");
|
|
651
|
+
const reserved = signal ? await pool.reserve?.({ signal }) : undefined;
|
|
652
|
+
const connection = reserved ?? pool;
|
|
653
|
+
try {
|
|
654
|
+
signal?.throwIfAborted();
|
|
655
|
+
if (!supportsTransactions(connection))
|
|
656
|
+
throw new Error("Reserved database connection does not support transactions.");
|
|
657
|
+
return await connection.begin(async (transaction) => {
|
|
658
|
+
return await runWithDatabaseConnection(transaction, () => transactionFrame.run({ connection: transaction, childActive: false }, () => runWithTransactionScope(async () => commitOrRollbackScope(await admittedOperation(createDatabaseConnection(transaction))))));
|
|
659
|
+
});
|
|
660
|
+
} finally {
|
|
661
|
+
await reserved?.release();
|
|
662
|
+
}
|
|
646
663
|
}));
|
|
647
664
|
}
|
|
648
665
|
var transactionFrame, outerFrames;
|
|
@@ -64,6 +64,7 @@ export { loadSeedersFromDirectory, runSeedersFromDirectory, } from "../core/data
|
|
|
64
64
|
export type { Seeder, SeederDatabase } from "../core/database/seeders/types.ts";
|
|
65
65
|
export { createSqliteConnection } from "../core/database/sqliteConnection.ts";
|
|
66
66
|
export { defineTable } from "../core/database/table.ts";
|
|
67
|
+
export type { TransactionOptions } from "../core/database/transaction.ts";
|
|
67
68
|
export { hasActiveTransaction, requestTransactionRollback, runInTransaction, } from "../core/database/transaction.ts";
|
|
68
69
|
export type { ModelWhere, ModelWriteValues, QueryJoin, QueryJoinOn, QueryOptions, QueryOrder, QuerySelectItem, QueryWhere, } from "../core/database/types.ts";
|
|
69
70
|
export type { WhereNode } from "../core/database/whereBuilder.ts";
|
|
@@ -72,8 +73,8 @@ export { BadRequestError, ConflictError, ForbiddenError, HttpError, InternalServ
|
|
|
72
73
|
export type { EventListener } from "../core/events/eventBus.ts";
|
|
73
74
|
export { EventBus, eventBus, readSharedEventBus } from "../core/events/eventBus.ts";
|
|
74
75
|
export { dispatchModelEvent, modelEventName, runWithDeferredModelEvents, } from "../core/events/index.ts";
|
|
75
|
-
export type { DurableEvent, DurableListener, SqlOutboxOptions, } from "../core/events/outbox/index.ts";
|
|
76
|
-
export { createOutboxMigration, SqlOutbox } from "../core/events/outbox/index.ts";
|
|
76
|
+
export type { DurableEvent, DurableListener, OutboxMetricState, OutboxMetricsCollector, OutboxMetricsOptions, OutboxMetricsSnapshot, SqlOutboxOptions, } from "../core/events/outbox/index.ts";
|
|
77
|
+
export { createOutboxMetricsCollector, createOutboxMigration, renderOutboxMetrics, SqlOutbox, } from "../core/events/outbox/index.ts";
|
|
77
78
|
export { auth, cache, config, events, log, mail, policyGate, queue, storage, } from "../core/facades/index.ts";
|
|
78
79
|
export { createBodySizeLimitMiddleware } from "../core/http/bodySizeLimitMiddleware.ts";
|
|
79
80
|
export { readClientIp, trustForwardedFor } from "../core/http/clientIp.ts";
|
package/dist/index.js
CHANGED
|
@@ -730,18 +730,35 @@ async function runInSavepoint(connection, operation) {
|
|
|
730
730
|
function supportsTransactions(connection) {
|
|
731
731
|
return typeof connection.begin === "function";
|
|
732
732
|
}
|
|
733
|
-
async function runInTransaction(operation) {
|
|
733
|
+
async function runInTransaction(operation, options = {}) {
|
|
734
|
+
const signal = options.acquisitionSignal;
|
|
735
|
+
signal?.throwIfAborted();
|
|
736
|
+
const admittedOperation = (connection) => {
|
|
737
|
+
signal?.throwIfAborted();
|
|
738
|
+
return operation(connection);
|
|
739
|
+
};
|
|
734
740
|
const pool = resolveRepositoryConnection();
|
|
735
741
|
if (hasActiveDatabaseConnection()) {
|
|
736
|
-
return await runInSavepoint(getActiveDatabaseConnection(pool),
|
|
742
|
+
return await runInSavepoint(getActiveDatabaseConnection(pool), admittedOperation);
|
|
737
743
|
}
|
|
738
744
|
if (!supportsTransactions(pool)) {
|
|
739
745
|
throw new Error("Active database connection does not support transactions. Bind a client with begin() via bindDatabaseConnection.");
|
|
740
746
|
}
|
|
741
747
|
return await settleTransaction(() => runWithDeferredModelEvents(async () => {
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
});
|
|
748
|
+
if (signal && !pool.reserve)
|
|
749
|
+
throw new Error("Active database connection does not support cancellable transaction acquisition.");
|
|
750
|
+
const reserved = signal ? await pool.reserve?.({ signal }) : undefined;
|
|
751
|
+
const connection = reserved ?? pool;
|
|
752
|
+
try {
|
|
753
|
+
signal?.throwIfAborted();
|
|
754
|
+
if (!supportsTransactions(connection))
|
|
755
|
+
throw new Error("Reserved database connection does not support transactions.");
|
|
756
|
+
return await connection.begin(async (transaction) => {
|
|
757
|
+
return await runWithDatabaseConnection(transaction, () => transactionFrame.run({ connection: transaction, childActive: false }, () => runWithTransactionScope(async () => commitOrRollbackScope(await admittedOperation(createDatabaseConnection(transaction))))));
|
|
758
|
+
});
|
|
759
|
+
} finally {
|
|
760
|
+
await reserved?.release();
|
|
761
|
+
}
|
|
745
762
|
}));
|
|
746
763
|
}
|
|
747
764
|
var transactionFrame, outerFrames;
|
|
@@ -9164,6 +9181,165 @@ init_databaseTenantContext();
|
|
|
9164
9181
|
init_tenantContext();
|
|
9165
9182
|
init_tenantDatabaseScope();
|
|
9166
9183
|
|
|
9184
|
+
// ../../src/core/events/outbox/metrics.ts
|
|
9185
|
+
init_dialect();
|
|
9186
|
+
init_repositoryConnection();
|
|
9187
|
+
init_transaction();
|
|
9188
|
+
init_databaseTenantContext();
|
|
9189
|
+
var specifications = [
|
|
9190
|
+
{ state: "pending_due", status: "pending", column: "available_at", predicate: "<=", age: true },
|
|
9191
|
+
{
|
|
9192
|
+
state: "pending_waiting",
|
|
9193
|
+
status: "pending",
|
|
9194
|
+
column: "available_at",
|
|
9195
|
+
predicate: ">",
|
|
9196
|
+
age: false
|
|
9197
|
+
},
|
|
9198
|
+
{
|
|
9199
|
+
state: "processing_active",
|
|
9200
|
+
status: "processing",
|
|
9201
|
+
column: "lease_until",
|
|
9202
|
+
predicate: ">",
|
|
9203
|
+
age: false
|
|
9204
|
+
},
|
|
9205
|
+
{
|
|
9206
|
+
state: "processing_expired",
|
|
9207
|
+
status: "processing",
|
|
9208
|
+
column: "lease_until",
|
|
9209
|
+
predicate: "<=",
|
|
9210
|
+
age: true
|
|
9211
|
+
},
|
|
9212
|
+
{
|
|
9213
|
+
state: "processing_unleased",
|
|
9214
|
+
status: "processing",
|
|
9215
|
+
column: "lease_until",
|
|
9216
|
+
predicate: "IS NULL",
|
|
9217
|
+
age: false
|
|
9218
|
+
},
|
|
9219
|
+
{ state: "failed", status: "failed", column: "available_at", predicate: "", age: false }
|
|
9220
|
+
];
|
|
9221
|
+
var snapshotQuery = `WITH snapshot_clock AS MATERIALIZED (
|
|
9222
|
+
SELECT CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT) AS now_ms
|
|
9223
|
+
), ${specifications.map((spec, index) => `sample_${index} AS (
|
|
9224
|
+
SELECT ${spec.column} AS timestamp_ms FROM strata_outbox_delivery
|
|
9225
|
+
WHERE status = '${spec.status}'${spec.predicate ? ` AND ${spec.column} ${spec.predicate}${spec.predicate === "IS NULL" ? "" : " (SELECT now_ms FROM snapshot_clock)"}` : ""}
|
|
9226
|
+
ORDER BY ${spec.column}, event_id, listener_name LIMIT $1
|
|
9227
|
+
)`).join(`,
|
|
9228
|
+
`)}
|
|
9229
|
+
${specifications.map((spec, index) => `SELECT '${spec.state}' AS state, COUNT(*) AS count,
|
|
9230
|
+
${spec.age ? `COALESCE(GREATEST(0, (SELECT now_ms FROM snapshot_clock) - MIN(timestamp_ms)), 0)` : "NULL::BIGINT"} AS lateness_ms
|
|
9231
|
+
FROM sample_${index}`).join(`
|
|
9232
|
+
UNION ALL
|
|
9233
|
+
`)}`;
|
|
9234
|
+
function createOutboxMetricsCollector(options = {}) {
|
|
9235
|
+
const timeoutMs = options.timeoutMs ?? 1000;
|
|
9236
|
+
const sampleLimit = options.sampleLimit ?? 500;
|
|
9237
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 5000)
|
|
9238
|
+
throw new TypeError("Outbox metrics timeout must be between 1 and 5000 milliseconds.");
|
|
9239
|
+
if (!Number.isSafeInteger(sampleLimit) || sampleLimit < 1 || sampleLimit > 1000)
|
|
9240
|
+
throw new TypeError("Outbox metrics sample limit must be between 1 and 1000.");
|
|
9241
|
+
let active;
|
|
9242
|
+
let closed = false;
|
|
9243
|
+
return {
|
|
9244
|
+
async collect() {
|
|
9245
|
+
if (closed)
|
|
9246
|
+
throw new Error("Outbox metrics collector is closed.");
|
|
9247
|
+
if (hasActiveTransaction())
|
|
9248
|
+
throw new Error("Outbox metrics require an independent transaction.");
|
|
9249
|
+
if (currentSqlDialect().driver !== "pgsql")
|
|
9250
|
+
throw new Error("Outbox metrics currently require Postgres.");
|
|
9251
|
+
if (active)
|
|
9252
|
+
throw new Error("Outbox metrics collection is still settling.");
|
|
9253
|
+
const deadline = performance.now() + timeoutMs;
|
|
9254
|
+
let expired = false;
|
|
9255
|
+
const acquisition = new AbortController;
|
|
9256
|
+
let timer;
|
|
9257
|
+
const remaining = () => {
|
|
9258
|
+
const value = Math.floor(deadline - performance.now());
|
|
9259
|
+
if (expired || value < 1)
|
|
9260
|
+
throw new Error("Outbox metrics collection timed out.");
|
|
9261
|
+
return value;
|
|
9262
|
+
};
|
|
9263
|
+
const work = runInTransaction(async () => {
|
|
9264
|
+
await repositoryConnection.unsafe("SELECT set_config('statement_timeout', $1, true)", [`${remaining()}ms`]);
|
|
9265
|
+
await repositoryConnection.unsafe("SET TRANSACTION READ ONLY");
|
|
9266
|
+
return runWithMigrationBypass(async () => {
|
|
9267
|
+
remaining();
|
|
9268
|
+
const rows = await repositoryConnection.unsafe(snapshotQuery, [sampleLimit + 1]);
|
|
9269
|
+
remaining();
|
|
9270
|
+
if (rows.length !== specifications.length)
|
|
9271
|
+
throw new Error("Invalid outbox metrics response.");
|
|
9272
|
+
const states = specifications.map((spec) => {
|
|
9273
|
+
const row = rows.find((value) => value.state === spec.state);
|
|
9274
|
+
const count = Number(row?.count);
|
|
9275
|
+
const lateness = row?.lateness_ms == null ? null : Number(row.lateness_ms);
|
|
9276
|
+
if (!Number.isSafeInteger(count) || count < 0 || count > sampleLimit + 1 || spec.age && (lateness === null || !Number.isFinite(lateness) || lateness < 0))
|
|
9277
|
+
throw new Error("Invalid outbox metrics values.");
|
|
9278
|
+
return {
|
|
9279
|
+
state: spec.state,
|
|
9280
|
+
count: Math.min(count, sampleLimit),
|
|
9281
|
+
capped: count > sampleLimit,
|
|
9282
|
+
actionableLatenessSeconds: lateness === null ? null : lateness / 1000
|
|
9283
|
+
};
|
|
9284
|
+
});
|
|
9285
|
+
return { sampleLimit, states };
|
|
9286
|
+
});
|
|
9287
|
+
}, { acquisitionSignal: acquisition.signal });
|
|
9288
|
+
active = work;
|
|
9289
|
+
work.finally(() => {
|
|
9290
|
+
if (active === work)
|
|
9291
|
+
active = undefined;
|
|
9292
|
+
}).catch(() => {});
|
|
9293
|
+
try {
|
|
9294
|
+
return await Promise.race([
|
|
9295
|
+
work,
|
|
9296
|
+
new Promise((_, reject) => {
|
|
9297
|
+
timer = setTimeout(() => {
|
|
9298
|
+
expired = true;
|
|
9299
|
+
const error = new Error("Outbox metrics collection timed out.");
|
|
9300
|
+
acquisition.abort(error);
|
|
9301
|
+
reject(error);
|
|
9302
|
+
}, timeoutMs);
|
|
9303
|
+
})
|
|
9304
|
+
]);
|
|
9305
|
+
} finally {
|
|
9306
|
+
if (timer !== undefined)
|
|
9307
|
+
clearTimeout(timer);
|
|
9308
|
+
}
|
|
9309
|
+
},
|
|
9310
|
+
async close() {
|
|
9311
|
+
closed = true;
|
|
9312
|
+
await active?.catch(() => {});
|
|
9313
|
+
}
|
|
9314
|
+
};
|
|
9315
|
+
}
|
|
9316
|
+
function renderOutboxMetrics(snapshot) {
|
|
9317
|
+
if (!Number.isSafeInteger(snapshot.sampleLimit) || snapshot.sampleLimit < 1 || snapshot.sampleLimit > 1000 || snapshot.states.length !== specifications.length)
|
|
9318
|
+
throw new TypeError("Invalid outbox metrics snapshot.");
|
|
9319
|
+
const lines = [
|
|
9320
|
+
"# HELP strata_outbox_collector_success Whether outbox collection succeeded.",
|
|
9321
|
+
"# TYPE strata_outbox_collector_success gauge",
|
|
9322
|
+
"strata_outbox_collector_success 1",
|
|
9323
|
+
"# HELP strata_outbox_deliveries_sample Bounded per-state delivery count; lower bound when capped.",
|
|
9324
|
+
"# TYPE strata_outbox_deliveries_sample gauge",
|
|
9325
|
+
"# HELP strata_outbox_sample_capped Whether the delivery sample is truncated.",
|
|
9326
|
+
"# TYPE strata_outbox_sample_capped gauge",
|
|
9327
|
+
"# HELP strata_outbox_actionable_lateness_seconds Oldest due-time or expired-lease lateness, not original event age.",
|
|
9328
|
+
"# TYPE strata_outbox_actionable_lateness_seconds gauge"
|
|
9329
|
+
];
|
|
9330
|
+
for (const spec of specifications) {
|
|
9331
|
+
const row = snapshot.states.find((value) => value.state === spec.state);
|
|
9332
|
+
if (!row || !Number.isSafeInteger(row.count) || row.count < 0 || row.count > snapshot.sampleLimit || typeof row.capped !== "boolean" || row.actionableLatenessSeconds !== null && (!Number.isFinite(row.actionableLatenessSeconds) || row.actionableLatenessSeconds < 0))
|
|
9333
|
+
throw new TypeError("Invalid outbox metrics snapshot.");
|
|
9334
|
+
lines.push(`strata_outbox_deliveries_sample{state="${spec.state}"} ${row.count}`);
|
|
9335
|
+
lines.push(`strata_outbox_sample_capped{state="${spec.state}"} ${Number(row.capped)}`);
|
|
9336
|
+
if (row.actionableLatenessSeconds !== null)
|
|
9337
|
+
lines.push(`strata_outbox_actionable_lateness_seconds{state="${spec.state}"} ${row.actionableLatenessSeconds}`);
|
|
9338
|
+
}
|
|
9339
|
+
return `${lines.join(`
|
|
9340
|
+
`)}
|
|
9341
|
+
`;
|
|
9342
|
+
}
|
|
9167
9343
|
// ../../src/core/events/outbox/migration.ts
|
|
9168
9344
|
function createOutboxMigration(name, driver, options = {}) {
|
|
9169
9345
|
const identity = driver === "mysql" ? " CHARACTER SET ascii COLLATE ascii_bin" : "";
|
|
@@ -15244,6 +15420,7 @@ export {
|
|
|
15244
15420
|
createMysqlPool,
|
|
15245
15421
|
createNotificationDispatcher,
|
|
15246
15422
|
createOidcHandshake,
|
|
15423
|
+
createOutboxMetricsCollector,
|
|
15247
15424
|
createOutboxMigration,
|
|
15248
15425
|
createProductionQueue,
|
|
15249
15426
|
createQueue,
|
|
@@ -15417,6 +15594,7 @@ export {
|
|
|
15417
15594
|
registerShutdownHandler,
|
|
15418
15595
|
renderKernelErrorChrome,
|
|
15419
15596
|
renderMarkdownMail,
|
|
15597
|
+
renderOutboxMetrics,
|
|
15420
15598
|
renderWebErrorHtml,
|
|
15421
15599
|
repositoryConnection,
|
|
15422
15600
|
requestIdMiddleware,
|