@getstrata/core 2.2.9 → 2.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/core/database/acquisitionMetrics.d.ts +23 -0
- package/dist/core/database/baseRepository.d.ts +5 -0
- package/dist/core/database/isolatedReadOnlyObservation.d.ts +15 -0
- package/dist/core/database/mysqlConnection.d.ts +3 -0
- package/dist/core/database/readOnlyCollector.d.ts +7 -0
- package/dist/core/database/sqliteConnection.d.ts +3 -0
- package/dist/core/database/transaction.d.ts +5 -0
- package/dist/core/events/outbox/metrics.d.ts +1 -1
- package/dist/core/queue/failedJobMetrics.d.ts +17 -0
- package/dist/core/queue/queueMetrics.d.ts +4 -0
- package/dist/core/tracing/health.d.ts +27 -0
- package/dist/core/tracing/otel.d.ts +2 -0
- package/dist/core/tracing/tracingMiddleware.d.ts +2 -0
- package/dist/entries/audit/exportAuditLogs.js +61 -3
- package/dist/entries/auth/scimAuthMiddleware.js +61 -3
- package/dist/entries/database/sqliteConnection.js +137 -1
- package/dist/entries/http/response.js +3 -0
- package/dist/entries/jobs/exportAuditLogsJob.js +61 -3
- package/dist/entries/queue/queueMetrics.js +1052 -7
- package/dist/entries/tenant/databaseTenantContext.js +61 -3
- package/dist/entries/tenant/tenantDatabaseScope.js +61 -3
- package/dist/entries/tracing/tracingMiddleware.js +188 -5
- package/dist/framework/public-api.d.ts +6 -0
- package/dist/index.js +704 -105
- package/package.json +2 -1
package/dist/index.js
CHANGED
|
@@ -687,6 +687,111 @@ var init_deferredModelEvents = __esm(() => {
|
|
|
687
687
|
deferredModelEvents = createAsyncContextStore("@getstrata/deferredModelEvents");
|
|
688
688
|
});
|
|
689
689
|
|
|
690
|
+
// ../../src/core/database/acquisitionMetrics.ts
|
|
691
|
+
function createTransactionAcquisitionMetrics() {
|
|
692
|
+
const state = {
|
|
693
|
+
inflight: 0,
|
|
694
|
+
outcomes: outcomes.map((outcome) => ({
|
|
695
|
+
outcome,
|
|
696
|
+
count: 0,
|
|
697
|
+
sumSeconds: 0,
|
|
698
|
+
buckets: ACQUISITION_BUCKETS_SECONDS.map(() => 0)
|
|
699
|
+
}))
|
|
700
|
+
};
|
|
701
|
+
const metrics = {
|
|
702
|
+
[metricsBrand]: true,
|
|
703
|
+
snapshot: () => ({
|
|
704
|
+
inflight: state.inflight,
|
|
705
|
+
outcomes: state.outcomes.map((histogram) => ({
|
|
706
|
+
...histogram,
|
|
707
|
+
buckets: [...histogram.buckets]
|
|
708
|
+
}))
|
|
709
|
+
})
|
|
710
|
+
};
|
|
711
|
+
states.set(metrics, state);
|
|
712
|
+
return metrics;
|
|
713
|
+
}
|
|
714
|
+
async function observeTransactionAcquisition(metrics, acquire, signal) {
|
|
715
|
+
const state = states.get(metrics);
|
|
716
|
+
if (!state)
|
|
717
|
+
throw new TypeError("Transaction acquisition metrics must be created by the framework factory.");
|
|
718
|
+
const started = performance.now();
|
|
719
|
+
state.inflight++;
|
|
720
|
+
let outcome = "acquired";
|
|
721
|
+
try {
|
|
722
|
+
return await acquire();
|
|
723
|
+
} catch (error) {
|
|
724
|
+
outcome = signal?.aborted ? "aborted" : "failed";
|
|
725
|
+
throw error;
|
|
726
|
+
} finally {
|
|
727
|
+
state.inflight--;
|
|
728
|
+
const elapsed = Math.max(0, performance.now() - started) / 1000;
|
|
729
|
+
const histogram = state.outcomes[outcomes.indexOf(outcome)];
|
|
730
|
+
if (histogram) {
|
|
731
|
+
histogram.count++;
|
|
732
|
+
histogram.sumSeconds += elapsed;
|
|
733
|
+
for (let index = 0;index < ACQUISITION_BUCKETS_SECONDS.length; index++) {
|
|
734
|
+
const bound = ACQUISITION_BUCKETS_SECONDS[index];
|
|
735
|
+
if (bound !== undefined && elapsed <= bound)
|
|
736
|
+
histogram.buckets[index] = (histogram.buckets[index] ?? 0) + 1;
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
function renderTransactionAcquisitionMetrics(snapshot) {
|
|
742
|
+
const invalid = () => new TypeError("Invalid transaction acquisition metrics.");
|
|
743
|
+
if (!Number.isSafeInteger(snapshot.inflight) || snapshot.inflight < 0 || snapshot.outcomes.length !== outcomes.length)
|
|
744
|
+
throw invalid();
|
|
745
|
+
const name = "strata_database_transaction_acquisition_duration_seconds";
|
|
746
|
+
const lines = [
|
|
747
|
+
"# HELP strata_database_acquisition_collector_success Whether transaction acquisition observation succeeded.",
|
|
748
|
+
"# TYPE strata_database_acquisition_collector_success gauge",
|
|
749
|
+
"strata_database_acquisition_collector_success 1",
|
|
750
|
+
"# HELP strata_database_transaction_acquisition_inflight Native transaction checkouts awaiting settlement in this process.",
|
|
751
|
+
"# TYPE strata_database_transaction_acquisition_inflight gauge",
|
|
752
|
+
`strata_database_transaction_acquisition_inflight ${snapshot.inflight}`,
|
|
753
|
+
`# HELP ${name} Native checkout duration, including pool wait and connection establishment, for explicitly observed transactions.`,
|
|
754
|
+
`# TYPE ${name} histogram`
|
|
755
|
+
];
|
|
756
|
+
for (let i = 0;i < outcomes.length; i++) {
|
|
757
|
+
const histogram = snapshot.outcomes[i];
|
|
758
|
+
if (!histogram || histogram.outcome !== outcomes[i] || !Number.isSafeInteger(histogram.count) || histogram.count < 0 || !Number.isFinite(histogram.sumSeconds) || histogram.sumSeconds < 0 || histogram.buckets.length !== ACQUISITION_BUCKETS_SECONDS.length)
|
|
759
|
+
throw invalid();
|
|
760
|
+
let previous = 0;
|
|
761
|
+
for (let index = 0;index < ACQUISITION_BUCKETS_SECONDS.length; index++) {
|
|
762
|
+
const value = histogram.buckets[index];
|
|
763
|
+
if (value === undefined || !Number.isSafeInteger(value) || value < previous || value > histogram.count)
|
|
764
|
+
throw invalid();
|
|
765
|
+
previous = value;
|
|
766
|
+
lines.push(`${name}_bucket{outcome="${histogram.outcome}",le="${ACQUISITION_BUCKETS_SECONDS[index]}"} ${value}`);
|
|
767
|
+
}
|
|
768
|
+
lines.push(`${name}_bucket{outcome="${histogram.outcome}",le="+Inf"} ${histogram.count}`, `${name}_count{outcome="${histogram.outcome}"} ${histogram.count}`, `${name}_sum{outcome="${histogram.outcome}"} ${histogram.sumSeconds}`);
|
|
769
|
+
}
|
|
770
|
+
return `${lines.join(`
|
|
771
|
+
`)}
|
|
772
|
+
`;
|
|
773
|
+
}
|
|
774
|
+
var ACQUISITION_BUCKETS_SECONDS, outcomes, metricsBrand, states;
|
|
775
|
+
var init_acquisitionMetrics = __esm(() => {
|
|
776
|
+
ACQUISITION_BUCKETS_SECONDS = [
|
|
777
|
+
0.001,
|
|
778
|
+
0.005,
|
|
779
|
+
0.01,
|
|
780
|
+
0.025,
|
|
781
|
+
0.05,
|
|
782
|
+
0.1,
|
|
783
|
+
0.25,
|
|
784
|
+
0.5,
|
|
785
|
+
1,
|
|
786
|
+
2.5,
|
|
787
|
+
5,
|
|
788
|
+
10
|
|
789
|
+
];
|
|
790
|
+
outcomes = ["acquired", "failed", "aborted"];
|
|
791
|
+
metricsBrand = Symbol("transaction acquisition metrics");
|
|
792
|
+
states = new WeakMap;
|
|
793
|
+
});
|
|
794
|
+
|
|
690
795
|
// ../../src/core/database/connection.ts
|
|
691
796
|
function createDatabaseConnection(source) {
|
|
692
797
|
return {
|
|
@@ -745,9 +850,13 @@ async function runInTransaction(operation, options = {}) {
|
|
|
745
850
|
throw new Error("Active database connection does not support transactions. Bind a client with begin() via bindDatabaseConnection.");
|
|
746
851
|
}
|
|
747
852
|
return await settleTransaction(() => runWithDeferredModelEvents(async () => {
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
853
|
+
let reserved;
|
|
854
|
+
if (signal || options.acquisitionMetrics) {
|
|
855
|
+
if (!pool.reserve)
|
|
856
|
+
throw new Error(signal ? "Active database connection does not support cancellable transaction acquisition." : "Active database connection does not support observable transaction acquisition.");
|
|
857
|
+
const reserve = pool.reserve.bind(pool);
|
|
858
|
+
reserved = options.acquisitionMetrics ? await observeTransactionAcquisition(options.acquisitionMetrics, () => reserve({ signal }), signal) : await reserve({ signal });
|
|
859
|
+
}
|
|
751
860
|
const connection = reserved ?? pool;
|
|
752
861
|
try {
|
|
753
862
|
signal?.throwIfAborted();
|
|
@@ -765,10 +874,12 @@ var transactionFrame, outerFrames;
|
|
|
765
874
|
var init_transaction = __esm(() => {
|
|
766
875
|
init_deferredModelEvents();
|
|
767
876
|
init_asyncContextStore();
|
|
877
|
+
init_acquisitionMetrics();
|
|
768
878
|
init_connection();
|
|
769
879
|
init_connectionContext();
|
|
770
880
|
init_repositoryConnection();
|
|
771
881
|
init_transactionControl();
|
|
882
|
+
init_acquisitionMetrics();
|
|
772
883
|
transactionFrame = createAsyncContextStore("@getstrata/transactionFrame");
|
|
773
884
|
outerFrames = new WeakMap;
|
|
774
885
|
});
|
|
@@ -4123,6 +4234,10 @@ var CACHE_TAGS = {
|
|
|
4123
4234
|
careers: "careers",
|
|
4124
4235
|
offers: "offers"
|
|
4125
4236
|
};
|
|
4237
|
+
|
|
4238
|
+
// ../../src/framework/public-api.ts
|
|
4239
|
+
init_acquisitionMetrics();
|
|
4240
|
+
|
|
4126
4241
|
// ../../src/core/database/baseRepository.ts
|
|
4127
4242
|
init_eventBus();
|
|
4128
4243
|
|
|
@@ -8194,6 +8309,125 @@ function registerModelRepository(model, repository) {
|
|
|
8194
8309
|
}
|
|
8195
8310
|
// ../../src/core/database/mysqlConnection.ts
|
|
8196
8311
|
init_connectionContext();
|
|
8312
|
+
|
|
8313
|
+
// ../../src/core/database/isolatedReadOnlyObservation.ts
|
|
8314
|
+
var childSource = `
|
|
8315
|
+
import { Database } from 'bun:sqlite';
|
|
8316
|
+
let db;
|
|
8317
|
+
let mysql;
|
|
8318
|
+
process.on('message', async message => {
|
|
8319
|
+
try {
|
|
8320
|
+
if (message.init) {
|
|
8321
|
+
if (message.driver === 'mysql') {
|
|
8322
|
+
const module = await import(message.module);
|
|
8323
|
+
mysql = await module.createConnection({ uri: message.url, timezone: 'Z', connectTimeout: message.timeoutMs });
|
|
8324
|
+
await mysql.query("SET time_zone = '+00:00'");
|
|
8325
|
+
await mysql.query('SET SESSION max_execution_time = ' + message.timeoutMs);
|
|
8326
|
+
await mysql.query('SET SESSION lock_wait_timeout = 1');
|
|
8327
|
+
await mysql.query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ');
|
|
8328
|
+
await mysql.query('START TRANSACTION READ ONLY');
|
|
8329
|
+
} else {
|
|
8330
|
+
db = new Database(message.filename, { readonly: true, create: false });
|
|
8331
|
+
db.exec('PRAGMA busy_timeout = 0');
|
|
8332
|
+
db.exec('PRAGMA query_only = ON');
|
|
8333
|
+
db.exec('BEGIN');
|
|
8334
|
+
}
|
|
8335
|
+
process.send({ ok: true, rows: [] });
|
|
8336
|
+
} else {
|
|
8337
|
+
const rows = mysql ? (message.params.length ? await mysql.execute(message.query, message.params) : await mysql.query(message.query))[0] : db.query(message.query).all(...message.params);
|
|
8338
|
+
process.send({ ok: true, rows });
|
|
8339
|
+
}
|
|
8340
|
+
} catch {
|
|
8341
|
+
// No SQL, file paths, payloads or driver exception text crosses the error channel.
|
|
8342
|
+
process.send({ ok: false });
|
|
8343
|
+
}
|
|
8344
|
+
});
|
|
8345
|
+
`;
|
|
8346
|
+
async function observeIsolatedReadOnly(profile, operation, options) {
|
|
8347
|
+
options.signal.throwIfAborted();
|
|
8348
|
+
if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1 || options.timeoutMs > 5000)
|
|
8349
|
+
throw new TypeError("SQL observation timeout must be between 1 and 5000 milliseconds.");
|
|
8350
|
+
const executable = Bun.which("bun", { PATH: process.env.PATH ?? "" });
|
|
8351
|
+
if (!executable)
|
|
8352
|
+
throw new Error("SQL observations require the Bun CLI on PATH.");
|
|
8353
|
+
let pending;
|
|
8354
|
+
let stopped = false;
|
|
8355
|
+
const { promise: aborted, reject: rejectAbort } = Promise.withResolvers();
|
|
8356
|
+
const fail = (error) => {
|
|
8357
|
+
pending?.reject(error);
|
|
8358
|
+
pending = undefined;
|
|
8359
|
+
};
|
|
8360
|
+
const child = Bun.spawn({
|
|
8361
|
+
cmd: [
|
|
8362
|
+
executable,
|
|
8363
|
+
"--no-env-file",
|
|
8364
|
+
`--config=${process.platform === "win32" ? "NUL" : "/dev/null"}`,
|
|
8365
|
+
"-e",
|
|
8366
|
+
childSource
|
|
8367
|
+
],
|
|
8368
|
+
env: {},
|
|
8369
|
+
stdout: "ignore",
|
|
8370
|
+
stderr: "ignore",
|
|
8371
|
+
serialization: "advanced",
|
|
8372
|
+
ipc(message) {
|
|
8373
|
+
const response = message;
|
|
8374
|
+
const request = pending;
|
|
8375
|
+
pending = undefined;
|
|
8376
|
+
if (response?.ok === true && Array.isArray(response.rows))
|
|
8377
|
+
request?.resolve(response.rows);
|
|
8378
|
+
else
|
|
8379
|
+
request?.reject(new Error("Read-only SQL observation failed."));
|
|
8380
|
+
}
|
|
8381
|
+
});
|
|
8382
|
+
const exited = child.exited.then(() => {
|
|
8383
|
+
fail(new Error("SQL observation process exited."));
|
|
8384
|
+
});
|
|
8385
|
+
const abort = (reason) => {
|
|
8386
|
+
stopped = true;
|
|
8387
|
+
child.kill("SIGKILL");
|
|
8388
|
+
fail(reason);
|
|
8389
|
+
rejectAbort(reason);
|
|
8390
|
+
};
|
|
8391
|
+
const onAbort = () => abort(options.signal.reason);
|
|
8392
|
+
options.signal.addEventListener("abort", onAbort, { once: true });
|
|
8393
|
+
const timer = setTimeout(() => abort(new Error("SQL observation timed out.")), options.timeoutMs);
|
|
8394
|
+
const request = (message) => {
|
|
8395
|
+
if (stopped)
|
|
8396
|
+
return Promise.reject(new Error("SQL observation is closed."));
|
|
8397
|
+
if (pending)
|
|
8398
|
+
return Promise.reject(new Error("Concurrent SQL observation queries are not supported."));
|
|
8399
|
+
return new Promise((resolve, reject) => {
|
|
8400
|
+
pending = { resolve, reject };
|
|
8401
|
+
child.send(message);
|
|
8402
|
+
});
|
|
8403
|
+
};
|
|
8404
|
+
try {
|
|
8405
|
+
if (options.signal.aborted)
|
|
8406
|
+
onAbort();
|
|
8407
|
+
const work = (async () => {
|
|
8408
|
+
await request({ init: true, ...profile, timeoutMs: options.timeoutMs });
|
|
8409
|
+
options.signal.throwIfAborted();
|
|
8410
|
+
return operation({
|
|
8411
|
+
async unsafe(query, params = []) {
|
|
8412
|
+
return await request({
|
|
8413
|
+
query,
|
|
8414
|
+
params: params.map((value) => profile.driver === "sqlite" && value instanceof Date ? value.toISOString() : value)
|
|
8415
|
+
});
|
|
8416
|
+
}
|
|
8417
|
+
});
|
|
8418
|
+
})();
|
|
8419
|
+
return await Promise.race([work, aborted]);
|
|
8420
|
+
} finally {
|
|
8421
|
+
stopped = true;
|
|
8422
|
+
clearTimeout(timer);
|
|
8423
|
+
options.signal.removeEventListener("abort", onAbort);
|
|
8424
|
+
child.kill("SIGKILL");
|
|
8425
|
+
fail(new Error("SQL observation is closed."));
|
|
8426
|
+
await exited;
|
|
8427
|
+
}
|
|
8428
|
+
}
|
|
8429
|
+
|
|
8430
|
+
// ../../src/core/database/mysqlConnection.ts
|
|
8197
8431
|
var MYSQL_SESSION_UTC = "SET time_zone = '+00:00'";
|
|
8198
8432
|
var mysqlModule;
|
|
8199
8433
|
var mysqlPending;
|
|
@@ -8253,8 +8487,10 @@ async function executeMysql(target, query, params) {
|
|
|
8253
8487
|
const [result] = params.length === 0 && target.query ? await target.query(query) : await target.execute(query, [...params]);
|
|
8254
8488
|
return rowsFromResult(result);
|
|
8255
8489
|
}
|
|
8256
|
-
function createMysqlAdapter(getPool, close) {
|
|
8490
|
+
function createMysqlAdapter(getPool, close, observer) {
|
|
8257
8491
|
const transactions = new WeakSet;
|
|
8492
|
+
const observations = new Set;
|
|
8493
|
+
let observationsClosed = false;
|
|
8258
8494
|
const connection = {
|
|
8259
8495
|
async unsafe(query, params = []) {
|
|
8260
8496
|
const active = getActiveDatabaseConnection(connection);
|
|
@@ -8306,7 +8542,27 @@ function createMysqlAdapter(getPool, close) {
|
|
|
8306
8542
|
reserved.release();
|
|
8307
8543
|
}
|
|
8308
8544
|
},
|
|
8309
|
-
|
|
8545
|
+
async observeReadOnly(operation, options) {
|
|
8546
|
+
if (observationsClosed)
|
|
8547
|
+
throw new Error("MySQL observations are closed.");
|
|
8548
|
+
if (!observer)
|
|
8549
|
+
throw new Error("MySQL observations require the URL-backed official adapter.");
|
|
8550
|
+
const work = observer(async (transaction) => {
|
|
8551
|
+
transactions.add(transaction);
|
|
8552
|
+
return await runWithDatabaseConnection(transaction, () => operation(transaction));
|
|
8553
|
+
}, options);
|
|
8554
|
+
observations.add(work);
|
|
8555
|
+
try {
|
|
8556
|
+
return await work;
|
|
8557
|
+
} finally {
|
|
8558
|
+
observations.delete(work);
|
|
8559
|
+
}
|
|
8560
|
+
},
|
|
8561
|
+
async close() {
|
|
8562
|
+
observationsClosed = true;
|
|
8563
|
+
await Promise.allSettled([...observations]);
|
|
8564
|
+
await close();
|
|
8565
|
+
}
|
|
8310
8566
|
};
|
|
8311
8567
|
return connection;
|
|
8312
8568
|
}
|
|
@@ -8355,6 +8611,10 @@ function createMysqlConnection(url) {
|
|
|
8355
8611
|
return;
|
|
8356
8612
|
});
|
|
8357
8613
|
await pool?.end();
|
|
8614
|
+
}, async (operation, options) => {
|
|
8615
|
+
await loadMysql();
|
|
8616
|
+
const module = import.meta.resolve("mysql2/promise");
|
|
8617
|
+
return await observeIsolatedReadOnly({ driver: "mysql", url, module }, operation, options);
|
|
8358
8618
|
});
|
|
8359
8619
|
}
|
|
8360
8620
|
// ../../src/core/database/namedConnections.ts
|
|
@@ -9081,6 +9341,7 @@ async function runSeedersFromDirectory(directory, db, options) {
|
|
|
9081
9341
|
// ../../src/core/database/sqliteConnection.ts
|
|
9082
9342
|
init_connectionContext();
|
|
9083
9343
|
import { Database } from "bun:sqlite";
|
|
9344
|
+
import { resolve } from "path";
|
|
9084
9345
|
function isRowReturning(sql) {
|
|
9085
9346
|
const upper = sql.replace(/\s+/g, " ").trim().toUpperCase();
|
|
9086
9347
|
if (upper.includes(" RETURNING ")) {
|
|
@@ -9092,7 +9353,8 @@ function createSqliteConnection(filename) {
|
|
|
9092
9353
|
if (!filename.trim()) {
|
|
9093
9354
|
throw new Error("SQLite path is not configured. Pass a filename or :memory:.");
|
|
9094
9355
|
}
|
|
9095
|
-
const
|
|
9356
|
+
const absoluteFilename = filename === ":memory:" ? filename : resolve(filename);
|
|
9357
|
+
const db = new Database(absoluteFilename, { create: true });
|
|
9096
9358
|
db.exec("PRAGMA foreign_keys = ON");
|
|
9097
9359
|
db.exec("PRAGMA busy_timeout = 5000");
|
|
9098
9360
|
if (filename !== ":memory:") {
|
|
@@ -9155,6 +9417,21 @@ function createSqliteConnection(filename) {
|
|
|
9155
9417
|
}
|
|
9156
9418
|
});
|
|
9157
9419
|
},
|
|
9420
|
+
async observeReadOnly(operation, options) {
|
|
9421
|
+
if (closed)
|
|
9422
|
+
throw new Error("SQLite connection is closed.");
|
|
9423
|
+
if (absoluteFilename === ":memory:")
|
|
9424
|
+
throw new Error("SQL observations require file-backed SQLite.");
|
|
9425
|
+
pending++;
|
|
9426
|
+
try {
|
|
9427
|
+
return await observeIsolatedReadOnly({ driver: "sqlite", filename: absoluteFilename }, async (transaction) => {
|
|
9428
|
+
transactions.add(transaction);
|
|
9429
|
+
return await runWithDatabaseConnection(transaction, () => operation(transaction));
|
|
9430
|
+
}, options);
|
|
9431
|
+
} finally {
|
|
9432
|
+
pending--;
|
|
9433
|
+
}
|
|
9434
|
+
},
|
|
9158
9435
|
close() {
|
|
9159
9436
|
if (pending > 0)
|
|
9160
9437
|
throw new Error("Drain SQLite operations before closing the connection.");
|
|
@@ -9183,8 +9460,89 @@ init_tenantDatabaseScope();
|
|
|
9183
9460
|
|
|
9184
9461
|
// ../../src/core/events/outbox/metrics.ts
|
|
9185
9462
|
init_dialect();
|
|
9463
|
+
|
|
9464
|
+
// ../../src/core/database/readOnlyCollector.ts
|
|
9465
|
+
init_connectionContext();
|
|
9466
|
+
init_dialect();
|
|
9186
9467
|
init_repositoryConnection();
|
|
9187
9468
|
init_transaction();
|
|
9469
|
+
function createReadOnlyCollector(name, timeoutMs, collect) {
|
|
9470
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 5000)
|
|
9471
|
+
throw new TypeError(`${name} timeout must be between 1 and 5000 milliseconds.`);
|
|
9472
|
+
let active;
|
|
9473
|
+
let closed = false;
|
|
9474
|
+
return {
|
|
9475
|
+
async collect() {
|
|
9476
|
+
if (closed)
|
|
9477
|
+
throw new Error(`${name} collector is closed.`);
|
|
9478
|
+
if (hasActiveTransaction() || hasActiveDatabaseConnection())
|
|
9479
|
+
throw new Error(`${name} require an independent transaction.`);
|
|
9480
|
+
const driver = currentSqlDialect().driver;
|
|
9481
|
+
const pool = resolveRepositoryConnection();
|
|
9482
|
+
if (driver !== "pgsql" && driver !== "sqlite" && driver !== "mysql")
|
|
9483
|
+
throw new Error(`${name} require Postgres or an interruptible SQLite/MySQL adapter.`);
|
|
9484
|
+
if (active)
|
|
9485
|
+
throw new Error(`${name} collection is still settling.`);
|
|
9486
|
+
const deadline = performance.now() + timeoutMs;
|
|
9487
|
+
let expired = false;
|
|
9488
|
+
const acquisition = new AbortController;
|
|
9489
|
+
let timer;
|
|
9490
|
+
const remaining = () => {
|
|
9491
|
+
const value = Math.floor(deadline - performance.now());
|
|
9492
|
+
if (expired || value < 1)
|
|
9493
|
+
throw new Error(`${name} collection timed out.`);
|
|
9494
|
+
return value;
|
|
9495
|
+
};
|
|
9496
|
+
const work = driver === "pgsql" ? runInTransaction(async () => {
|
|
9497
|
+
await repositoryConnection.unsafe("SELECT set_config('statement_timeout', $1, true)", [
|
|
9498
|
+
`${remaining()}ms`
|
|
9499
|
+
]);
|
|
9500
|
+
await repositoryConnection.unsafe("SET TRANSACTION READ ONLY");
|
|
9501
|
+
remaining();
|
|
9502
|
+
const result = await collect(remaining);
|
|
9503
|
+
remaining();
|
|
9504
|
+
return result;
|
|
9505
|
+
}, { acquisitionSignal: acquisition.signal }) : (async () => {
|
|
9506
|
+
if (!pool.observeReadOnly)
|
|
9507
|
+
throw new Error(`${name} require Postgres or an interruptible SQLite/MySQL adapter.`);
|
|
9508
|
+
return pool.observeReadOnly(async (connection) => await runWithDatabaseConnection(connection, async () => {
|
|
9509
|
+
remaining();
|
|
9510
|
+
const result = await collect(remaining);
|
|
9511
|
+
remaining();
|
|
9512
|
+
return result;
|
|
9513
|
+
}), { signal: acquisition.signal, timeoutMs: remaining() });
|
|
9514
|
+
})();
|
|
9515
|
+
active = work;
|
|
9516
|
+
work.finally(() => {
|
|
9517
|
+
if (active === work)
|
|
9518
|
+
active = undefined;
|
|
9519
|
+
}).catch(() => {});
|
|
9520
|
+
try {
|
|
9521
|
+
return await Promise.race([
|
|
9522
|
+
work,
|
|
9523
|
+
new Promise((_, reject) => {
|
|
9524
|
+
timer = setTimeout(() => {
|
|
9525
|
+
expired = true;
|
|
9526
|
+
const error = new Error(`${name} collection timed out.`);
|
|
9527
|
+
acquisition.abort(error);
|
|
9528
|
+
reject(error);
|
|
9529
|
+
}, timeoutMs);
|
|
9530
|
+
})
|
|
9531
|
+
]);
|
|
9532
|
+
} finally {
|
|
9533
|
+
if (timer !== undefined)
|
|
9534
|
+
clearTimeout(timer);
|
|
9535
|
+
}
|
|
9536
|
+
},
|
|
9537
|
+
async close() {
|
|
9538
|
+
closed = true;
|
|
9539
|
+
await active?.catch(() => {});
|
|
9540
|
+
}
|
|
9541
|
+
};
|
|
9542
|
+
}
|
|
9543
|
+
|
|
9544
|
+
// ../../src/core/events/outbox/metrics.ts
|
|
9545
|
+
init_repositoryConnection();
|
|
9188
9546
|
init_databaseTenantContext();
|
|
9189
9547
|
var specifications = [
|
|
9190
9548
|
{ state: "pending_due", status: "pending", column: "available_at", predicate: "<=", age: true },
|
|
@@ -9231,87 +9589,78 @@ ${specifications.map((spec, index) => `SELECT '${spec.state}' AS state, COUNT(*)
|
|
|
9231
9589
|
FROM sample_${index}`).join(`
|
|
9232
9590
|
UNION ALL
|
|
9233
9591
|
`)}`;
|
|
9592
|
+
async function readIndexedSnapshot(sampleLimit, remaining) {
|
|
9593
|
+
const mysql = currentSqlDialect().driver === "mysql";
|
|
9594
|
+
if (mysql)
|
|
9595
|
+
await repositoryConnection.unsafe("SELECT event_id FROM strata_outbox_delivery LIMIT 0");
|
|
9596
|
+
const schema = await repositoryConnection.unsafe(mysql ? "SELECT 1 AS valid FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='strata_outbox_delivery' AND TABLE_TYPE='BASE TABLE' AND ENGINE='InnoDB' LIMIT 1" : "SELECT 1 AS valid FROM main.sqlite_schema WHERE name='strata_outbox_delivery' AND type='table' AND rootpage>0 LIMIT 1");
|
|
9597
|
+
if (schema.length !== 1 || schema[0]?.valid !== 1)
|
|
9598
|
+
throw new Error("Outbox metrics require the standard delivery table.");
|
|
9599
|
+
for (const [index, column] of [
|
|
9600
|
+
["strata_outbox_due", "available_at"],
|
|
9601
|
+
["strata_outbox_expired", "lease_until"]
|
|
9602
|
+
]) {
|
|
9603
|
+
const rows = await repositoryConnection.unsafe(mysql ? "SELECT IF(SUB_PART IS NULL AND COLLATION='A' AND IS_VISIBLE='YES', COLUMN_NAME, NULL) AS name, SEQ_IN_INDEX-1 AS seqno FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='strata_outbox_delivery' AND INDEX_NAME=? ORDER BY SEQ_IN_INDEX LIMIT 5" : "SELECT name, seqno FROM pragma_index_info(?, 'main') ORDER BY seqno LIMIT 5", [index]);
|
|
9604
|
+
const expected = ["status", column, "event_id", "listener_name"];
|
|
9605
|
+
if (rows.length !== expected.length || rows.some((row, i) => row.seqno !== i || row.name !== expected[i]))
|
|
9606
|
+
throw new Error("Outbox metrics require the standard delivery indexes.");
|
|
9607
|
+
remaining();
|
|
9608
|
+
}
|
|
9609
|
+
const [clock] = await repositoryConnection.unsafe(mysql ? "SELECT CAST(UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3))*1000 AS UNSIGNED) AS now_ms" : "SELECT CAST(unixepoch('subsec') * 1000 AS INTEGER) AS now_ms");
|
|
9610
|
+
if (!Number.isSafeInteger(clock?.now_ms) || !clock || clock.now_ms < 0)
|
|
9611
|
+
throw new Error("Invalid outbox clock.");
|
|
9612
|
+
const rows = [];
|
|
9613
|
+
for (const spec of specifications) {
|
|
9614
|
+
remaining();
|
|
9615
|
+
const index = spec.column === "lease_until" ? "strata_outbox_expired" : "strata_outbox_due";
|
|
9616
|
+
const predicate = spec.predicate ? ` AND ${spec.column} ${spec.predicate}${spec.predicate === "IS NULL" ? "" : " ?"}` : "";
|
|
9617
|
+
const values = await repositoryConnection.unsafe(`SELECT ${spec.column} AS timestamp_ms
|
|
9618
|
+
FROM strata_outbox_delivery ${mysql ? `FORCE INDEX (${index})` : `INDEXED BY ${index}`} WHERE status = ?${predicate}
|
|
9619
|
+
ORDER BY ${spec.column}, event_id, listener_name LIMIT ?`, [
|
|
9620
|
+
spec.status,
|
|
9621
|
+
...spec.predicate && spec.predicate !== "IS NULL" ? [clock.now_ms] : [],
|
|
9622
|
+
sampleLimit + 1
|
|
9623
|
+
]);
|
|
9624
|
+
const timestamp = values[0]?.timestamp_ms;
|
|
9625
|
+
if (spec.age && values.length && (!Number.isSafeInteger(timestamp) || timestamp == null || timestamp < 0))
|
|
9626
|
+
throw new Error("Invalid outbox timestamp.");
|
|
9627
|
+
rows.push({
|
|
9628
|
+
state: spec.state,
|
|
9629
|
+
count: values.length,
|
|
9630
|
+
lateness_ms: spec.age ? Math.max(0, clock.now_ms - (timestamp ?? clock.now_ms)) : null
|
|
9631
|
+
});
|
|
9632
|
+
}
|
|
9633
|
+
return rows;
|
|
9634
|
+
}
|
|
9234
9635
|
function createOutboxMetricsCollector(options = {}) {
|
|
9235
9636
|
const timeoutMs = options.timeoutMs ?? 1000;
|
|
9236
9637
|
const sampleLimit = options.sampleLimit ?? 500;
|
|
9237
|
-
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 5000)
|
|
9238
|
-
throw new TypeError("Outbox metrics timeout must be between 1 and 5000 milliseconds.");
|
|
9239
9638
|
if (!Number.isSafeInteger(sampleLimit) || sampleLimit < 1 || sampleLimit > 1000)
|
|
9240
9639
|
throw new TypeError("Outbox metrics sample limit must be between 1 and 1000.");
|
|
9241
|
-
|
|
9242
|
-
|
|
9243
|
-
|
|
9244
|
-
|
|
9245
|
-
|
|
9246
|
-
|
|
9247
|
-
|
|
9248
|
-
|
|
9249
|
-
|
|
9250
|
-
|
|
9251
|
-
|
|
9252
|
-
|
|
9253
|
-
|
|
9254
|
-
|
|
9255
|
-
|
|
9256
|
-
|
|
9257
|
-
|
|
9258
|
-
|
|
9259
|
-
|
|
9260
|
-
|
|
9261
|
-
|
|
9262
|
-
|
|
9263
|
-
|
|
9264
|
-
|
|
9265
|
-
await repositoryConnection.unsafe("SET TRANSACTION READ ONLY");
|
|
9266
|
-
return runWithMigrationBypass(async () => {
|
|
9267
|
-
remaining();
|
|
9268
|
-
const rows = await repositoryConnection.unsafe(snapshotQuery, [sampleLimit + 1]);
|
|
9269
|
-
remaining();
|
|
9270
|
-
if (rows.length !== specifications.length)
|
|
9271
|
-
throw new Error("Invalid outbox metrics response.");
|
|
9272
|
-
const states = specifications.map((spec) => {
|
|
9273
|
-
const row = rows.find((value) => value.state === spec.state);
|
|
9274
|
-
const count = Number(row?.count);
|
|
9275
|
-
const lateness = row?.lateness_ms == null ? null : Number(row.lateness_ms);
|
|
9276
|
-
if (!Number.isSafeInteger(count) || count < 0 || count > sampleLimit + 1 || spec.age && (lateness === null || !Number.isFinite(lateness) || lateness < 0))
|
|
9277
|
-
throw new Error("Invalid outbox metrics values.");
|
|
9278
|
-
return {
|
|
9279
|
-
state: spec.state,
|
|
9280
|
-
count: Math.min(count, sampleLimit),
|
|
9281
|
-
capped: count > sampleLimit,
|
|
9282
|
-
actionableLatenessSeconds: lateness === null ? null : lateness / 1000
|
|
9283
|
-
};
|
|
9284
|
-
});
|
|
9285
|
-
return { sampleLimit, states };
|
|
9286
|
-
});
|
|
9287
|
-
}, { acquisitionSignal: acquisition.signal });
|
|
9288
|
-
active = work;
|
|
9289
|
-
work.finally(() => {
|
|
9290
|
-
if (active === work)
|
|
9291
|
-
active = undefined;
|
|
9292
|
-
}).catch(() => {});
|
|
9293
|
-
try {
|
|
9294
|
-
return await Promise.race([
|
|
9295
|
-
work,
|
|
9296
|
-
new Promise((_, reject) => {
|
|
9297
|
-
timer = setTimeout(() => {
|
|
9298
|
-
expired = true;
|
|
9299
|
-
const error = new Error("Outbox metrics collection timed out.");
|
|
9300
|
-
acquisition.abort(error);
|
|
9301
|
-
reject(error);
|
|
9302
|
-
}, timeoutMs);
|
|
9303
|
-
})
|
|
9304
|
-
]);
|
|
9305
|
-
} finally {
|
|
9306
|
-
if (timer !== undefined)
|
|
9307
|
-
clearTimeout(timer);
|
|
9308
|
-
}
|
|
9309
|
-
},
|
|
9310
|
-
async close() {
|
|
9311
|
-
closed = true;
|
|
9312
|
-
await active?.catch(() => {});
|
|
9313
|
-
}
|
|
9314
|
-
};
|
|
9640
|
+
return createReadOnlyCollector("Outbox metrics", timeoutMs, async (remaining) => {
|
|
9641
|
+
const read = async () => {
|
|
9642
|
+
remaining();
|
|
9643
|
+
const rows = currentSqlDialect().driver !== "pgsql" ? await readIndexedSnapshot(sampleLimit, remaining) : await repositoryConnection.unsafe(snapshotQuery, [sampleLimit + 1]);
|
|
9644
|
+
remaining();
|
|
9645
|
+
if (rows.length !== specifications.length)
|
|
9646
|
+
throw new Error("Invalid outbox metrics response.");
|
|
9647
|
+
const states = specifications.map((spec) => {
|
|
9648
|
+
const row = rows.find((value) => value.state === spec.state);
|
|
9649
|
+
const count = Number(row?.count);
|
|
9650
|
+
const lateness = row?.lateness_ms == null ? null : Number(row.lateness_ms);
|
|
9651
|
+
if (!Number.isSafeInteger(count) || count < 0 || count > sampleLimit + 1 || spec.age && (lateness === null || !Number.isFinite(lateness) || lateness < 0))
|
|
9652
|
+
throw new Error("Invalid outbox metrics values.");
|
|
9653
|
+
return {
|
|
9654
|
+
state: spec.state,
|
|
9655
|
+
count: Math.min(count, sampleLimit),
|
|
9656
|
+
capped: count > sampleLimit,
|
|
9657
|
+
actionableLatenessSeconds: lateness === null ? null : lateness / 1000
|
|
9658
|
+
};
|
|
9659
|
+
});
|
|
9660
|
+
return { sampleLimit, states };
|
|
9661
|
+
};
|
|
9662
|
+
return currentSqlDialect().driver === "pgsql" ? runWithMigrationBypass(read) : read();
|
|
9663
|
+
});
|
|
9315
9664
|
}
|
|
9316
9665
|
function renderOutboxMetrics(snapshot) {
|
|
9317
9666
|
if (!Number.isSafeInteger(snapshot.sampleLimit) || snapshot.sampleLimit < 1 || snapshot.sampleLimit > 1000 || snapshot.states.length !== specifications.length)
|
|
@@ -9909,7 +10258,7 @@ import { dirname } from "path";
|
|
|
9909
10258
|
var {S3Client } = globalThis.Bun;
|
|
9910
10259
|
|
|
9911
10260
|
// ../../src/core/security/safePath.ts
|
|
9912
|
-
import { isAbsolute, resolve, sep } from "path";
|
|
10261
|
+
import { isAbsolute, resolve as resolve2, sep } from "path";
|
|
9913
10262
|
function assertPathUnderRoot(rootDirectory, userPath) {
|
|
9914
10263
|
const trimmed = userPath.trim();
|
|
9915
10264
|
if (trimmed.length === 0) {
|
|
@@ -9918,8 +10267,8 @@ function assertPathUnderRoot(rootDirectory, userPath) {
|
|
|
9918
10267
|
if (trimmed.includes("\x00")) {
|
|
9919
10268
|
throw new Error("Path must not contain a null byte.");
|
|
9920
10269
|
}
|
|
9921
|
-
const root =
|
|
9922
|
-
const candidate = isAbsolute(trimmed) ?
|
|
10270
|
+
const root = resolve2(rootDirectory);
|
|
10271
|
+
const candidate = isAbsolute(trimmed) ? resolve2(trimmed) : resolve2(root, trimmed.replace(/^\/+/u, ""));
|
|
9923
10272
|
if (candidate === root || candidate.startsWith(`${root}${sep}`)) {
|
|
9924
10273
|
return candidate;
|
|
9925
10274
|
}
|
|
@@ -12876,6 +13225,86 @@ class Notification {
|
|
|
12876
13225
|
return null;
|
|
12877
13226
|
}
|
|
12878
13227
|
}
|
|
13228
|
+
// ../../src/core/queue/failedJobMetrics.ts
|
|
13229
|
+
init_dialect();
|
|
13230
|
+
init_repositoryConnection();
|
|
13231
|
+
|
|
13232
|
+
// ../../src/core/queue/failedJobTable.ts
|
|
13233
|
+
var failedJobTable = defineTable({
|
|
13234
|
+
name: "failed_job",
|
|
13235
|
+
primaryKey: "id",
|
|
13236
|
+
columns: ["id", "job_name", "job_id", "payload", "exception", "failed_at"],
|
|
13237
|
+
defaultOrderBy: { column: "failed_at", direction: "DESC" }
|
|
13238
|
+
});
|
|
13239
|
+
|
|
13240
|
+
// ../../src/core/queue/failedJobRepository.ts
|
|
13241
|
+
class FailedJobRepository extends BaseRepository {
|
|
13242
|
+
constructor() {
|
|
13243
|
+
super(failedJobTable);
|
|
13244
|
+
}
|
|
13245
|
+
}
|
|
13246
|
+
var failedJobRepository_default = FailedJobRepository;
|
|
13247
|
+
|
|
13248
|
+
// ../../src/core/queue/failedJobMetrics.ts
|
|
13249
|
+
var schemaQuery = `SELECT c.relkind = 'r' AND c.relpersistence = 'p' AND NOT c.relrowsecurity AND EXISTS (
|
|
13250
|
+
SELECT 1 FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = i.indkey[0]
|
|
13251
|
+
WHERE i.indrelid = c.oid AND i.indisprimary AND i.indisvalid AND i.indisready
|
|
13252
|
+
AND i.indnkeyatts = 1 AND a.attname = 'id' AND NOT a.attisdropped
|
|
13253
|
+
AND a.atttypid IN ('int2'::regtype, 'int4'::regtype, 'int8'::regtype)
|
|
13254
|
+
) AS supported FROM pg_class c WHERE c.oid = 'failed_job'::regclass`;
|
|
13255
|
+
var sqliteSchemaQuery = `SELECT 1 AS supported FROM main.sqlite_schema
|
|
13256
|
+
WHERE name='failed_job' AND type='table' AND rootpage > 0
|
|
13257
|
+
AND (SELECT COUNT(*) FROM pragma_table_xinfo('failed_job', 'main') WHERE pk > 0) = 1
|
|
13258
|
+
AND EXISTS (SELECT 1 FROM pragma_table_xinfo('failed_job', 'main')
|
|
13259
|
+
WHERE name='id' AND pk=1 AND upper(type)='INTEGER' AND hidden=0) LIMIT 1`;
|
|
13260
|
+
var mysqlSchemaQuery = `SELECT 1 AS supported FROM information_schema.TABLES t
|
|
13261
|
+
WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME='failed_job' AND t.TABLE_TYPE='BASE TABLE' AND t.ENGINE='InnoDB'
|
|
13262
|
+
AND (SELECT COUNT(*) FROM information_schema.STATISTICS i WHERE i.TABLE_SCHEMA=t.TABLE_SCHEMA
|
|
13263
|
+
AND i.TABLE_NAME=t.TABLE_NAME AND i.INDEX_NAME='PRIMARY')=1
|
|
13264
|
+
AND EXISTS(SELECT 1 FROM information_schema.STATISTICS i JOIN information_schema.COLUMNS c
|
|
13265
|
+
ON c.TABLE_SCHEMA=i.TABLE_SCHEMA AND c.TABLE_NAME=i.TABLE_NAME AND c.COLUMN_NAME=i.COLUMN_NAME
|
|
13266
|
+
WHERE i.TABLE_SCHEMA=t.TABLE_SCHEMA AND i.TABLE_NAME=t.TABLE_NAME AND i.INDEX_NAME='PRIMARY'
|
|
13267
|
+
AND i.SEQ_IN_INDEX=1 AND i.COLUMN_NAME='id' AND c.DATA_TYPE IN ('tinyint','smallint','mediumint','int','bigint')) LIMIT 1`;
|
|
13268
|
+
function createFailedJobMetricsCollector(options = {}) {
|
|
13269
|
+
const sampleLimit = options.sampleLimit ?? 500;
|
|
13270
|
+
if (!Number.isSafeInteger(sampleLimit) || sampleLimit < 1 || sampleLimit > 1000)
|
|
13271
|
+
throw new TypeError("Failed-job metrics sample limit must be between 1 and 1000.");
|
|
13272
|
+
const repository = new failedJobRepository_default;
|
|
13273
|
+
return createReadOnlyCollector("Failed-job metrics", options.timeoutMs ?? 1000, async (remaining) => {
|
|
13274
|
+
const driver = currentSqlDialect().driver;
|
|
13275
|
+
const postgres = driver === "pgsql";
|
|
13276
|
+
if (postgres)
|
|
13277
|
+
await repositoryConnection.unsafe('LOCK TABLE "failed_job" IN ACCESS SHARE MODE');
|
|
13278
|
+
if (driver === "mysql")
|
|
13279
|
+
await repositoryConnection.unsafe("SELECT id FROM failed_job LIMIT 0");
|
|
13280
|
+
const schema = await repositoryConnection.unsafe(postgres ? schemaQuery : driver === "mysql" ? mysqlSchemaQuery : sqliteSchemaQuery);
|
|
13281
|
+
if (schema.length !== 1 || schema[0]?.supported !== (postgres ? true : 1))
|
|
13282
|
+
throw new Error("Failed-job metrics require the standard global table with an ID primary key and no RLS.");
|
|
13283
|
+
remaining();
|
|
13284
|
+
const rows = await repository.query().orderBy({ column: "id", direction: "ASC" }).project(["id"], sampleLimit + 1);
|
|
13285
|
+
if (rows.length > sampleLimit + 1 || rows.some((row) => row.id == null))
|
|
13286
|
+
throw new Error("Invalid failed-job metrics response.");
|
|
13287
|
+
return {
|
|
13288
|
+
sampleLimit,
|
|
13289
|
+
count: Math.min(rows.length, sampleLimit),
|
|
13290
|
+
capped: rows.length > sampleLimit
|
|
13291
|
+
};
|
|
13292
|
+
});
|
|
13293
|
+
}
|
|
13294
|
+
function renderFailedJobMetrics(snapshot) {
|
|
13295
|
+
if (!Number.isSafeInteger(snapshot.sampleLimit) || snapshot.sampleLimit < 1 || snapshot.sampleLimit > 1000 || !Number.isSafeInteger(snapshot.count) || snapshot.count < 0 || snapshot.count > snapshot.sampleLimit || typeof snapshot.capped !== "boolean" || snapshot.capped && snapshot.count !== snapshot.sampleLimit)
|
|
13296
|
+
throw new TypeError("Invalid failed-job metrics snapshot.");
|
|
13297
|
+
return `# HELP strata_queue_failed_job_collector_success Whether failed-job collection succeeded.
|
|
13298
|
+
# TYPE strata_queue_failed_job_collector_success gauge
|
|
13299
|
+
strata_queue_failed_job_collector_success 1
|
|
13300
|
+
# HELP strata_queue_failed_jobs_sample Capped retained failed-job count; a lower bound when capped.
|
|
13301
|
+
# TYPE strata_queue_failed_jobs_sample gauge
|
|
13302
|
+
strata_queue_failed_jobs_sample ${snapshot.count}
|
|
13303
|
+
# HELP strata_queue_failed_jobs_sample_capped Whether the failed-job sample is truncated.
|
|
13304
|
+
# TYPE strata_queue_failed_jobs_sample_capped gauge
|
|
13305
|
+
strata_queue_failed_jobs_sample_capped ${Number(snapshot.capped)}
|
|
13306
|
+
`;
|
|
13307
|
+
}
|
|
12879
13308
|
// ../../src/core/queue/jobRegistry.ts
|
|
12880
13309
|
class JobRegistry {
|
|
12881
13310
|
factories = new Map;
|
|
@@ -12970,22 +13399,6 @@ class AsyncQueue {
|
|
|
12970
13399
|
function createQueue(driver) {
|
|
12971
13400
|
return driver === "async" ? new AsyncQueue : new SyncQueue;
|
|
12972
13401
|
}
|
|
12973
|
-
// ../../src/core/queue/failedJobTable.ts
|
|
12974
|
-
var failedJobTable = defineTable({
|
|
12975
|
-
name: "failed_job",
|
|
12976
|
-
primaryKey: "id",
|
|
12977
|
-
columns: ["id", "job_name", "job_id", "payload", "exception", "failed_at"],
|
|
12978
|
-
defaultOrderBy: { column: "failed_at", direction: "DESC" }
|
|
12979
|
-
});
|
|
12980
|
-
|
|
12981
|
-
// ../../src/core/queue/failedJobRepository.ts
|
|
12982
|
-
class FailedJobRepository extends BaseRepository {
|
|
12983
|
-
constructor() {
|
|
12984
|
-
super(failedJobTable);
|
|
12985
|
-
}
|
|
12986
|
-
}
|
|
12987
|
-
var failedJobRepository_default = FailedJobRepository;
|
|
12988
|
-
|
|
12989
13402
|
// ../../src/core/queue/failedJobService.ts
|
|
12990
13403
|
class FailedJobService {
|
|
12991
13404
|
repository;
|
|
@@ -14688,6 +15101,171 @@ function createTenantMiddleware(options = {}) {
|
|
|
14688
15101
|
}
|
|
14689
15102
|
};
|
|
14690
15103
|
}
|
|
15104
|
+
// ../../src/core/tracing/health.ts
|
|
15105
|
+
import {
|
|
15106
|
+
createNoopMeter
|
|
15107
|
+
} from "@opentelemetry/api";
|
|
15108
|
+
import { ExportResultCode } from "@opentelemetry/core";
|
|
15109
|
+
function observeExportDeadline(state, timeoutMillis) {
|
|
15110
|
+
const deadline = setTimeout(() => {
|
|
15111
|
+
state.exportDeadlineExceededBatches++;
|
|
15112
|
+
}, timeoutMillis);
|
|
15113
|
+
deadline.unref();
|
|
15114
|
+
return deadline;
|
|
15115
|
+
}
|
|
15116
|
+
function createTracingHealth(enabled, exporterConfigured, timeoutMillis) {
|
|
15117
|
+
const state = {
|
|
15118
|
+
enabled,
|
|
15119
|
+
exporterConfigured,
|
|
15120
|
+
active: true,
|
|
15121
|
+
queueSize: null,
|
|
15122
|
+
queueCapacity: null,
|
|
15123
|
+
droppedSpans: 0,
|
|
15124
|
+
exportedBatches: 0,
|
|
15125
|
+
failedBatches: 0,
|
|
15126
|
+
exportedSpans: 0,
|
|
15127
|
+
failedSpans: 0,
|
|
15128
|
+
exportDeadlineExceededBatches: 0,
|
|
15129
|
+
flushFailures: 0,
|
|
15130
|
+
shutdownFailures: 0
|
|
15131
|
+
};
|
|
15132
|
+
let exportQueueFullSpans = 0;
|
|
15133
|
+
let queueCallback;
|
|
15134
|
+
const noop = createNoopMeter();
|
|
15135
|
+
const meter = {
|
|
15136
|
+
createGauge: noop.createGauge.bind(noop),
|
|
15137
|
+
createHistogram: noop.createHistogram.bind(noop),
|
|
15138
|
+
createObservableGauge: noop.createObservableGauge.bind(noop),
|
|
15139
|
+
createObservableCounter: noop.createObservableCounter.bind(noop),
|
|
15140
|
+
addBatchObservableCallback: noop.addBatchObservableCallback.bind(noop),
|
|
15141
|
+
removeBatchObservableCallback: noop.removeBatchObservableCallback.bind(noop),
|
|
15142
|
+
createCounter(name, options) {
|
|
15143
|
+
if (name !== "otel.sdk.processor.span.processed")
|
|
15144
|
+
return noop.createCounter(name, options);
|
|
15145
|
+
return {
|
|
15146
|
+
add(value, attributes) {
|
|
15147
|
+
if (attributes?.["error.type"] === "queue_full")
|
|
15148
|
+
state.droppedSpans += value;
|
|
15149
|
+
}
|
|
15150
|
+
};
|
|
15151
|
+
},
|
|
15152
|
+
createUpDownCounter(name, options) {
|
|
15153
|
+
if (name !== "otel.sdk.processor.span.queue.capacity")
|
|
15154
|
+
return noop.createUpDownCounter(name, options);
|
|
15155
|
+
return {
|
|
15156
|
+
add(value) {
|
|
15157
|
+
state.queueCapacity = (state.queueCapacity ?? 0) + value;
|
|
15158
|
+
}
|
|
15159
|
+
};
|
|
15160
|
+
},
|
|
15161
|
+
createObservableUpDownCounter(name, options) {
|
|
15162
|
+
if (name !== "otel.sdk.processor.span.queue.size")
|
|
15163
|
+
return noop.createObservableUpDownCounter(name, options);
|
|
15164
|
+
return {
|
|
15165
|
+
addCallback(callback) {
|
|
15166
|
+
queueCallback = callback;
|
|
15167
|
+
},
|
|
15168
|
+
removeCallback(callback) {
|
|
15169
|
+
if (queueCallback === callback)
|
|
15170
|
+
queueCallback = undefined;
|
|
15171
|
+
}
|
|
15172
|
+
};
|
|
15173
|
+
}
|
|
15174
|
+
};
|
|
15175
|
+
const meterProvider = { getMeter: () => meter };
|
|
15176
|
+
return {
|
|
15177
|
+
state,
|
|
15178
|
+
meterProvider,
|
|
15179
|
+
snapshot() {
|
|
15180
|
+
let queueSize = null;
|
|
15181
|
+
queueCallback?.({
|
|
15182
|
+
observe(value) {
|
|
15183
|
+
queueSize = value;
|
|
15184
|
+
}
|
|
15185
|
+
});
|
|
15186
|
+
return {
|
|
15187
|
+
...state,
|
|
15188
|
+
queueSize,
|
|
15189
|
+
droppedSpans: Math.max(0, state.droppedSpans - exportQueueFullSpans)
|
|
15190
|
+
};
|
|
15191
|
+
},
|
|
15192
|
+
wrap(exporter) {
|
|
15193
|
+
return {
|
|
15194
|
+
export(spans, callback) {
|
|
15195
|
+
const count = spans.length;
|
|
15196
|
+
let completed = false;
|
|
15197
|
+
const deadline = observeExportDeadline(state, timeoutMillis);
|
|
15198
|
+
const finish = (success) => {
|
|
15199
|
+
completed = true;
|
|
15200
|
+
clearTimeout(deadline);
|
|
15201
|
+
if (success) {
|
|
15202
|
+
state.exportedBatches++;
|
|
15203
|
+
state.exportedSpans += count;
|
|
15204
|
+
} else {
|
|
15205
|
+
state.failedBatches++;
|
|
15206
|
+
state.failedSpans += count;
|
|
15207
|
+
}
|
|
15208
|
+
};
|
|
15209
|
+
try {
|
|
15210
|
+
exporter.export(spans, (result) => {
|
|
15211
|
+
if (completed)
|
|
15212
|
+
return;
|
|
15213
|
+
finish(result.code === ExportResultCode.SUCCESS);
|
|
15214
|
+
if (result.error?.name === "queue_full")
|
|
15215
|
+
exportQueueFullSpans += count;
|
|
15216
|
+
callback(result);
|
|
15217
|
+
});
|
|
15218
|
+
} catch (error) {
|
|
15219
|
+
if (!completed)
|
|
15220
|
+
finish(false);
|
|
15221
|
+
throw error;
|
|
15222
|
+
}
|
|
15223
|
+
},
|
|
15224
|
+
shutdown: () => exporter.shutdown(),
|
|
15225
|
+
...exporter.forceFlush ? { forceFlush: exporter.forceFlush.bind(exporter) } : {}
|
|
15226
|
+
};
|
|
15227
|
+
}
|
|
15228
|
+
};
|
|
15229
|
+
}
|
|
15230
|
+
function renderTracingMetrics(snapshot) {
|
|
15231
|
+
for (const key of ["enabled", "exporterConfigured", "active"])
|
|
15232
|
+
if (typeof snapshot[key] !== "boolean")
|
|
15233
|
+
throw new TypeError("Invalid tracing metrics.");
|
|
15234
|
+
const values = {
|
|
15235
|
+
strata_tracing_enabled: Number(snapshot.enabled),
|
|
15236
|
+
strata_tracing_exporter_configured: Number(snapshot.exporterConfigured),
|
|
15237
|
+
strata_tracing_active: Number(snapshot.active),
|
|
15238
|
+
strata_tracing_queue_size: snapshot.queueSize,
|
|
15239
|
+
strata_tracing_queue_capacity: snapshot.queueCapacity,
|
|
15240
|
+
strata_tracing_dropped_spans_total: snapshot.droppedSpans,
|
|
15241
|
+
strata_tracing_exported_batches_total: snapshot.exportedBatches,
|
|
15242
|
+
strata_tracing_failed_batches_total: snapshot.failedBatches,
|
|
15243
|
+
strata_tracing_exported_spans_total: snapshot.exportedSpans,
|
|
15244
|
+
strata_tracing_failed_spans_total: snapshot.failedSpans,
|
|
15245
|
+
strata_tracing_export_deadline_exceeded_batches_total: snapshot.exportDeadlineExceededBatches,
|
|
15246
|
+
strata_tracing_flush_failures_total: snapshot.flushFailures,
|
|
15247
|
+
strata_tracing_shutdown_failures_total: snapshot.shutdownFailures
|
|
15248
|
+
};
|
|
15249
|
+
let output = `# HELP strata_tracing_collector_success Whether tracing observation succeeded.
|
|
15250
|
+
# TYPE strata_tracing_collector_success gauge
|
|
15251
|
+
strata_tracing_collector_success 1
|
|
15252
|
+
`;
|
|
15253
|
+
for (const [name, value] of Object.entries(values)) {
|
|
15254
|
+
if (value === null) {
|
|
15255
|
+
if (name === "strata_tracing_queue_size" || name === "strata_tracing_queue_capacity")
|
|
15256
|
+
continue;
|
|
15257
|
+
throw new TypeError("Invalid tracing metrics.");
|
|
15258
|
+
}
|
|
15259
|
+
if (!Number.isSafeInteger(value) || value < 0)
|
|
15260
|
+
throw new TypeError("Invalid tracing metrics.");
|
|
15261
|
+
const type = name.endsWith("_total") ? "counter" : "gauge";
|
|
15262
|
+
output += `# HELP ${name} Process-local tracing ${name.slice(15).replaceAll("_", " ")}.
|
|
15263
|
+
# TYPE ${name} ${type}
|
|
15264
|
+
${name} ${value}
|
|
15265
|
+
`;
|
|
15266
|
+
}
|
|
15267
|
+
return output;
|
|
15268
|
+
}
|
|
14691
15269
|
// ../../src/core/tracing/traceContext.ts
|
|
14692
15270
|
init_asyncContextStore();
|
|
14693
15271
|
var traceContextStorage = createAsyncContextStore("@getstrata/traceContext");
|
|
@@ -14706,10 +15284,10 @@ import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-ho
|
|
|
14706
15284
|
import { W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
14707
15285
|
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
14708
15286
|
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
15287
|
+
import { BatchSpanProcessor } from "@opentelemetry/sdk-trace";
|
|
14709
15288
|
import {
|
|
14710
15289
|
AlwaysOffSampler,
|
|
14711
15290
|
AlwaysOnSampler,
|
|
14712
|
-
BatchSpanProcessor,
|
|
14713
15291
|
ParentBasedSampler,
|
|
14714
15292
|
TraceIdRatioBasedSampler
|
|
14715
15293
|
} from "@opentelemetry/sdk-trace-base";
|
|
@@ -14763,11 +15341,14 @@ function createTracingRuntime(options = {}) {
|
|
|
14763
15341
|
throw new TypeError("Invalid tracing collector endpoint.");
|
|
14764
15342
|
}
|
|
14765
15343
|
const exporter = disabled ? undefined : options.exporter ?? (endpoint ? new OTLPTraceExporter({ url: endpoint, timeoutMillis: timeout, concurrencyLimit: 1 }) : undefined);
|
|
15344
|
+
const health = createTracingHealth(!disabled, exporter !== undefined, timeout);
|
|
14766
15345
|
const provider = new NodeTracerProvider({
|
|
14767
15346
|
resource: resourceFromAttributes({ "service.name": options.serviceName ?? otelServiceName() }),
|
|
14768
15347
|
sampler,
|
|
14769
15348
|
spanProcessors: exporter ? [
|
|
14770
|
-
new BatchSpanProcessor(
|
|
15349
|
+
new BatchSpanProcessor({
|
|
15350
|
+
exporter: health.wrap(exporter),
|
|
15351
|
+
selfObsMeterProvider: health.meterProvider,
|
|
14771
15352
|
maxQueueSize: queueSize,
|
|
14772
15353
|
maxExportBatchSize: batchSize,
|
|
14773
15354
|
scheduledDelayMillis: delay,
|
|
@@ -14787,8 +15368,21 @@ function createTracingRuntime(options = {}) {
|
|
|
14787
15368
|
tracer: provider.getTracer("@getstrata/http"),
|
|
14788
15369
|
propagator: new W3CTraceContextPropagator,
|
|
14789
15370
|
run: (parent, callback) => context.with(parent, callback),
|
|
14790
|
-
|
|
14791
|
-
|
|
15371
|
+
metrics: () => health.snapshot(),
|
|
15372
|
+
async forceFlush() {
|
|
15373
|
+
try {
|
|
15374
|
+
await provider.forceFlush();
|
|
15375
|
+
} catch (error) {
|
|
15376
|
+
health.state.flushFailures++;
|
|
15377
|
+
throw error;
|
|
15378
|
+
}
|
|
15379
|
+
},
|
|
15380
|
+
shutdown: () => shutdown ??= provider.shutdown().catch((error) => {
|
|
15381
|
+
health.state.shutdownFailures++;
|
|
15382
|
+
throw error;
|
|
15383
|
+
}).finally(() => {
|
|
15384
|
+
health.state.active = false;
|
|
15385
|
+
})
|
|
14792
15386
|
};
|
|
14793
15387
|
}
|
|
14794
15388
|
var runtimeKey = Symbol.for("@getstrata/otel-runtime");
|
|
@@ -15408,6 +16002,7 @@ export {
|
|
|
15408
16002
|
createCsrfTokenCookie,
|
|
15409
16003
|
createDatabaseConnection,
|
|
15410
16004
|
createDatabaseQueryProxy,
|
|
16005
|
+
createFailedJobMetricsCollector,
|
|
15411
16006
|
createFailedJobService,
|
|
15412
16007
|
createFlashMiddleware,
|
|
15413
16008
|
createLoginThrottleMiddleware,
|
|
@@ -15444,6 +16039,7 @@ export {
|
|
|
15444
16039
|
createTracingMiddleware,
|
|
15445
16040
|
createTracingRuntime,
|
|
15446
16041
|
createTrackedJob,
|
|
16042
|
+
createTransactionAcquisitionMetrics,
|
|
15447
16043
|
createValidateSignatureMiddleware,
|
|
15448
16044
|
createdResponse,
|
|
15449
16045
|
currentAuthUser,
|
|
@@ -15592,9 +16188,12 @@ export {
|
|
|
15592
16188
|
registerModelRepository,
|
|
15593
16189
|
registerNamedConnection,
|
|
15594
16190
|
registerShutdownHandler,
|
|
16191
|
+
renderFailedJobMetrics,
|
|
15595
16192
|
renderKernelErrorChrome,
|
|
15596
16193
|
renderMarkdownMail,
|
|
15597
16194
|
renderOutboxMetrics,
|
|
16195
|
+
renderTracingMetrics,
|
|
16196
|
+
renderTransactionAcquisitionMetrics,
|
|
15598
16197
|
renderWebErrorHtml,
|
|
15599
16198
|
repositoryConnection,
|
|
15600
16199
|
requestIdMiddleware,
|