deepline 0.3.293 → 0.3.295
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/compat.ts +35 -7
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +269 -267
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +3 -4
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +14 -1
- package/dist/cli/auth-EDTRERZP.mjs +31 -0
- package/dist/cli/auth-TDHKB277.js +31 -0
- package/dist/cli/billing-ELIDPILY.js +1749 -0
- package/dist/cli/billing-EXGXJVAU.mjs +1749 -0
- package/dist/cli/chunk-2S5Y3TZP.js +5688 -0
- package/dist/cli/chunk-3KC45UVF.js +150 -0
- package/dist/cli/chunk-3XX5TEA4.js +15 -0
- package/dist/cli/chunk-4MBOJQPL.mjs +0 -0
- package/dist/cli/chunk-4NEFYBF7.js +2240 -0
- package/dist/cli/chunk-5JLQPVNX.js +3056 -0
- package/dist/cli/chunk-5KDKFHZI.js +1269 -0
- package/dist/cli/chunk-6A3TJUWC.js +76 -0
- package/dist/cli/chunk-6O5YZOVN.mjs +890 -0
- package/dist/cli/chunk-7P5AA3SN.js +344 -0
- package/dist/cli/chunk-AYHLUHE5.js +52 -0
- package/dist/cli/chunk-CBVIJSHF.mjs +560 -0
- package/dist/cli/chunk-CFVVCN4A.js +735 -0
- package/dist/cli/chunk-COGTRVWW.js +30 -0
- package/dist/cli/chunk-D4AZUZO4.js +190 -0
- package/dist/cli/chunk-DLIRSHV6.mjs +28 -0
- package/dist/cli/chunk-EACVZI6N.mjs +1995 -0
- package/dist/cli/chunk-EDWJY6T4.mjs +3056 -0
- package/dist/cli/chunk-EXMLR72R.mjs +0 -0
- package/dist/cli/chunk-F35VOIQL.js +890 -0
- package/dist/cli/chunk-GAMFK2ZP.mjs +148 -0
- package/dist/cli/chunk-GHRQQPW2.mjs +2240 -0
- package/dist/cli/chunk-GONRLVV3.js +3440 -0
- package/dist/cli/chunk-GVUJJJ5G.mjs +76 -0
- package/dist/cli/chunk-H7HIDGZ7.mjs +150 -0
- package/dist/cli/chunk-HUZXWOSK.mjs +503 -0
- package/dist/cli/chunk-I5YZLGMG.js +1 -0
- package/dist/cli/chunk-ITOFSXIU.mjs +15 -0
- package/dist/cli/chunk-J5NA2TTX.js +503 -0
- package/dist/cli/chunk-JCS5TPNY.mjs +3440 -0
- package/dist/cli/chunk-JNNOCMQY.js +47 -0
- package/dist/cli/chunk-KRD6H754.mjs +30 -0
- package/dist/cli/chunk-LA46MMTL.js +560 -0
- package/dist/cli/chunk-LDWT7RJK.mjs +190 -0
- package/dist/cli/chunk-M2V26OR5.js +499 -0
- package/dist/cli/chunk-MIMASXO4.mjs +389 -0
- package/dist/cli/chunk-MROPCKW2.mjs +190 -0
- package/dist/cli/chunk-NXDRDJ4K.js +389 -0
- package/dist/cli/chunk-NXGCO4JB.mjs +344 -0
- package/dist/cli/chunk-OG64RBAC.js +6 -0
- package/dist/cli/chunk-OQYEC4GL.mjs +34 -0
- package/dist/cli/chunk-OWJZUE3Y.mjs +103 -0
- package/dist/cli/chunk-SDK3ZA2C.js +103 -0
- package/dist/cli/chunk-SFLARO4Y.mjs +47 -0
- package/dist/cli/chunk-SVIIIYIQ.js +923 -0
- package/dist/cli/chunk-TAC3JIFE.mjs +354 -0
- package/dist/cli/chunk-U3PI7HBM.js +34 -0
- package/dist/cli/chunk-UFS752TU.mjs +52 -0
- package/dist/cli/chunk-UI37DQE7.mjs +499 -0
- package/dist/cli/chunk-UNEMDKXN.js +1 -0
- package/dist/cli/chunk-WB6Z2Q44.mjs +1622 -0
- package/dist/cli/chunk-WJYPBHTE.js +148 -0
- package/dist/cli/chunk-WOHBA5PV.js +354 -0
- package/dist/cli/chunk-WQICC2B3.js +190 -0
- package/dist/cli/chunk-WZYKRD7T.mjs +6 -0
- package/dist/cli/chunk-XL2FO6CW.mjs +5688 -0
- package/dist/cli/chunk-XNYIZMNQ.js +28 -0
- package/dist/cli/chunk-XRK2TZT5.js +1995 -0
- package/dist/cli/chunk-YJYPL5OK.mjs +1269 -0
- package/dist/cli/chunk-YWNDYCXR.mjs +735 -0
- package/dist/cli/chunk-YWWAF7KZ.mjs +923 -0
- package/dist/cli/chunk-ZDO3TO3M.js +1622 -0
- package/dist/cli/client-4JVOYVC5.js +20 -0
- package/dist/cli/client-EEN2IJN2.mjs +20 -0
- package/dist/cli/csv-BOHBEP2I.mjs +585 -0
- package/dist/cli/csv-S5NYRUEX.js +585 -0
- package/dist/cli/db-CLODWCOB.mjs +486 -0
- package/dist/cli/db-F6PKFOOV.js +486 -0
- package/dist/cli/executions-5TQAKLJU.js +28 -0
- package/dist/cli/executions-MAKW6QOY.mjs +28 -0
- package/dist/cli/feedback-GV4QJFEN.js +107 -0
- package/dist/cli/feedback-XY24TW7H.mjs +107 -0
- package/dist/cli/getting-started.js +18 -76
- package/dist/cli/getting-started.mjs +10 -37
- package/dist/cli/git-6S2C22T5.js +23 -0
- package/dist/cli/git-OZLBD5HH.mjs +23 -0
- package/dist/cli/index.js +291 -54294
- package/dist/cli/index.mjs +209 -54352
- package/dist/cli/legacy-noops-LLCGDPQE.js +157 -0
- package/dist/cli/legacy-noops-TYH3TW2D.mjs +157 -0
- package/dist/cli/monitors-CABMB4DW.js +3251 -0
- package/dist/cli/monitors-TTD55HMK.mjs +3251 -0
- package/dist/cli/org-MSF3RG76.js +849 -0
- package/dist/cli/org-SFJ77MWY.mjs +849 -0
- package/dist/cli/play-NXHJSCKL.mjs +18439 -0
- package/dist/cli/play-OKSFNNO4.js +18439 -0
- package/dist/cli/providers-QC2ERERL.js +95 -0
- package/dist/cli/providers-TIIAZEUA.mjs +95 -0
- package/dist/cli/proxy-env-EBANZHPV.js +161 -0
- package/dist/cli/proxy-env-M4ZHXHRL.mjs +161 -0
- package/dist/cli/quickstart-ICANHLMI.js +69 -0
- package/dist/cli/quickstart-KWUOP3FI.mjs +69 -0
- package/dist/cli/search-U5H2FDOO.mjs +18 -0
- package/dist/cli/search-XQRQOLWS.js +18 -0
- package/dist/cli/secrets-3J6YTWFA.mjs +400 -0
- package/dist/cli/secrets-YNMNZNOM.js +400 -0
- package/dist/cli/sessions-CLEES4L6.mjs +15 -0
- package/dist/cli/sessions-J6VR47FG.js +15 -0
- package/dist/cli/settings-EEKZMTY4.js +846 -0
- package/dist/cli/settings-UVLOQZRH.mjs +846 -0
- package/dist/cli/setup-DLMDXPGJ.mjs +1257 -0
- package/dist/cli/setup-HC4Q7QM4.js +1257 -0
- package/dist/cli/skills-33PHFBSB.mjs +28 -0
- package/dist/cli/skills-LFRY3KPX.js +28 -0
- package/dist/cli/tools-3COEPKML.mjs +2955 -0
- package/dist/cli/tools-QXEVCN45.js +2955 -0
- package/dist/cli/update-LSAC4FF5.mjs +28 -0
- package/dist/cli/update-NXHSPYXA.js +28 -0
- package/dist/cli/workflow-7S44OECF.mjs +1936 -0
- package/dist/cli/workflow-QPZWYSZE.js +1936 -0
- package/dist/helpers.js +6 -32
- package/dist/helpers.mjs +2 -0
- package/dist/index.js +1972 -12825
- package/dist/index.mjs +1910 -12672
- package/dist/install-integrity.json +110 -0
- package/dist/monitor-fleet-contract.js +31 -730
- package/dist/monitor-fleet-contract.mjs +18 -667
- package/dist/monitor-input-contract.js +10 -390
- package/dist/monitor-input-contract.mjs +7 -348
- package/dist/plays/bundle-play-file.mjs +13 -5178
- package/dist/plays/csv-data-input.js +5 -3132
- package/dist/plays/csv-data-input.mjs +5 -3105
- package/dist/plays/run-request.js +56 -664
- package/dist/plays/run-request.mjs +30 -588
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +5 -212
- package/dist/release.mjs +5 -185
- package/package.json +1 -1
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
import { SDK_API_CONTRACT, SDK_API_MAJOR, SDK_VERSION } from './version.js';
|
|
2
|
-
import {
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import {
|
|
4
|
+
existsSync,
|
|
5
|
+
mkdirSync,
|
|
6
|
+
readFileSync,
|
|
7
|
+
renameSync,
|
|
8
|
+
rmSync,
|
|
9
|
+
writeFileSync,
|
|
10
|
+
} from 'node:fs';
|
|
3
11
|
import { homedir } from 'node:os';
|
|
4
12
|
import { dirname, join } from 'node:path';
|
|
5
13
|
import { sdkCliStateDirPath } from './config.js';
|
|
@@ -96,6 +104,21 @@ type CompatCacheFile = {
|
|
|
96
104
|
>;
|
|
97
105
|
};
|
|
98
106
|
|
|
107
|
+
function writeCompatibilityCacheFile(
|
|
108
|
+
path: string,
|
|
109
|
+
cache: CompatCacheFile,
|
|
110
|
+
): void {
|
|
111
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
112
|
+
const tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
113
|
+
try {
|
|
114
|
+
writeFileSync(tempPath, `${JSON.stringify(cache, null, 2)}\n`);
|
|
115
|
+
// Concurrent CLI processes must never see a partially written policy.
|
|
116
|
+
renameSync(tempPath, path);
|
|
117
|
+
} finally {
|
|
118
|
+
rmSync(tempPath, { force: true });
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
99
122
|
function shouldSkipCompatibilityCheck(): boolean {
|
|
100
123
|
const value =
|
|
101
124
|
process.env.DEEPLINE_SKIP_SDK_COMPAT_CHECK?.trim().toLowerCase();
|
|
@@ -273,7 +296,7 @@ function invalidateCachedCompatibility(
|
|
|
273
296
|
const entries = existing.entries ?? {};
|
|
274
297
|
delete entries[compatibilityCacheKey(baseUrl, command, skillsVersion)];
|
|
275
298
|
delete entries[legacyCompatibilityCacheKey(baseUrl, command)];
|
|
276
|
-
|
|
299
|
+
writeCompatibilityCacheFile(cachePath, { entries });
|
|
277
300
|
} catch {
|
|
278
301
|
// A cache that cannot be edited simply expires on its own TTL.
|
|
279
302
|
}
|
|
@@ -287,16 +310,21 @@ function writeCachedCompatibility(
|
|
|
287
310
|
): void {
|
|
288
311
|
try {
|
|
289
312
|
const path = sdkCompatibilityCachePath(baseUrl);
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
313
|
+
let existing: CompatCacheFile = {};
|
|
314
|
+
if (existsSync(path)) {
|
|
315
|
+
try {
|
|
316
|
+
existing = JSON.parse(readFileSync(path, 'utf8')) as CompatCacheFile;
|
|
317
|
+
} catch {
|
|
318
|
+
// A failed or overlapping old write can corrupt the optional cache.
|
|
319
|
+
// Replace it with this live policy instead of failing every future write.
|
|
320
|
+
}
|
|
321
|
+
}
|
|
293
322
|
const entries = existing.entries ?? {};
|
|
294
323
|
entries[compatibilityCacheKey(baseUrl, command, skillsVersion)] = {
|
|
295
324
|
savedAt: Date.now(),
|
|
296
325
|
response,
|
|
297
326
|
};
|
|
298
|
-
|
|
299
|
-
writeFileSync(path, `${JSON.stringify({ entries }, null, 2)}\n`);
|
|
327
|
+
writeCompatibilityCacheFile(path, { entries });
|
|
300
328
|
} catch {
|
|
301
329
|
// Compatibility checks are advisory unless the server explicitly responds
|
|
302
330
|
// with a blocking policy. Cache failures must not block normal CLI usage.
|
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.295',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -374,12 +374,13 @@ export class RuntimeSchedulerPoolAdmission {
|
|
|
374
374
|
acquire(
|
|
375
375
|
priority: RuntimeSchedulerPoolPriority,
|
|
376
376
|
signal?: AbortSignal,
|
|
377
|
+
remainingWaitMs: number = this.maxWaitMs,
|
|
377
378
|
): Promise<() => void> {
|
|
378
379
|
throwIfSchedulerAborted(signal);
|
|
379
380
|
if (this.#active < this.maxActive && this.waitingCount === 0) {
|
|
380
381
|
return Promise.resolve(this.#grant());
|
|
381
382
|
}
|
|
382
|
-
if (this.waitingCount >= this.maxWaiting) {
|
|
383
|
+
if (remainingWaitMs <= 0 || this.waitingCount >= this.maxWaiting) {
|
|
383
384
|
throw new RuntimeSchedulerSaturatedError(this.waitingCount);
|
|
384
385
|
}
|
|
385
386
|
return new Promise((resolve, reject) => {
|
|
@@ -388,13 +389,16 @@ export class RuntimeSchedulerPoolAdmission {
|
|
|
388
389
|
resolve,
|
|
389
390
|
reject,
|
|
390
391
|
signal,
|
|
391
|
-
timeout: setTimeout(
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
392
|
+
timeout: setTimeout(
|
|
393
|
+
() => {
|
|
394
|
+
const index = this.#waiting[priority].indexOf(waiter);
|
|
395
|
+
if (index < 0) return;
|
|
396
|
+
this.#waiting[priority].splice(index, 1);
|
|
397
|
+
signal?.removeEventListener('abort', waiter.onAbort!);
|
|
398
|
+
reject(new RuntimeSchedulerSaturatedError(this.waitingCount));
|
|
399
|
+
},
|
|
400
|
+
Math.max(0, Math.min(this.maxWaitMs, remainingWaitMs)),
|
|
401
|
+
),
|
|
398
402
|
};
|
|
399
403
|
waiter.onAbort = () => {
|
|
400
404
|
const index = this.#waiting[priority].indexOf(waiter);
|
|
@@ -942,291 +946,289 @@ export async function withRuntimeSchedulerClient<T>(
|
|
|
942
946
|
),
|
|
943
947
|
);
|
|
944
948
|
const operationStartedAt = performance.now();
|
|
945
|
-
|
|
946
|
-
let
|
|
947
|
-
try {
|
|
948
|
-
releaseAdmission = await entry.admission.acquire(
|
|
949
|
-
trace?.priority ?? 'ordinary',
|
|
950
|
-
signal,
|
|
951
|
-
);
|
|
952
|
-
} catch (error) {
|
|
953
|
-
if (
|
|
954
|
-
error instanceof RuntimeSchedulerSaturatedError &&
|
|
955
|
-
!(signal?.aborted && signal.reason === error)
|
|
956
|
-
) {
|
|
957
|
-
recordAdmissionSaturation(
|
|
958
|
-
entry,
|
|
959
|
-
trace?.priority ?? 'ordinary',
|
|
960
|
-
performance.now() - admissionStartedAt,
|
|
961
|
-
);
|
|
962
|
-
}
|
|
963
|
-
throw error;
|
|
964
|
-
}
|
|
965
|
-
const admissionWaitMs = performance.now() - admissionStartedAt;
|
|
966
|
-
let unsampledAdmissionWaitMs = admissionWaitMs;
|
|
949
|
+
let admissionWaitMs = 0;
|
|
950
|
+
let unsampledAdmissionWaitMs = 0;
|
|
967
951
|
let lastError: unknown = null;
|
|
968
|
-
|
|
952
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
969
953
|
throwIfSchedulerAborted(signal);
|
|
970
|
-
|
|
954
|
+
if (attempt > 0) {
|
|
955
|
+
await sleep(jitteredBackoffMs(attempt - 1));
|
|
971
956
|
throwIfSchedulerAborted(signal);
|
|
972
|
-
|
|
973
|
-
|
|
957
|
+
}
|
|
958
|
+
const admissionStartedAt = performance.now();
|
|
959
|
+
let releaseAdmission: () => void;
|
|
960
|
+
try {
|
|
961
|
+
releaseAdmission = await entry.admission.acquire(
|
|
962
|
+
trace?.priority ?? 'ordinary',
|
|
963
|
+
signal,
|
|
964
|
+
entry.admission.maxWaitMs - admissionWaitMs,
|
|
965
|
+
);
|
|
966
|
+
} catch (error) {
|
|
967
|
+
if (
|
|
968
|
+
error instanceof RuntimeSchedulerSaturatedError &&
|
|
969
|
+
!(signal?.aborted && signal.reason === error)
|
|
970
|
+
) {
|
|
971
|
+
recordAdmissionSaturation(
|
|
972
|
+
entry,
|
|
973
|
+
trace?.priority ?? 'ordinary',
|
|
974
|
+
performance.now() - admissionStartedAt,
|
|
975
|
+
);
|
|
976
|
+
}
|
|
977
|
+
throw error;
|
|
978
|
+
}
|
|
979
|
+
const attemptAdmissionWaitMs = performance.now() - admissionStartedAt;
|
|
980
|
+
admissionWaitMs += attemptAdmissionWaitMs;
|
|
981
|
+
unsampledAdmissionWaitMs += attemptAdmissionWaitMs;
|
|
982
|
+
let halfOpenProbe: symbol | null = null;
|
|
983
|
+
let capacityObserved = false;
|
|
984
|
+
try {
|
|
985
|
+
halfOpenProbe = breakerGate(entry);
|
|
986
|
+
if (entry.pool.waitingCount >= entry.maxAcquireQueueDepth) {
|
|
987
|
+
entry.stats.saturationRejections += 1;
|
|
988
|
+
throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
|
|
989
|
+
}
|
|
990
|
+
const acquireStart = performance.now();
|
|
991
|
+
entry.stats.peakTotalClients = Math.max(
|
|
992
|
+
entry.stats.peakTotalClients,
|
|
993
|
+
entry.pool.totalCount,
|
|
994
|
+
);
|
|
995
|
+
entry.stats.peakWaitingClients = Math.max(
|
|
996
|
+
entry.stats.peakWaitingClients,
|
|
997
|
+
entry.pool.waitingCount,
|
|
998
|
+
);
|
|
999
|
+
let poolClient;
|
|
974
1000
|
try {
|
|
975
|
-
|
|
1001
|
+
poolClient = await entry.pool.connect();
|
|
1002
|
+
} catch (error) {
|
|
1003
|
+
if (isRuntimeSchedulerLocalAcquireTimeout(error)) {
|
|
976
1004
|
entry.stats.saturationRejections += 1;
|
|
1005
|
+
console.warn('[runtime-scheduler-pool.local_acquire_saturated]', {
|
|
1006
|
+
maxClients: entry.maxClients,
|
|
1007
|
+
totalClients: entry.pool.totalCount,
|
|
1008
|
+
idleClients: entry.pool.idleCount,
|
|
1009
|
+
waitingClients: entry.pool.waitingCount,
|
|
1010
|
+
});
|
|
977
1011
|
throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
|
|
978
1012
|
}
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
1013
|
+
if (isPgBouncerCapacityError(error)) {
|
|
1014
|
+
recordCapacityRejection(entry, error);
|
|
1015
|
+
capacityObserved = true;
|
|
1016
|
+
throw new RuntimeSchedulerPgBouncerCapacityError(
|
|
1017
|
+
error instanceof Error ? error.message : String(error),
|
|
1018
|
+
);
|
|
1019
|
+
}
|
|
1020
|
+
recordBreakerFailure(entry, error, halfOpenProbe);
|
|
1021
|
+
entry.stats.connectionErrors += 1;
|
|
1022
|
+
lastError = error;
|
|
1023
|
+
if (attempt + 1 < maxAttempts) {
|
|
1024
|
+
entry.stats.retries += 1;
|
|
1025
|
+
continue;
|
|
1026
|
+
}
|
|
1027
|
+
throw error;
|
|
1028
|
+
}
|
|
1029
|
+
entry.stats.acquires += 1;
|
|
1030
|
+
const acquireWaitMs = performance.now() - acquireStart;
|
|
1031
|
+
entry.stats.peakTotalClients = Math.max(
|
|
1032
|
+
entry.stats.peakTotalClients,
|
|
1033
|
+
entry.pool.totalCount,
|
|
1034
|
+
);
|
|
1035
|
+
// The admission gate owns the local queue before pg.Pool sees this
|
|
1036
|
+
// checkout. Carry waits from failed checkouts forward and include each
|
|
1037
|
+
// admission wait once when a client is acquired.
|
|
1038
|
+
pushSample(
|
|
1039
|
+
entry.stats.acquireWait,
|
|
1040
|
+
unsampledAdmissionWaitMs + acquireWaitMs,
|
|
1041
|
+
);
|
|
1042
|
+
unsampledAdmissionWaitMs = 0;
|
|
1043
|
+
|
|
1044
|
+
// A checked-out pg client emits 'error' (not via the pool) when its TCP
|
|
1045
|
+
// connection dies while idle — e.g. a worker holding a client across a
|
|
1046
|
+
// long play execution. With no listener that is an uncaught exception
|
|
1047
|
+
// that kills the process (observed: worker crash mid-150k-row run).
|
|
1048
|
+
// Swallow it here; the next query on the dead client fails normally and
|
|
1049
|
+
// flows through the connection-error retry/poison path below.
|
|
1050
|
+
let checkedOutError: unknown = null;
|
|
1051
|
+
const onCheckedOutError = (error: Error) => {
|
|
1052
|
+
checkedOutError = error;
|
|
1053
|
+
entry.stats.connectionErrors += 1;
|
|
1054
|
+
console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
|
|
1055
|
+
error: error.message,
|
|
1056
|
+
});
|
|
1057
|
+
};
|
|
1058
|
+
poolClient.on('error', onCheckedOutError);
|
|
1059
|
+
let clientReleased = false;
|
|
1060
|
+
const releasePoolClient = (error?: Error) => {
|
|
1061
|
+
if (clientReleased) return;
|
|
1062
|
+
clientReleased = true;
|
|
1063
|
+
poolClient.release(error);
|
|
1064
|
+
};
|
|
1065
|
+
const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
|
|
1066
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1067
|
+
|
|
1068
|
+
let queryCount = 0;
|
|
1069
|
+
let queryMs = 0;
|
|
1070
|
+
try {
|
|
1071
|
+
if (!entry.databasePoolAttached) {
|
|
1072
|
+
attachDatabasePool(entry.pool);
|
|
1073
|
+
entry.databasePoolAttached = true;
|
|
1074
|
+
}
|
|
1075
|
+
throwIfSchedulerAborted(signal);
|
|
1076
|
+
const result = await fn({
|
|
1077
|
+
query: async (sql, params) => {
|
|
1078
|
+
throwIfSchedulerAborted(signal);
|
|
1079
|
+
const queryStartedAt = performance.now();
|
|
1080
|
+
queryCount += 1;
|
|
1081
|
+
try {
|
|
1082
|
+
try {
|
|
1083
|
+
const queryResult = await poolClient.query(sql, params);
|
|
1084
|
+
throwIfSchedulerAborted(signal);
|
|
1085
|
+
return { rows: queryResult.rows };
|
|
1086
|
+
} catch (error) {
|
|
1087
|
+
// node-postgres' query timeout and an AbortSignal reject the
|
|
1088
|
+
// promise, but the server may still be processing the SQL.
|
|
1089
|
+
// Never return that socket to the pool: doing so lets the
|
|
1090
|
+
// next retry overlap the first statement on the same backend.
|
|
1091
|
+
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
1092
|
+
releasePoolClient(
|
|
1093
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1094
|
+
);
|
|
1095
|
+
}
|
|
1096
|
+
throw error;
|
|
1097
|
+
}
|
|
1098
|
+
} finally {
|
|
1099
|
+
queryMs += performance.now() - queryStartedAt;
|
|
1100
|
+
}
|
|
1101
|
+
},
|
|
1102
|
+
});
|
|
1103
|
+
throwIfSchedulerAborted(signal);
|
|
1104
|
+
recordBreakerSuccess(entry);
|
|
1105
|
+
signal?.removeEventListener('abort', onAbort);
|
|
1106
|
+
poolClient.removeListener('error', onCheckedOutError);
|
|
1107
|
+
// The connection died mid-fn but fn happened to finish without another
|
|
1108
|
+
// query — destroy the dead connection rather than pooling it.
|
|
1109
|
+
releasePoolClient(
|
|
1110
|
+
checkedOutError instanceof Error ? checkedOutError : undefined,
|
|
987
1111
|
);
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1112
|
+
if (trace) {
|
|
1113
|
+
console.info(
|
|
1114
|
+
'[runtime-scheduler-pool.operation] ' +
|
|
1115
|
+
JSON.stringify({
|
|
1116
|
+
runId: trace.runId,
|
|
1117
|
+
phase: trace.phase,
|
|
1118
|
+
outcome: 'success',
|
|
1119
|
+
attempt: attempt + 1,
|
|
1120
|
+
priority: trace.priority ?? 'ordinary',
|
|
1121
|
+
admissionWaitMs,
|
|
1122
|
+
acquireWaitMs,
|
|
1123
|
+
queryCount,
|
|
1124
|
+
queryMs,
|
|
1125
|
+
totalMs: performance.now() - operationStartedAt,
|
|
1126
|
+
}),
|
|
1127
|
+
);
|
|
1128
|
+
}
|
|
1129
|
+
return result;
|
|
1130
|
+
} catch (error) {
|
|
1131
|
+
if (trace) {
|
|
1132
|
+
console.info(
|
|
1133
|
+
'[runtime-scheduler-pool.operation] ' +
|
|
1134
|
+
JSON.stringify({
|
|
1135
|
+
runId: trace.runId,
|
|
1136
|
+
phase: trace.phase,
|
|
1137
|
+
outcome: 'failed',
|
|
1138
|
+
attempt: attempt + 1,
|
|
1139
|
+
priority: trace.priority ?? 'ordinary',
|
|
1140
|
+
admissionWaitMs,
|
|
1141
|
+
acquireWaitMs,
|
|
1142
|
+
queryCount,
|
|
1143
|
+
queryMs,
|
|
1144
|
+
totalMs: performance.now() - operationStartedAt,
|
|
1145
|
+
errorName: error instanceof Error ? error.name : 'UnknownError',
|
|
1146
|
+
}),
|
|
1147
|
+
);
|
|
1148
|
+
}
|
|
1149
|
+
signal?.removeEventListener('abort', onAbort);
|
|
1150
|
+
poolClient.removeListener('error', onCheckedOutError);
|
|
1151
|
+
if (signal?.aborted) {
|
|
1152
|
+
const cancellation = schedulerAbortReason(signal);
|
|
1153
|
+
releasePoolClient(cancellation);
|
|
1154
|
+
throw cancellation;
|
|
1155
|
+
}
|
|
1156
|
+
if (isPgBouncerCapacityError(error)) {
|
|
1157
|
+
recordCapacityRejection(entry, error);
|
|
1158
|
+
capacityObserved = true;
|
|
1159
|
+
releasePoolClient();
|
|
1160
|
+
throw new RuntimeSchedulerPgBouncerCapacityError(
|
|
1161
|
+
error instanceof Error ? error.message : String(error),
|
|
1162
|
+
);
|
|
1163
|
+
}
|
|
1164
|
+
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
1165
|
+
entry.stats.queryErrors += 1;
|
|
1166
|
+
recordBreakerSuccess(entry);
|
|
1167
|
+
releasePoolClient(
|
|
1168
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1169
|
+
);
|
|
1170
|
+
throw error;
|
|
1171
|
+
}
|
|
1172
|
+
if (isConnectionCategoryError(error)) {
|
|
1009
1173
|
recordBreakerFailure(entry, error, halfOpenProbe);
|
|
1010
1174
|
entry.stats.connectionErrors += 1;
|
|
1011
1175
|
lastError = error;
|
|
1176
|
+
// Destroy the broken connection instead of returning it to the pool.
|
|
1177
|
+
releasePoolClient(
|
|
1178
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1179
|
+
);
|
|
1012
1180
|
if (attempt + 1 < maxAttempts) {
|
|
1013
1181
|
entry.stats.retries += 1;
|
|
1014
|
-
await sleep(jitteredBackoffMs(attempt));
|
|
1015
|
-
throwIfSchedulerAborted(signal);
|
|
1016
1182
|
continue;
|
|
1017
1183
|
}
|
|
1018
1184
|
throw error;
|
|
1019
1185
|
}
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
entry.
|
|
1025
|
-
);
|
|
1026
|
-
// The admission gate owns the local queue before pg.Pool sees this
|
|
1027
|
-
// checkout. Include that wait once, on the operation's first acquired
|
|
1028
|
-
// client; retries retain the permit and only add their own connect wait.
|
|
1029
|
-
pushSample(
|
|
1030
|
-
entry.stats.acquireWait,
|
|
1031
|
-
unsampledAdmissionWaitMs + acquireWaitMs,
|
|
1032
|
-
);
|
|
1033
|
-
unsampledAdmissionWaitMs = 0;
|
|
1034
|
-
|
|
1035
|
-
// A checked-out pg client emits 'error' (not via the pool) when its TCP
|
|
1036
|
-
// connection dies while idle — e.g. a worker holding a client across a
|
|
1037
|
-
// long play execution. With no listener that is an uncaught exception
|
|
1038
|
-
// that kills the process (observed: worker crash mid-150k-row run).
|
|
1039
|
-
// Swallow it here; the next query on the dead client fails normally and
|
|
1040
|
-
// flows through the connection-error retry/poison path below.
|
|
1041
|
-
let checkedOutError: unknown = null;
|
|
1042
|
-
const onCheckedOutError = (error: Error) => {
|
|
1043
|
-
checkedOutError = error;
|
|
1044
|
-
entry.stats.connectionErrors += 1;
|
|
1045
|
-
console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
|
|
1046
|
-
error: error.message,
|
|
1047
|
-
});
|
|
1048
|
-
};
|
|
1049
|
-
poolClient.on('error', onCheckedOutError);
|
|
1050
|
-
let clientReleased = false;
|
|
1051
|
-
const releasePoolClient = (error?: Error) => {
|
|
1052
|
-
if (clientReleased) return;
|
|
1053
|
-
clientReleased = true;
|
|
1054
|
-
poolClient.release(error);
|
|
1055
|
-
};
|
|
1056
|
-
const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
|
|
1057
|
-
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1058
|
-
|
|
1059
|
-
let queryCount = 0;
|
|
1060
|
-
let queryMs = 0;
|
|
1061
|
-
try {
|
|
1062
|
-
if (!entry.databasePoolAttached) {
|
|
1063
|
-
attachDatabasePool(entry.pool);
|
|
1064
|
-
entry.databasePoolAttached = true;
|
|
1065
|
-
}
|
|
1066
|
-
throwIfSchedulerAborted(signal);
|
|
1067
|
-
const result = await fn({
|
|
1068
|
-
query: async (sql, params) => {
|
|
1069
|
-
throwIfSchedulerAborted(signal);
|
|
1070
|
-
const queryStartedAt = performance.now();
|
|
1071
|
-
queryCount += 1;
|
|
1072
|
-
try {
|
|
1073
|
-
try {
|
|
1074
|
-
const queryResult = await poolClient.query(sql, params);
|
|
1075
|
-
throwIfSchedulerAborted(signal);
|
|
1076
|
-
return { rows: queryResult.rows };
|
|
1077
|
-
} catch (error) {
|
|
1078
|
-
// node-postgres' query timeout and an AbortSignal reject the
|
|
1079
|
-
// promise, but the server may still be processing the SQL.
|
|
1080
|
-
// Never return that socket to the pool: doing so lets the
|
|
1081
|
-
// next retry overlap the first statement on the same backend.
|
|
1082
|
-
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
1083
|
-
releasePoolClient(
|
|
1084
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
1085
|
-
);
|
|
1086
|
-
}
|
|
1087
|
-
throw error;
|
|
1088
|
-
}
|
|
1089
|
-
} finally {
|
|
1090
|
-
queryMs += performance.now() - queryStartedAt;
|
|
1091
|
-
}
|
|
1092
|
-
},
|
|
1093
|
-
});
|
|
1094
|
-
throwIfSchedulerAborted(signal);
|
|
1186
|
+
if (isRuntimeSchedulerRetryableTransactionError(error)) {
|
|
1187
|
+
// PostgreSQL has aborted the transaction, so never return this
|
|
1188
|
+
// client to the pool. Re-run the idempotent scheduler operation on a
|
|
1189
|
+
// new session without opening the connection-failure breaker.
|
|
1190
|
+
entry.stats.queryErrors += 1;
|
|
1095
1191
|
recordBreakerSuccess(entry);
|
|
1096
|
-
|
|
1097
|
-
poolClient.removeListener('error', onCheckedOutError);
|
|
1098
|
-
// The connection died mid-fn but fn happened to finish without another
|
|
1099
|
-
// query — destroy the dead connection rather than pooling it.
|
|
1192
|
+
lastError = error;
|
|
1100
1193
|
releasePoolClient(
|
|
1101
|
-
|
|
1194
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1102
1195
|
);
|
|
1103
|
-
if (
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
JSON.stringify({
|
|
1107
|
-
runId: trace.runId,
|
|
1108
|
-
phase: trace.phase,
|
|
1109
|
-
outcome: 'success',
|
|
1110
|
-
attempt: attempt + 1,
|
|
1111
|
-
priority: trace.priority ?? 'ordinary',
|
|
1112
|
-
admissionWaitMs,
|
|
1113
|
-
acquireWaitMs,
|
|
1114
|
-
queryCount,
|
|
1115
|
-
queryMs,
|
|
1116
|
-
totalMs: performance.now() - operationStartedAt,
|
|
1117
|
-
}),
|
|
1118
|
-
);
|
|
1119
|
-
}
|
|
1120
|
-
return result;
|
|
1121
|
-
} catch (error) {
|
|
1122
|
-
if (trace) {
|
|
1123
|
-
console.info(
|
|
1124
|
-
'[runtime-scheduler-pool.operation] ' +
|
|
1125
|
-
JSON.stringify({
|
|
1126
|
-
runId: trace.runId,
|
|
1127
|
-
phase: trace.phase,
|
|
1128
|
-
outcome: 'failed',
|
|
1129
|
-
attempt: attempt + 1,
|
|
1130
|
-
priority: trace.priority ?? 'ordinary',
|
|
1131
|
-
admissionWaitMs,
|
|
1132
|
-
acquireWaitMs,
|
|
1133
|
-
queryCount,
|
|
1134
|
-
queryMs,
|
|
1135
|
-
totalMs: performance.now() - operationStartedAt,
|
|
1136
|
-
errorName:
|
|
1137
|
-
error instanceof Error ? error.name : 'UnknownError',
|
|
1138
|
-
}),
|
|
1139
|
-
);
|
|
1140
|
-
}
|
|
1141
|
-
signal?.removeEventListener('abort', onAbort);
|
|
1142
|
-
poolClient.removeListener('error', onCheckedOutError);
|
|
1143
|
-
if (signal?.aborted) {
|
|
1144
|
-
const cancellation = schedulerAbortReason(signal);
|
|
1145
|
-
releasePoolClient(cancellation);
|
|
1146
|
-
throw cancellation;
|
|
1147
|
-
}
|
|
1148
|
-
if (isPgBouncerCapacityError(error)) {
|
|
1149
|
-
recordCapacityRejection(entry, error);
|
|
1150
|
-
capacityObserved = true;
|
|
1151
|
-
releasePoolClient();
|
|
1152
|
-
throw new RuntimeSchedulerPgBouncerCapacityError(
|
|
1153
|
-
error instanceof Error ? error.message : String(error),
|
|
1154
|
-
);
|
|
1155
|
-
}
|
|
1156
|
-
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
1157
|
-
entry.stats.queryErrors += 1;
|
|
1158
|
-
recordBreakerSuccess(entry);
|
|
1159
|
-
releasePoolClient(
|
|
1160
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
1161
|
-
);
|
|
1162
|
-
throw error;
|
|
1163
|
-
}
|
|
1164
|
-
if (isConnectionCategoryError(error)) {
|
|
1165
|
-
recordBreakerFailure(entry, error, halfOpenProbe);
|
|
1166
|
-
entry.stats.connectionErrors += 1;
|
|
1167
|
-
lastError = error;
|
|
1168
|
-
// Destroy the broken connection instead of returning it to the pool.
|
|
1169
|
-
releasePoolClient(
|
|
1170
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
1171
|
-
);
|
|
1172
|
-
if (attempt + 1 < maxAttempts) {
|
|
1173
|
-
entry.stats.retries += 1;
|
|
1174
|
-
await sleep(jitteredBackoffMs(attempt));
|
|
1175
|
-
throwIfSchedulerAborted(signal);
|
|
1176
|
-
continue;
|
|
1177
|
-
}
|
|
1178
|
-
throw error;
|
|
1179
|
-
}
|
|
1180
|
-
if (isRuntimeSchedulerRetryableTransactionError(error)) {
|
|
1181
|
-
// PostgreSQL has aborted the transaction, so never return this
|
|
1182
|
-
// client to the pool. Re-run the idempotent scheduler operation on a
|
|
1183
|
-
// new session without opening the connection-failure breaker.
|
|
1184
|
-
entry.stats.queryErrors += 1;
|
|
1185
|
-
recordBreakerSuccess(entry);
|
|
1186
|
-
lastError = error;
|
|
1187
|
-
releasePoolClient(
|
|
1188
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
1189
|
-
);
|
|
1190
|
-
if (attempt + 1 < maxAttempts) {
|
|
1191
|
-
entry.stats.retries += 1;
|
|
1192
|
-
await sleep(jitteredBackoffMs(attempt));
|
|
1193
|
-
throwIfSchedulerAborted(signal);
|
|
1194
|
-
continue;
|
|
1195
|
-
}
|
|
1196
|
-
throw error;
|
|
1196
|
+
if (attempt + 1 < maxAttempts) {
|
|
1197
|
+
entry.stats.retries += 1;
|
|
1198
|
+
continue;
|
|
1197
1199
|
}
|
|
1198
|
-
// Query-level failure on a healthy connection: count it, return the
|
|
1199
|
-
// connection to the pool, surface the error without retry.
|
|
1200
|
-
entry.stats.queryErrors += 1;
|
|
1201
|
-
recordBreakerSuccess(entry);
|
|
1202
|
-
releasePoolClient();
|
|
1203
1200
|
throw error;
|
|
1204
1201
|
}
|
|
1205
|
-
|
|
1206
|
-
//
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1202
|
+
// Query-level failure on a healthy connection: count it, return the
|
|
1203
|
+
// connection to the pool, surface the error without retry.
|
|
1204
|
+
entry.stats.queryErrors += 1;
|
|
1205
|
+
recordBreakerSuccess(entry);
|
|
1206
|
+
releasePoolClient();
|
|
1207
|
+
throw error;
|
|
1208
|
+
}
|
|
1209
|
+
} finally {
|
|
1210
|
+
// A canceled, locally saturated, or capacity-rejected trial did not
|
|
1211
|
+
// call either breaker recorder. Never let it own the probe forever.
|
|
1212
|
+
// Only the trial that claimed this token may release it: another
|
|
1213
|
+
// operation can close and reopen the breaker while this one awaits.
|
|
1214
|
+
if (
|
|
1215
|
+
halfOpenProbe !== null &&
|
|
1216
|
+
entry.breaker.halfOpenProbe === halfOpenProbe
|
|
1217
|
+
) {
|
|
1218
|
+
entry.breaker.halfOpenProbe = null;
|
|
1219
|
+
if (capacityObserved && entry.breaker.openedAt !== null) {
|
|
1220
|
+
// Capacity is not a connection failure. Keep the existing open
|
|
1221
|
+
// state and try once more after the ordinary cooldown instead of
|
|
1222
|
+
// probing the still-full database on every request.
|
|
1223
|
+
entry.breaker.openedAt = Date.now();
|
|
1221
1224
|
}
|
|
1222
1225
|
}
|
|
1226
|
+
releaseAdmission();
|
|
1223
1227
|
}
|
|
1224
|
-
throw lastError instanceof Error
|
|
1225
|
-
? lastError
|
|
1226
|
-
: new Error('Runtime scheduler pool operation failed after retries.');
|
|
1227
|
-
} finally {
|
|
1228
|
-
releaseAdmission();
|
|
1229
1228
|
}
|
|
1229
|
+
throw lastError instanceof Error
|
|
1230
|
+
? lastError
|
|
1231
|
+
: new Error('Runtime scheduler pool operation failed after retries.');
|
|
1230
1232
|
}
|
|
1231
1233
|
|
|
1232
1234
|
/**
|
|
@@ -1293,7 +1295,7 @@ export type RuntimeSchedulerPoolStats = {
|
|
|
1293
1295
|
idleCount: number;
|
|
1294
1296
|
waitingCount: number;
|
|
1295
1297
|
acquires: number;
|
|
1296
|
-
/** Successful checkout latency, including local admission once
|
|
1298
|
+
/** Successful checkout latency, including each local admission wait once. */
|
|
1297
1299
|
acquireWaitP50Ms: number | null;
|
|
1298
1300
|
acquireWaitP95Ms: number | null;
|
|
1299
1301
|
saturationRejections: number;
|