deepline 0.3.293 → 0.3.294
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +269 -267
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +3 -4
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.294',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -374,12 +374,13 @@ export class RuntimeSchedulerPoolAdmission {
|
|
|
374
374
|
acquire(
|
|
375
375
|
priority: RuntimeSchedulerPoolPriority,
|
|
376
376
|
signal?: AbortSignal,
|
|
377
|
+
remainingWaitMs: number = this.maxWaitMs,
|
|
377
378
|
): Promise<() => void> {
|
|
378
379
|
throwIfSchedulerAborted(signal);
|
|
379
380
|
if (this.#active < this.maxActive && this.waitingCount === 0) {
|
|
380
381
|
return Promise.resolve(this.#grant());
|
|
381
382
|
}
|
|
382
|
-
if (this.waitingCount >= this.maxWaiting) {
|
|
383
|
+
if (remainingWaitMs <= 0 || this.waitingCount >= this.maxWaiting) {
|
|
383
384
|
throw new RuntimeSchedulerSaturatedError(this.waitingCount);
|
|
384
385
|
}
|
|
385
386
|
return new Promise((resolve, reject) => {
|
|
@@ -388,13 +389,16 @@ export class RuntimeSchedulerPoolAdmission {
|
|
|
388
389
|
resolve,
|
|
389
390
|
reject,
|
|
390
391
|
signal,
|
|
391
|
-
timeout: setTimeout(
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
392
|
+
timeout: setTimeout(
|
|
393
|
+
() => {
|
|
394
|
+
const index = this.#waiting[priority].indexOf(waiter);
|
|
395
|
+
if (index < 0) return;
|
|
396
|
+
this.#waiting[priority].splice(index, 1);
|
|
397
|
+
signal?.removeEventListener('abort', waiter.onAbort!);
|
|
398
|
+
reject(new RuntimeSchedulerSaturatedError(this.waitingCount));
|
|
399
|
+
},
|
|
400
|
+
Math.max(0, Math.min(this.maxWaitMs, remainingWaitMs)),
|
|
401
|
+
),
|
|
398
402
|
};
|
|
399
403
|
waiter.onAbort = () => {
|
|
400
404
|
const index = this.#waiting[priority].indexOf(waiter);
|
|
@@ -942,291 +946,289 @@ export async function withRuntimeSchedulerClient<T>(
|
|
|
942
946
|
),
|
|
943
947
|
);
|
|
944
948
|
const operationStartedAt = performance.now();
|
|
945
|
-
|
|
946
|
-
let
|
|
947
|
-
try {
|
|
948
|
-
releaseAdmission = await entry.admission.acquire(
|
|
949
|
-
trace?.priority ?? 'ordinary',
|
|
950
|
-
signal,
|
|
951
|
-
);
|
|
952
|
-
} catch (error) {
|
|
953
|
-
if (
|
|
954
|
-
error instanceof RuntimeSchedulerSaturatedError &&
|
|
955
|
-
!(signal?.aborted && signal.reason === error)
|
|
956
|
-
) {
|
|
957
|
-
recordAdmissionSaturation(
|
|
958
|
-
entry,
|
|
959
|
-
trace?.priority ?? 'ordinary',
|
|
960
|
-
performance.now() - admissionStartedAt,
|
|
961
|
-
);
|
|
962
|
-
}
|
|
963
|
-
throw error;
|
|
964
|
-
}
|
|
965
|
-
const admissionWaitMs = performance.now() - admissionStartedAt;
|
|
966
|
-
let unsampledAdmissionWaitMs = admissionWaitMs;
|
|
949
|
+
let admissionWaitMs = 0;
|
|
950
|
+
let unsampledAdmissionWaitMs = 0;
|
|
967
951
|
let lastError: unknown = null;
|
|
968
|
-
|
|
952
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
969
953
|
throwIfSchedulerAborted(signal);
|
|
970
|
-
|
|
954
|
+
if (attempt > 0) {
|
|
955
|
+
await sleep(jitteredBackoffMs(attempt - 1));
|
|
971
956
|
throwIfSchedulerAborted(signal);
|
|
972
|
-
|
|
973
|
-
|
|
957
|
+
}
|
|
958
|
+
const admissionStartedAt = performance.now();
|
|
959
|
+
let releaseAdmission: () => void;
|
|
960
|
+
try {
|
|
961
|
+
releaseAdmission = await entry.admission.acquire(
|
|
962
|
+
trace?.priority ?? 'ordinary',
|
|
963
|
+
signal,
|
|
964
|
+
entry.admission.maxWaitMs - admissionWaitMs,
|
|
965
|
+
);
|
|
966
|
+
} catch (error) {
|
|
967
|
+
if (
|
|
968
|
+
error instanceof RuntimeSchedulerSaturatedError &&
|
|
969
|
+
!(signal?.aborted && signal.reason === error)
|
|
970
|
+
) {
|
|
971
|
+
recordAdmissionSaturation(
|
|
972
|
+
entry,
|
|
973
|
+
trace?.priority ?? 'ordinary',
|
|
974
|
+
performance.now() - admissionStartedAt,
|
|
975
|
+
);
|
|
976
|
+
}
|
|
977
|
+
throw error;
|
|
978
|
+
}
|
|
979
|
+
const attemptAdmissionWaitMs = performance.now() - admissionStartedAt;
|
|
980
|
+
admissionWaitMs += attemptAdmissionWaitMs;
|
|
981
|
+
unsampledAdmissionWaitMs += attemptAdmissionWaitMs;
|
|
982
|
+
let halfOpenProbe: symbol | null = null;
|
|
983
|
+
let capacityObserved = false;
|
|
984
|
+
try {
|
|
985
|
+
halfOpenProbe = breakerGate(entry);
|
|
986
|
+
if (entry.pool.waitingCount >= entry.maxAcquireQueueDepth) {
|
|
987
|
+
entry.stats.saturationRejections += 1;
|
|
988
|
+
throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
|
|
989
|
+
}
|
|
990
|
+
const acquireStart = performance.now();
|
|
991
|
+
entry.stats.peakTotalClients = Math.max(
|
|
992
|
+
entry.stats.peakTotalClients,
|
|
993
|
+
entry.pool.totalCount,
|
|
994
|
+
);
|
|
995
|
+
entry.stats.peakWaitingClients = Math.max(
|
|
996
|
+
entry.stats.peakWaitingClients,
|
|
997
|
+
entry.pool.waitingCount,
|
|
998
|
+
);
|
|
999
|
+
let poolClient;
|
|
974
1000
|
try {
|
|
975
|
-
|
|
1001
|
+
poolClient = await entry.pool.connect();
|
|
1002
|
+
} catch (error) {
|
|
1003
|
+
if (isRuntimeSchedulerLocalAcquireTimeout(error)) {
|
|
976
1004
|
entry.stats.saturationRejections += 1;
|
|
1005
|
+
console.warn('[runtime-scheduler-pool.local_acquire_saturated]', {
|
|
1006
|
+
maxClients: entry.maxClients,
|
|
1007
|
+
totalClients: entry.pool.totalCount,
|
|
1008
|
+
idleClients: entry.pool.idleCount,
|
|
1009
|
+
waitingClients: entry.pool.waitingCount,
|
|
1010
|
+
});
|
|
977
1011
|
throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
|
|
978
1012
|
}
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
1013
|
+
if (isPgBouncerCapacityError(error)) {
|
|
1014
|
+
recordCapacityRejection(entry, error);
|
|
1015
|
+
capacityObserved = true;
|
|
1016
|
+
throw new RuntimeSchedulerPgBouncerCapacityError(
|
|
1017
|
+
error instanceof Error ? error.message : String(error),
|
|
1018
|
+
);
|
|
1019
|
+
}
|
|
1020
|
+
recordBreakerFailure(entry, error, halfOpenProbe);
|
|
1021
|
+
entry.stats.connectionErrors += 1;
|
|
1022
|
+
lastError = error;
|
|
1023
|
+
if (attempt + 1 < maxAttempts) {
|
|
1024
|
+
entry.stats.retries += 1;
|
|
1025
|
+
continue;
|
|
1026
|
+
}
|
|
1027
|
+
throw error;
|
|
1028
|
+
}
|
|
1029
|
+
entry.stats.acquires += 1;
|
|
1030
|
+
const acquireWaitMs = performance.now() - acquireStart;
|
|
1031
|
+
entry.stats.peakTotalClients = Math.max(
|
|
1032
|
+
entry.stats.peakTotalClients,
|
|
1033
|
+
entry.pool.totalCount,
|
|
1034
|
+
);
|
|
1035
|
+
// The admission gate owns the local queue before pg.Pool sees this
|
|
1036
|
+
// checkout. Carry waits from failed checkouts forward and include each
|
|
1037
|
+
// admission wait once when a client is acquired.
|
|
1038
|
+
pushSample(
|
|
1039
|
+
entry.stats.acquireWait,
|
|
1040
|
+
unsampledAdmissionWaitMs + acquireWaitMs,
|
|
1041
|
+
);
|
|
1042
|
+
unsampledAdmissionWaitMs = 0;
|
|
1043
|
+
|
|
1044
|
+
// A checked-out pg client emits 'error' (not via the pool) when its TCP
|
|
1045
|
+
// connection dies while idle — e.g. a worker holding a client across a
|
|
1046
|
+
// long play execution. With no listener that is an uncaught exception
|
|
1047
|
+
// that kills the process (observed: worker crash mid-150k-row run).
|
|
1048
|
+
// Swallow it here; the next query on the dead client fails normally and
|
|
1049
|
+
// flows through the connection-error retry/poison path below.
|
|
1050
|
+
let checkedOutError: unknown = null;
|
|
1051
|
+
const onCheckedOutError = (error: Error) => {
|
|
1052
|
+
checkedOutError = error;
|
|
1053
|
+
entry.stats.connectionErrors += 1;
|
|
1054
|
+
console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
|
|
1055
|
+
error: error.message,
|
|
1056
|
+
});
|
|
1057
|
+
};
|
|
1058
|
+
poolClient.on('error', onCheckedOutError);
|
|
1059
|
+
let clientReleased = false;
|
|
1060
|
+
const releasePoolClient = (error?: Error) => {
|
|
1061
|
+
if (clientReleased) return;
|
|
1062
|
+
clientReleased = true;
|
|
1063
|
+
poolClient.release(error);
|
|
1064
|
+
};
|
|
1065
|
+
const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
|
|
1066
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1067
|
+
|
|
1068
|
+
let queryCount = 0;
|
|
1069
|
+
let queryMs = 0;
|
|
1070
|
+
try {
|
|
1071
|
+
if (!entry.databasePoolAttached) {
|
|
1072
|
+
attachDatabasePool(entry.pool);
|
|
1073
|
+
entry.databasePoolAttached = true;
|
|
1074
|
+
}
|
|
1075
|
+
throwIfSchedulerAborted(signal);
|
|
1076
|
+
const result = await fn({
|
|
1077
|
+
query: async (sql, params) => {
|
|
1078
|
+
throwIfSchedulerAborted(signal);
|
|
1079
|
+
const queryStartedAt = performance.now();
|
|
1080
|
+
queryCount += 1;
|
|
1081
|
+
try {
|
|
1082
|
+
try {
|
|
1083
|
+
const queryResult = await poolClient.query(sql, params);
|
|
1084
|
+
throwIfSchedulerAborted(signal);
|
|
1085
|
+
return { rows: queryResult.rows };
|
|
1086
|
+
} catch (error) {
|
|
1087
|
+
// node-postgres' query timeout and an AbortSignal reject the
|
|
1088
|
+
// promise, but the server may still be processing the SQL.
|
|
1089
|
+
// Never return that socket to the pool: doing so lets the
|
|
1090
|
+
// next retry overlap the first statement on the same backend.
|
|
1091
|
+
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
1092
|
+
releasePoolClient(
|
|
1093
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1094
|
+
);
|
|
1095
|
+
}
|
|
1096
|
+
throw error;
|
|
1097
|
+
}
|
|
1098
|
+
} finally {
|
|
1099
|
+
queryMs += performance.now() - queryStartedAt;
|
|
1100
|
+
}
|
|
1101
|
+
},
|
|
1102
|
+
});
|
|
1103
|
+
throwIfSchedulerAborted(signal);
|
|
1104
|
+
recordBreakerSuccess(entry);
|
|
1105
|
+
signal?.removeEventListener('abort', onAbort);
|
|
1106
|
+
poolClient.removeListener('error', onCheckedOutError);
|
|
1107
|
+
// The connection died mid-fn but fn happened to finish without another
|
|
1108
|
+
// query — destroy the dead connection rather than pooling it.
|
|
1109
|
+
releasePoolClient(
|
|
1110
|
+
checkedOutError instanceof Error ? checkedOutError : undefined,
|
|
987
1111
|
);
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1112
|
+
if (trace) {
|
|
1113
|
+
console.info(
|
|
1114
|
+
'[runtime-scheduler-pool.operation] ' +
|
|
1115
|
+
JSON.stringify({
|
|
1116
|
+
runId: trace.runId,
|
|
1117
|
+
phase: trace.phase,
|
|
1118
|
+
outcome: 'success',
|
|
1119
|
+
attempt: attempt + 1,
|
|
1120
|
+
priority: trace.priority ?? 'ordinary',
|
|
1121
|
+
admissionWaitMs,
|
|
1122
|
+
acquireWaitMs,
|
|
1123
|
+
queryCount,
|
|
1124
|
+
queryMs,
|
|
1125
|
+
totalMs: performance.now() - operationStartedAt,
|
|
1126
|
+
}),
|
|
1127
|
+
);
|
|
1128
|
+
}
|
|
1129
|
+
return result;
|
|
1130
|
+
} catch (error) {
|
|
1131
|
+
if (trace) {
|
|
1132
|
+
console.info(
|
|
1133
|
+
'[runtime-scheduler-pool.operation] ' +
|
|
1134
|
+
JSON.stringify({
|
|
1135
|
+
runId: trace.runId,
|
|
1136
|
+
phase: trace.phase,
|
|
1137
|
+
outcome: 'failed',
|
|
1138
|
+
attempt: attempt + 1,
|
|
1139
|
+
priority: trace.priority ?? 'ordinary',
|
|
1140
|
+
admissionWaitMs,
|
|
1141
|
+
acquireWaitMs,
|
|
1142
|
+
queryCount,
|
|
1143
|
+
queryMs,
|
|
1144
|
+
totalMs: performance.now() - operationStartedAt,
|
|
1145
|
+
errorName: error instanceof Error ? error.name : 'UnknownError',
|
|
1146
|
+
}),
|
|
1147
|
+
);
|
|
1148
|
+
}
|
|
1149
|
+
signal?.removeEventListener('abort', onAbort);
|
|
1150
|
+
poolClient.removeListener('error', onCheckedOutError);
|
|
1151
|
+
if (signal?.aborted) {
|
|
1152
|
+
const cancellation = schedulerAbortReason(signal);
|
|
1153
|
+
releasePoolClient(cancellation);
|
|
1154
|
+
throw cancellation;
|
|
1155
|
+
}
|
|
1156
|
+
if (isPgBouncerCapacityError(error)) {
|
|
1157
|
+
recordCapacityRejection(entry, error);
|
|
1158
|
+
capacityObserved = true;
|
|
1159
|
+
releasePoolClient();
|
|
1160
|
+
throw new RuntimeSchedulerPgBouncerCapacityError(
|
|
1161
|
+
error instanceof Error ? error.message : String(error),
|
|
1162
|
+
);
|
|
1163
|
+
}
|
|
1164
|
+
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
1165
|
+
entry.stats.queryErrors += 1;
|
|
1166
|
+
recordBreakerSuccess(entry);
|
|
1167
|
+
releasePoolClient(
|
|
1168
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1169
|
+
);
|
|
1170
|
+
throw error;
|
|
1171
|
+
}
|
|
1172
|
+
if (isConnectionCategoryError(error)) {
|
|
1009
1173
|
recordBreakerFailure(entry, error, halfOpenProbe);
|
|
1010
1174
|
entry.stats.connectionErrors += 1;
|
|
1011
1175
|
lastError = error;
|
|
1176
|
+
// Destroy the broken connection instead of returning it to the pool.
|
|
1177
|
+
releasePoolClient(
|
|
1178
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1179
|
+
);
|
|
1012
1180
|
if (attempt + 1 < maxAttempts) {
|
|
1013
1181
|
entry.stats.retries += 1;
|
|
1014
|
-
await sleep(jitteredBackoffMs(attempt));
|
|
1015
|
-
throwIfSchedulerAborted(signal);
|
|
1016
1182
|
continue;
|
|
1017
1183
|
}
|
|
1018
1184
|
throw error;
|
|
1019
1185
|
}
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
entry.
|
|
1025
|
-
);
|
|
1026
|
-
// The admission gate owns the local queue before pg.Pool sees this
|
|
1027
|
-
// checkout. Include that wait once, on the operation's first acquired
|
|
1028
|
-
// client; retries retain the permit and only add their own connect wait.
|
|
1029
|
-
pushSample(
|
|
1030
|
-
entry.stats.acquireWait,
|
|
1031
|
-
unsampledAdmissionWaitMs + acquireWaitMs,
|
|
1032
|
-
);
|
|
1033
|
-
unsampledAdmissionWaitMs = 0;
|
|
1034
|
-
|
|
1035
|
-
// A checked-out pg client emits 'error' (not via the pool) when its TCP
|
|
1036
|
-
// connection dies while idle — e.g. a worker holding a client across a
|
|
1037
|
-
// long play execution. With no listener that is an uncaught exception
|
|
1038
|
-
// that kills the process (observed: worker crash mid-150k-row run).
|
|
1039
|
-
// Swallow it here; the next query on the dead client fails normally and
|
|
1040
|
-
// flows through the connection-error retry/poison path below.
|
|
1041
|
-
let checkedOutError: unknown = null;
|
|
1042
|
-
const onCheckedOutError = (error: Error) => {
|
|
1043
|
-
checkedOutError = error;
|
|
1044
|
-
entry.stats.connectionErrors += 1;
|
|
1045
|
-
console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
|
|
1046
|
-
error: error.message,
|
|
1047
|
-
});
|
|
1048
|
-
};
|
|
1049
|
-
poolClient.on('error', onCheckedOutError);
|
|
1050
|
-
let clientReleased = false;
|
|
1051
|
-
const releasePoolClient = (error?: Error) => {
|
|
1052
|
-
if (clientReleased) return;
|
|
1053
|
-
clientReleased = true;
|
|
1054
|
-
poolClient.release(error);
|
|
1055
|
-
};
|
|
1056
|
-
const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
|
|
1057
|
-
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1058
|
-
|
|
1059
|
-
let queryCount = 0;
|
|
1060
|
-
let queryMs = 0;
|
|
1061
|
-
try {
|
|
1062
|
-
if (!entry.databasePoolAttached) {
|
|
1063
|
-
attachDatabasePool(entry.pool);
|
|
1064
|
-
entry.databasePoolAttached = true;
|
|
1065
|
-
}
|
|
1066
|
-
throwIfSchedulerAborted(signal);
|
|
1067
|
-
const result = await fn({
|
|
1068
|
-
query: async (sql, params) => {
|
|
1069
|
-
throwIfSchedulerAborted(signal);
|
|
1070
|
-
const queryStartedAt = performance.now();
|
|
1071
|
-
queryCount += 1;
|
|
1072
|
-
try {
|
|
1073
|
-
try {
|
|
1074
|
-
const queryResult = await poolClient.query(sql, params);
|
|
1075
|
-
throwIfSchedulerAborted(signal);
|
|
1076
|
-
return { rows: queryResult.rows };
|
|
1077
|
-
} catch (error) {
|
|
1078
|
-
// node-postgres' query timeout and an AbortSignal reject the
|
|
1079
|
-
// promise, but the server may still be processing the SQL.
|
|
1080
|
-
// Never return that socket to the pool: doing so lets the
|
|
1081
|
-
// next retry overlap the first statement on the same backend.
|
|
1082
|
-
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
1083
|
-
releasePoolClient(
|
|
1084
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
1085
|
-
);
|
|
1086
|
-
}
|
|
1087
|
-
throw error;
|
|
1088
|
-
}
|
|
1089
|
-
} finally {
|
|
1090
|
-
queryMs += performance.now() - queryStartedAt;
|
|
1091
|
-
}
|
|
1092
|
-
},
|
|
1093
|
-
});
|
|
1094
|
-
throwIfSchedulerAborted(signal);
|
|
1186
|
+
if (isRuntimeSchedulerRetryableTransactionError(error)) {
|
|
1187
|
+
// PostgreSQL has aborted the transaction, so never return this
|
|
1188
|
+
// client to the pool. Re-run the idempotent scheduler operation on a
|
|
1189
|
+
// new session without opening the connection-failure breaker.
|
|
1190
|
+
entry.stats.queryErrors += 1;
|
|
1095
1191
|
recordBreakerSuccess(entry);
|
|
1096
|
-
|
|
1097
|
-
poolClient.removeListener('error', onCheckedOutError);
|
|
1098
|
-
// The connection died mid-fn but fn happened to finish without another
|
|
1099
|
-
// query — destroy the dead connection rather than pooling it.
|
|
1192
|
+
lastError = error;
|
|
1100
1193
|
releasePoolClient(
|
|
1101
|
-
|
|
1194
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1102
1195
|
);
|
|
1103
|
-
if (
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
JSON.stringify({
|
|
1107
|
-
runId: trace.runId,
|
|
1108
|
-
phase: trace.phase,
|
|
1109
|
-
outcome: 'success',
|
|
1110
|
-
attempt: attempt + 1,
|
|
1111
|
-
priority: trace.priority ?? 'ordinary',
|
|
1112
|
-
admissionWaitMs,
|
|
1113
|
-
acquireWaitMs,
|
|
1114
|
-
queryCount,
|
|
1115
|
-
queryMs,
|
|
1116
|
-
totalMs: performance.now() - operationStartedAt,
|
|
1117
|
-
}),
|
|
1118
|
-
);
|
|
1119
|
-
}
|
|
1120
|
-
return result;
|
|
1121
|
-
} catch (error) {
|
|
1122
|
-
if (trace) {
|
|
1123
|
-
console.info(
|
|
1124
|
-
'[runtime-scheduler-pool.operation] ' +
|
|
1125
|
-
JSON.stringify({
|
|
1126
|
-
runId: trace.runId,
|
|
1127
|
-
phase: trace.phase,
|
|
1128
|
-
outcome: 'failed',
|
|
1129
|
-
attempt: attempt + 1,
|
|
1130
|
-
priority: trace.priority ?? 'ordinary',
|
|
1131
|
-
admissionWaitMs,
|
|
1132
|
-
acquireWaitMs,
|
|
1133
|
-
queryCount,
|
|
1134
|
-
queryMs,
|
|
1135
|
-
totalMs: performance.now() - operationStartedAt,
|
|
1136
|
-
errorName:
|
|
1137
|
-
error instanceof Error ? error.name : 'UnknownError',
|
|
1138
|
-
}),
|
|
1139
|
-
);
|
|
1140
|
-
}
|
|
1141
|
-
signal?.removeEventListener('abort', onAbort);
|
|
1142
|
-
poolClient.removeListener('error', onCheckedOutError);
|
|
1143
|
-
if (signal?.aborted) {
|
|
1144
|
-
const cancellation = schedulerAbortReason(signal);
|
|
1145
|
-
releasePoolClient(cancellation);
|
|
1146
|
-
throw cancellation;
|
|
1147
|
-
}
|
|
1148
|
-
if (isPgBouncerCapacityError(error)) {
|
|
1149
|
-
recordCapacityRejection(entry, error);
|
|
1150
|
-
capacityObserved = true;
|
|
1151
|
-
releasePoolClient();
|
|
1152
|
-
throw new RuntimeSchedulerPgBouncerCapacityError(
|
|
1153
|
-
error instanceof Error ? error.message : String(error),
|
|
1154
|
-
);
|
|
1155
|
-
}
|
|
1156
|
-
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
1157
|
-
entry.stats.queryErrors += 1;
|
|
1158
|
-
recordBreakerSuccess(entry);
|
|
1159
|
-
releasePoolClient(
|
|
1160
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
1161
|
-
);
|
|
1162
|
-
throw error;
|
|
1163
|
-
}
|
|
1164
|
-
if (isConnectionCategoryError(error)) {
|
|
1165
|
-
recordBreakerFailure(entry, error, halfOpenProbe);
|
|
1166
|
-
entry.stats.connectionErrors += 1;
|
|
1167
|
-
lastError = error;
|
|
1168
|
-
// Destroy the broken connection instead of returning it to the pool.
|
|
1169
|
-
releasePoolClient(
|
|
1170
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
1171
|
-
);
|
|
1172
|
-
if (attempt + 1 < maxAttempts) {
|
|
1173
|
-
entry.stats.retries += 1;
|
|
1174
|
-
await sleep(jitteredBackoffMs(attempt));
|
|
1175
|
-
throwIfSchedulerAborted(signal);
|
|
1176
|
-
continue;
|
|
1177
|
-
}
|
|
1178
|
-
throw error;
|
|
1179
|
-
}
|
|
1180
|
-
if (isRuntimeSchedulerRetryableTransactionError(error)) {
|
|
1181
|
-
// PostgreSQL has aborted the transaction, so never return this
|
|
1182
|
-
// client to the pool. Re-run the idempotent scheduler operation on a
|
|
1183
|
-
// new session without opening the connection-failure breaker.
|
|
1184
|
-
entry.stats.queryErrors += 1;
|
|
1185
|
-
recordBreakerSuccess(entry);
|
|
1186
|
-
lastError = error;
|
|
1187
|
-
releasePoolClient(
|
|
1188
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
1189
|
-
);
|
|
1190
|
-
if (attempt + 1 < maxAttempts) {
|
|
1191
|
-
entry.stats.retries += 1;
|
|
1192
|
-
await sleep(jitteredBackoffMs(attempt));
|
|
1193
|
-
throwIfSchedulerAborted(signal);
|
|
1194
|
-
continue;
|
|
1195
|
-
}
|
|
1196
|
-
throw error;
|
|
1196
|
+
if (attempt + 1 < maxAttempts) {
|
|
1197
|
+
entry.stats.retries += 1;
|
|
1198
|
+
continue;
|
|
1197
1199
|
}
|
|
1198
|
-
// Query-level failure on a healthy connection: count it, return the
|
|
1199
|
-
// connection to the pool, surface the error without retry.
|
|
1200
|
-
entry.stats.queryErrors += 1;
|
|
1201
|
-
recordBreakerSuccess(entry);
|
|
1202
|
-
releasePoolClient();
|
|
1203
1200
|
throw error;
|
|
1204
1201
|
}
|
|
1205
|
-
|
|
1206
|
-
//
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1202
|
+
// Query-level failure on a healthy connection: count it, return the
|
|
1203
|
+
// connection to the pool, surface the error without retry.
|
|
1204
|
+
entry.stats.queryErrors += 1;
|
|
1205
|
+
recordBreakerSuccess(entry);
|
|
1206
|
+
releasePoolClient();
|
|
1207
|
+
throw error;
|
|
1208
|
+
}
|
|
1209
|
+
} finally {
|
|
1210
|
+
// A canceled, locally saturated, or capacity-rejected trial did not
|
|
1211
|
+
// call either breaker recorder. Never let it own the probe forever.
|
|
1212
|
+
// Only the trial that claimed this token may release it: another
|
|
1213
|
+
// operation can close and reopen the breaker while this one awaits.
|
|
1214
|
+
if (
|
|
1215
|
+
halfOpenProbe !== null &&
|
|
1216
|
+
entry.breaker.halfOpenProbe === halfOpenProbe
|
|
1217
|
+
) {
|
|
1218
|
+
entry.breaker.halfOpenProbe = null;
|
|
1219
|
+
if (capacityObserved && entry.breaker.openedAt !== null) {
|
|
1220
|
+
// Capacity is not a connection failure. Keep the existing open
|
|
1221
|
+
// state and try once more after the ordinary cooldown instead of
|
|
1222
|
+
// probing the still-full database on every request.
|
|
1223
|
+
entry.breaker.openedAt = Date.now();
|
|
1221
1224
|
}
|
|
1222
1225
|
}
|
|
1226
|
+
releaseAdmission();
|
|
1223
1227
|
}
|
|
1224
|
-
throw lastError instanceof Error
|
|
1225
|
-
? lastError
|
|
1226
|
-
: new Error('Runtime scheduler pool operation failed after retries.');
|
|
1227
|
-
} finally {
|
|
1228
|
-
releaseAdmission();
|
|
1229
1228
|
}
|
|
1229
|
+
throw lastError instanceof Error
|
|
1230
|
+
? lastError
|
|
1231
|
+
: new Error('Runtime scheduler pool operation failed after retries.');
|
|
1230
1232
|
}
|
|
1231
1233
|
|
|
1232
1234
|
/**
|
|
@@ -1293,7 +1295,7 @@ export type RuntimeSchedulerPoolStats = {
|
|
|
1293
1295
|
idleCount: number;
|
|
1294
1296
|
waitingCount: number;
|
|
1295
1297
|
acquires: number;
|
|
1296
|
-
/** Successful checkout latency, including local admission once
|
|
1298
|
+
/** Successful checkout latency, including each local admission wait once. */
|
|
1297
1299
|
acquireWaitP50Ms: number | null;
|
|
1298
1300
|
acquireWaitP95Ms: number | null;
|
|
1299
1301
|
saturationRejections: number;
|
|
@@ -17963,7 +17963,7 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
17963
17963
|
: runLedgerUrgentLifecyclePendingSql(alias, options);
|
|
17964
17964
|
const ingressPrefixSql =
|
|
17965
17965
|
supportsTerminalBatchMarker && input.allowIngressPrefix === true
|
|
17966
|
-
? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
17966
|
+
? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})`
|
|
17967
17967
|
: 'false';
|
|
17968
17968
|
return await client.query<{
|
|
17969
17969
|
row_kind: 'claimed' | 'delivered';
|
|
@@ -18089,8 +18089,7 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
18089
18089
|
AND ${ingressPrefixSql})
|
|
18090
18090
|
)
|
|
18091
18091
|
-- Only a receiver with the exact-prefix verifier may combine ingress
|
|
18092
|
-
-- facts
|
|
18093
|
-
-- schemas retain each producer's existing singleton receipt.
|
|
18092
|
+
-- facts; old launch origins and schemas retain singleton receipts.
|
|
18094
18093
|
AND (outbox.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
18095
18094
|
OR outbox.event_id = head.event_id
|
|
18096
18095
|
OR ${ingressPrefixSql})
|
|
@@ -18172,7 +18171,7 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
18172
18171
|
${
|
|
18173
18172
|
supportsTerminalBatchMarker
|
|
18174
18173
|
? `, delivery_batch_started_at = CASE
|
|
18175
|
-
WHEN terminal_batch.has_terminal ${input.allowIngressPrefix === true ? `OR ${urgentLifecyclePendingSql('outbox')}` : ''}
|
|
18174
|
+
WHEN terminal_batch.has_terminal ${input.allowIngressPrefix === true ? `OR ${urgentLifecyclePendingSql('outbox')} OR EXISTS (SELECT 1 FROM head WHERE head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})` : ''}
|
|
18176
18175
|
THEN COALESCE(outbox.delivery_batch_started_at, statement_timestamp())
|
|
18177
18176
|
ELSE outbox.delivery_batch_started_at
|
|
18178
18177
|
END`
|
package/dist/cli/index.js
CHANGED
|
@@ -3515,7 +3515,7 @@ var SDK_RELEASE = {
|
|
|
3515
3515
|
// getters keep their established compatibility behavior.
|
|
3516
3516
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3517
3517
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3518
|
-
version: "0.3.
|
|
3518
|
+
version: "0.3.294",
|
|
3519
3519
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3520
3520
|
packageCapabilities: {
|
|
3521
3521
|
updatePreferences: 1
|
package/dist/cli/index.mjs
CHANGED
|
@@ -3510,7 +3510,7 @@ var SDK_RELEASE = {
|
|
|
3510
3510
|
// getters keep their established compatibility behavior.
|
|
3511
3511
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3512
3512
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3513
|
-
version: "0.3.
|
|
3513
|
+
version: "0.3.294",
|
|
3514
3514
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3515
3515
|
packageCapabilities: {
|
|
3516
3516
|
updatePreferences: 1
|
package/dist/index.js
CHANGED
|
@@ -1328,7 +1328,7 @@ var SDK_RELEASE = {
|
|
|
1328
1328
|
// getters keep their established compatibility behavior.
|
|
1329
1329
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
1330
1330
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
1331
|
-
version: "0.3.
|
|
1331
|
+
version: "0.3.294",
|
|
1332
1332
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
1333
1333
|
packageCapabilities: {
|
|
1334
1334
|
updatePreferences: 1
|
package/dist/index.mjs
CHANGED
|
@@ -1232,7 +1232,7 @@ var SDK_RELEASE = {
|
|
|
1232
1232
|
// getters keep their established compatibility behavior.
|
|
1233
1233
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
1234
1234
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
1235
|
-
version: "0.3.
|
|
1235
|
+
version: "0.3.294",
|
|
1236
1236
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
1237
1237
|
packageCapabilities: {
|
|
1238
1238
|
updatePreferences: 1
|
package/dist/release.d.mts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.294";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.294";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.js
CHANGED
|
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
|
|
|
74
74
|
// getters keep their established compatibility behavior.
|
|
75
75
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
76
76
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
77
|
-
version: "0.3.
|
|
77
|
+
version: "0.3.294",
|
|
78
78
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
79
79
|
packageCapabilities: {
|
|
80
80
|
updatePreferences: 1
|
package/dist/release.mjs
CHANGED
|
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
|
|
|
48
48
|
// getters keep their established compatibility behavior.
|
|
49
49
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
50
50
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
51
|
-
version: "0.3.
|
|
51
|
+
version: "0.3.294",
|
|
52
52
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
53
53
|
packageCapabilities: {
|
|
54
54
|
updatePreferences: 1
|