deepline 0.3.349 → 0.3.351
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts +30 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +47 -22
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +114 -25
- package/dist/cli/{auth-MPHXSP64.mjs → auth-NQLQ5QBV.mjs} +4 -4
- package/dist/cli/auth-YTDZKDCB.js +31 -0
- package/dist/cli/{billing-K5WCTRBE.mjs → billing-AXSNXOTN.mjs} +4 -4
- package/dist/cli/{billing-2KUUGVDZ.js → billing-AZCBAJ6Q.js} +73 -73
- package/dist/cli/{chunk-JKJSTQ2I.mjs → chunk-722274AA.mjs} +1 -1
- package/dist/cli/{chunk-GHYMVZID.js → chunk-73J3BGJ3.js} +60 -60
- package/dist/cli/{chunk-DOHULRK2.mjs → chunk-7BGGBA4V.mjs} +4 -4
- package/dist/cli/{chunk-QCSLIEMC.js → chunk-7ZAG2XG5.js} +6 -6
- package/dist/cli/{chunk-WOFHIUOT.mjs → chunk-AQQFVUDI.mjs} +2 -2
- package/dist/cli/{chunk-I4VZKXMX.js → chunk-BKOYPUK2.js} +26 -26
- package/dist/cli/{chunk-MVQ3JAJA.js → chunk-BOOZ7D7V.js} +225 -225
- package/dist/cli/{chunk-O2FM2MS5.mjs → chunk-BR3XFSXN.mjs} +6 -6
- package/dist/cli/{chunk-QAM2K3CG.js → chunk-BWFUBETD.js} +48 -48
- package/dist/cli/{chunk-BPJTGASI.mjs → chunk-CFALTV4O.mjs} +2 -2
- package/dist/cli/{chunk-S347PHNB.js → chunk-CIDP2FSO.js} +9 -9
- package/dist/cli/{chunk-UE7DS2YV.mjs → chunk-DBWASDYS.mjs} +3 -3
- package/dist/cli/{chunk-ZQ6UF55Y.mjs → chunk-GT5HPOM3.mjs} +1 -1
- package/dist/cli/{chunk-BEGPN6NG.mjs → chunk-H5JUSKHB.mjs} +1 -1
- package/dist/cli/{chunk-M42H2BPA.js → chunk-ICZTTYDS.js} +8 -8
- package/dist/cli/{chunk-VDOH4A34.mjs → chunk-IKC7EPDS.mjs} +1 -1
- package/dist/cli/{chunk-CRSMVSER.js → chunk-JV6GFIQQ.js} +45 -45
- package/dist/cli/{chunk-KW25DKRC.js → chunk-L7YXXM5W.js} +1 -1
- package/dist/cli/{chunk-JBGR6U57.js → chunk-NUD2WO3E.js} +4 -4
- package/dist/cli/{chunk-SPB6CF4S.mjs → chunk-OF4N5X7Z.mjs} +2 -2
- package/dist/cli/{chunk-IHYVEM5Q.js → chunk-QD7W3OBA.js} +8 -8
- package/dist/cli/{chunk-GXN7DGNL.mjs → chunk-QOYD2THS.mjs} +2 -2
- package/dist/cli/{chunk-TVS3TAFR.js → chunk-SDLFAGQS.js} +15 -15
- package/dist/cli/{chunk-HVD3DK7S.mjs → chunk-URBPUPJS.mjs} +5 -5
- package/dist/cli/{chunk-6YZDQZE6.js → chunk-WY5N3SSC.js} +2 -2
- package/dist/cli/{chunk-6OVLJLYH.mjs → chunk-XFTGWGZ5.mjs} +1 -1
- package/dist/cli/client-3GCQER5K.js +20 -0
- package/dist/cli/{client-JWKYNLCR.mjs → client-NTEDBLQK.mjs} +3 -3
- package/dist/cli/{csv-IW75DRFX.js → csv-6PZZ4ARC.js} +12 -12
- package/dist/cli/{csv-Y2O237NO.mjs → csv-SSSPKSKU.mjs} +3 -3
- package/dist/cli/{db-6Q4VK5T2.js → db-3YL7YWTM.js} +29 -29
- package/dist/cli/{db-JW6UUJXC.mjs → db-XEVVDWOP.mjs} +4 -4
- package/dist/cli/{describe-DI65QISF.js → describe-XZNQXFXD.js} +15 -15
- package/dist/cli/{describe-JLN4RTGM.mjs → describe-YZ365BKT.mjs} +8 -8
- package/dist/cli/{executions-GB2E3BU6.js → executions-WBTIT2RU.js} +6 -6
- package/dist/cli/{executions-UK6LA2O5.mjs → executions-ZAV7YCQC.mjs} +4 -4
- package/dist/cli/{feedback-N2KAFS3R.mjs → feedback-NOYG63QE.mjs} +3 -3
- package/dist/cli/{feedback-MJE3A5ON.js → feedback-PRVJ26X7.js} +7 -7
- package/dist/cli/{git-NWRJFNII.mjs → git-AOSMM4US.mjs} +4 -4
- package/dist/cli/git-ZI4H72YR.js +23 -0
- package/dist/cli/index.js +79 -79
- package/dist/cli/index.mjs +31 -31
- package/dist/cli/{legacy-noops-EIO65N6W.js → legacy-noops-RTNAEYVF.js} +7 -7
- package/dist/cli/{legacy-noops-RR4W2TTE.mjs → legacy-noops-T6DKTGZY.mjs} +4 -4
- package/dist/cli/{monitors-U7NVJDQL.js → monitors-7ERJ4277.js} +85 -85
- package/dist/cli/{monitors-HNUCJYET.mjs → monitors-QIR3T2QR.mjs} +4 -4
- package/dist/cli/{org-UBHNTBSB.js → org-7P46IVVU.js} +60 -60
- package/dist/cli/{org-O7MFRQNG.mjs → org-WNRQQZNP.mjs} +4 -4
- package/dist/cli/{play-4EIIEFQK.mjs → play-57XTHKER.mjs} +7 -7
- package/dist/cli/play-QIX7RZ74.js +153 -0
- package/dist/cli/{providers-REJIT7CX.mjs → providers-ET6UBGWJ.mjs} +4 -4
- package/dist/cli/{providers-CJ56HZIT.js → providers-USRX4UVZ.js} +7 -7
- package/dist/cli/{quickstart-BZUQWQK5.js → quickstart-BPPQFMWI.js} +5 -5
- package/dist/cli/{quickstart-DQN2LYBC.mjs → quickstart-OGZ7BZKS.mjs} +3 -3
- package/dist/cli/{search-35UOMFL2.mjs → search-MZARI5XA.mjs} +5 -5
- package/dist/cli/search-P447HWLU.js +26 -0
- package/dist/cli/{secrets-G4RYNUUD.js → secrets-O6XG35SA.js} +13 -13
- package/dist/cli/{secrets-YJB66UXU.mjs → secrets-WMUJ6PB4.mjs} +4 -4
- package/dist/cli/{sessions-BUWSRUSJ.mjs → sessions-EYALESPC.mjs} +4 -4
- package/dist/cli/{sessions-DGS7S6XX.js → sessions-LCHVGXMV.js} +5 -5
- package/dist/cli/{settings-OTWDMCM5.js → settings-25YT2IQK.js} +74 -74
- package/dist/cli/{settings-OCMJXIFD.mjs → settings-LUGKMDDX.mjs} +7 -7
- package/dist/cli/{setup-MYD34DUG.js → setup-AKZWHNKU.js} +41 -41
- package/dist/cli/{setup-UXYFRGIX.mjs → setup-KSRA3V42.mjs} +6 -6
- package/dist/cli/skills-GYWMMOFQ.js +28 -0
- package/dist/cli/{skills-LOWITZWQ.mjs → skills-VJJIB4F5.mjs} +4 -4
- package/dist/cli/tools-HN6E2AH6.js +40 -0
- package/dist/cli/{tools-YJHZJLES.mjs → tools-NYQD3VSU.mjs} +6 -6
- package/dist/cli/{update-PB5UCW4I.mjs → update-GU4TALAI.mjs} +6 -6
- package/dist/cli/update-IIEKVC75.js +28 -0
- package/dist/cli/{workflow-IH2FQSYZ.mjs → workflow-T4VNPJA4.mjs} +5 -5
- package/dist/cli/{workflow-DJYTA2YM.js → workflow-TFOUWNEJ.js} +15 -15
- package/dist/index.js +6 -6
- package/dist/index.mjs +3 -3
- package/dist/install-integrity.json +74 -74
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/cli/auth-KGXZELZ3.js +0 -31
- package/dist/cli/client-O4DPNWVJ.js +0 -20
- package/dist/cli/git-A7J4SSRV.js +0 -23
- package/dist/cli/play-IHLNNVPR.js +0 -153
- package/dist/cli/search-4EPCIU2T.js +0 -26
- package/dist/cli/skills-7UHWU7XL.js +0 -28
- package/dist/cli/tools-CBYKQ7IY.js +0 -40
- package/dist/cli/update-5M3OKPEU.js +0 -28
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.351',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts
CHANGED
|
@@ -8,6 +8,26 @@ import { stringifyPostgresJson } from '../postgres-json';
|
|
|
8
8
|
|
|
9
9
|
const MAX_BUDGET_CHARGES_PER_REQUEST = 16;
|
|
10
10
|
|
|
11
|
+
/** The caller sent a charge this request can never satisfy. Retrying is futile. */
|
|
12
|
+
export class BudgetStateRequestError extends Error {
|
|
13
|
+
constructor(message: string) {
|
|
14
|
+
super(message);
|
|
15
|
+
this.name = 'BudgetStateRequestError';
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Only a limit or a malformed request is the budget's own answer. Anything
|
|
21
|
+
* else (a dropped connection, an aborted transaction) says nothing about the
|
|
22
|
+
* budget and must stay retryable.
|
|
23
|
+
*/
|
|
24
|
+
export function isBudgetStateRejection(error: unknown): boolean {
|
|
25
|
+
return (
|
|
26
|
+
error instanceof BudgetStateLimitError ||
|
|
27
|
+
error instanceof BudgetStateRequestError
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
|
|
11
31
|
function quoteIdent(value: string): string {
|
|
12
32
|
return `"${value.replace(/"/g, '""')}"`;
|
|
13
33
|
}
|
|
@@ -52,7 +72,7 @@ export async function ensurePostgresBudgetStateSchema(
|
|
|
52
72
|
|
|
53
73
|
function normalizedCharges(input: PlayRunnerBudgetChargeInput) {
|
|
54
74
|
if (input.charges.length > MAX_BUDGET_CHARGES_PER_REQUEST) {
|
|
55
|
-
throw new
|
|
75
|
+
throw new BudgetStateRequestError(
|
|
56
76
|
`Governor budget accepts at most ${MAX_BUDGET_CHARGES_PER_REQUEST} charges per request.`,
|
|
57
77
|
);
|
|
58
78
|
}
|
|
@@ -60,15 +80,17 @@ function normalizedCharges(input: PlayRunnerBudgetChargeInput) {
|
|
|
60
80
|
for (const charge of input.charges) {
|
|
61
81
|
const key = charge.key.trim();
|
|
62
82
|
if (!key || key.length > 256) {
|
|
63
|
-
throw new
|
|
83
|
+
throw new BudgetStateRequestError(
|
|
84
|
+
'Governor budget key must be 1-256 characters.',
|
|
85
|
+
);
|
|
64
86
|
}
|
|
65
87
|
if (!Number.isSafeInteger(charge.amount) || charge.amount <= 0) {
|
|
66
|
-
throw new
|
|
88
|
+
throw new BudgetStateRequestError(
|
|
67
89
|
`Governor budget ${key} amount must be a positive integer.`,
|
|
68
90
|
);
|
|
69
91
|
}
|
|
70
92
|
if (!Number.isSafeInteger(charge.limit) || charge.limit <= 0) {
|
|
71
|
-
throw new
|
|
93
|
+
throw new BudgetStateRequestError(
|
|
72
94
|
`Governor budget ${key} limit must be a positive integer.`,
|
|
73
95
|
);
|
|
74
96
|
}
|
|
@@ -108,7 +130,9 @@ export async function chargePostgresBudgetState(
|
|
|
108
130
|
if (charges.length === 0) return { counters: {} };
|
|
109
131
|
const reservationId = input.reservationId.trim();
|
|
110
132
|
if (!reservationId || reservationId.length > 256) {
|
|
111
|
-
throw new
|
|
133
|
+
throw new BudgetStateRequestError(
|
|
134
|
+
'Governor budget reservationId must be 1-256 characters.',
|
|
135
|
+
);
|
|
112
136
|
}
|
|
113
137
|
for (const charge of charges) {
|
|
114
138
|
if (charge.amount > charge.limit) {
|
|
@@ -155,7 +179,7 @@ export async function chargePostgresBudgetState(
|
|
|
155
179
|
throw new Error('Governor budget reservation disappeared.');
|
|
156
180
|
}
|
|
157
181
|
if (!sameCharges(reservation.charges_json, charges)) {
|
|
158
|
-
throw new
|
|
182
|
+
throw new BudgetStateRequestError(
|
|
159
183
|
'Governor budget reservationId was reused with different charges.',
|
|
160
184
|
);
|
|
161
185
|
}
|
|
@@ -608,7 +608,9 @@ export function isRuntimeSchedulerServerUnavailableError(
|
|
|
608
608
|
code === '57P03' ||
|
|
609
609
|
(code === '08P01' &&
|
|
610
610
|
typeof message === 'string' &&
|
|
611
|
-
recoveryMessage.test(message.trim())
|
|
611
|
+
(recoveryMessage.test(message.trim()) ||
|
|
612
|
+
// PgBouncer's exact report that its backend connection dropped.
|
|
613
|
+
message.trim() === 'server conn crashed?')) ||
|
|
612
614
|
// Preserve legacy connection messages without accepting every 08* code.
|
|
613
615
|
(legacyConnectionCode &&
|
|
614
616
|
typeof message === 'string' &&
|
|
@@ -18182,6 +18182,33 @@ function runLedgerOrderSql(alias: string): string {
|
|
|
18182
18182
|
return `CASE ${alias}.event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END, ${alias}.created_at, ${alias}.event_id`;
|
|
18183
18183
|
}
|
|
18184
18184
|
|
|
18185
|
+
/** Enumerate pending runs by index seeks, skipping each run's entire fact tail. */
|
|
18186
|
+
function runLedgerPendingRunsSql(
|
|
18187
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
18188
|
+
searchOnly?: boolean,
|
|
18189
|
+
): string {
|
|
18190
|
+
const lane =
|
|
18191
|
+
searchOnly === undefined
|
|
18192
|
+
? ''
|
|
18193
|
+
: `AND ${searchLaneSql('pending')} = ${String(searchOnly)}`;
|
|
18194
|
+
return `pending_runs AS (
|
|
18195
|
+
(SELECT pending.run_id
|
|
18196
|
+
FROM ${tableName('outbox', options)} AS pending
|
|
18197
|
+
WHERE pending.delivered_at IS NULL ${lane}
|
|
18198
|
+
ORDER BY pending.run_id LIMIT 1)
|
|
18199
|
+
UNION ALL
|
|
18200
|
+
SELECT next_run.run_id
|
|
18201
|
+
FROM pending_runs AS previous_run
|
|
18202
|
+
CROSS JOIN LATERAL (
|
|
18203
|
+
SELECT pending.run_id
|
|
18204
|
+
FROM ${tableName('outbox', options)} AS pending
|
|
18205
|
+
WHERE pending.delivered_at IS NULL ${lane}
|
|
18206
|
+
AND pending.run_id > previous_run.run_id
|
|
18207
|
+
ORDER BY pending.run_id LIMIT 1
|
|
18208
|
+
) AS next_run
|
|
18209
|
+
)`;
|
|
18210
|
+
}
|
|
18211
|
+
|
|
18185
18212
|
/** Prioritize a terminal's causal head without skipping its predecessors. */
|
|
18186
18213
|
function runLedgerTerminalPendingSql(
|
|
18187
18214
|
alias: string,
|
|
@@ -18712,14 +18739,8 @@ export async function readPostgresSchedulerOutboxNextDueAt(
|
|
|
18712
18739
|
): Promise<Date | null> {
|
|
18713
18740
|
const result = await client.query<{ due_at: Date | string | null }>(
|
|
18714
18741
|
`
|
|
18715
|
-
--
|
|
18716
|
-
|
|
18717
|
-
-- a large receipt backlog, even while another replica drains that run.
|
|
18718
|
-
WITH pending_runs AS MATERIALIZED (
|
|
18719
|
-
SELECT DISTINCT pending.run_id
|
|
18720
|
-
FROM ${tableName('outbox', options)} AS pending
|
|
18721
|
-
WHERE pending.delivered_at IS NULL
|
|
18722
|
-
), heads AS MATERIALIZED (
|
|
18742
|
+
-- Seek distinct run identities without reading their pending fact tails.
|
|
18743
|
+
WITH RECURSIVE ${runLedgerPendingRunsSql(options)}, heads AS MATERIALIZED (
|
|
18723
18744
|
SELECT pending_run.run_id, false AS search_only, head.*
|
|
18724
18745
|
FROM pending_runs AS pending_run
|
|
18725
18746
|
CROSS JOIN LATERAL (
|
|
@@ -20332,7 +20353,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20332
20353
|
// run-scoped outbox prefix: combining it with global recovery through a
|
|
20333
20354
|
// nullable predicate made PostgreSQL materialize every pending run head
|
|
20334
20355
|
// for each lifecycle transition. Only the two dedicated projectors may
|
|
20335
|
-
// perform the global
|
|
20356
|
+
// perform the global indexed head discovery below.
|
|
20336
20357
|
const selected =
|
|
20337
20358
|
(input.headEventId ? null : lockedRun) ??
|
|
20338
20359
|
(targetRunId
|
|
@@ -20379,19 +20400,23 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20379
20400
|
run_id: string;
|
|
20380
20401
|
dispatch_launch_json: unknown;
|
|
20381
20402
|
}>(`
|
|
20382
|
-
WITH heads AS MATERIALIZED (
|
|
20383
|
-
SELECT
|
|
20384
|
-
|
|
20385
|
-
|
|
20386
|
-
pending.
|
|
20387
|
-
|
|
20388
|
-
|
|
20389
|
-
|
|
20390
|
-
|
|
20391
|
-
|
|
20392
|
-
|
|
20393
|
-
|
|
20394
|
-
|
|
20403
|
+
WITH RECURSIVE ${runLedgerPendingRunsSql(options, input.searchOnly === true)}, heads AS MATERIALIZED (
|
|
20404
|
+
SELECT pending_run.run_id, head.*
|
|
20405
|
+
FROM pending_runs AS pending_run
|
|
20406
|
+
CROSS JOIN LATERAL (
|
|
20407
|
+
SELECT pending.event_id,
|
|
20408
|
+
pending.event_type,
|
|
20409
|
+
${needsLaunch ? 'pending.payload_json,' : ''}
|
|
20410
|
+
pending.created_at,
|
|
20411
|
+
pending.next_attempt_at,
|
|
20412
|
+
pending.failed_at
|
|
20413
|
+
FROM ${tableName('outbox', options)} AS pending
|
|
20414
|
+
WHERE pending.run_id = pending_run.run_id
|
|
20415
|
+
AND pending.delivered_at IS NULL
|
|
20416
|
+
AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
20417
|
+
ORDER BY ${runLedgerOrderSql('pending')}
|
|
20418
|
+
LIMIT 1
|
|
20419
|
+
) AS head
|
|
20395
20420
|
)
|
|
20396
20421
|
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
|
|
20397
20422
|
FROM heads AS head
|
|
@@ -80,6 +80,9 @@ const RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT = 4;
|
|
|
80
80
|
// from starving behind continuous new admissions. An empty class never costs
|
|
81
81
|
// another query: either order can select any eligible run.
|
|
82
82
|
const ADMISSION_TURNS_PER_OLDEST_READY_TURN = 3;
|
|
83
|
+
// Notifications are hints, not another durable queue. Overflow requests the
|
|
84
|
+
// existing global drain instead of retaining an unbounded set of run ids.
|
|
85
|
+
const MAX_NOTIFICATION_RUN_HINTS = RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT;
|
|
83
86
|
|
|
84
87
|
type RecoveryTimer = ReturnType<typeof setTimeout>;
|
|
85
88
|
|
|
@@ -89,9 +92,12 @@ export type RunLedgerOutboxRecoveryDeps = {
|
|
|
89
92
|
/** One bounded durable claim/delivery attempt. */
|
|
90
93
|
dispatchOnce: (
|
|
91
94
|
headPriority: RunLedgerOutboxHeadPriority,
|
|
95
|
+
runId?: string,
|
|
92
96
|
) => Promise<PostgresSchedulerOutboxProjectionResult>;
|
|
93
97
|
/** Independent index deliveries share this controller and notification source. */
|
|
94
|
-
dispatchObservabilityOnce?: (
|
|
98
|
+
dispatchObservabilityOnce?: (
|
|
99
|
+
runId?: string,
|
|
100
|
+
) => Promise<PostgresSchedulerOutboxProjectionResult>;
|
|
95
101
|
observabilityParallelism?: number;
|
|
96
102
|
/** Earliest pending retry time or active lease expiry in the durable outbox. */
|
|
97
103
|
readNextDueAt: () => Promise<Date | null>;
|
|
@@ -139,6 +145,18 @@ function boundedFailureDelayMs(failures: number): number {
|
|
|
139
145
|
);
|
|
140
146
|
}
|
|
141
147
|
|
|
148
|
+
function notificationRunId(payload: string): string | undefined {
|
|
149
|
+
try {
|
|
150
|
+
const message = JSON.parse(payload) as { runId?: unknown } | null;
|
|
151
|
+
if (typeof message?.runId !== 'string') return undefined;
|
|
152
|
+
const runId = message.runId.trim();
|
|
153
|
+
return runId && runId.length <= 256 ? runId : undefined;
|
|
154
|
+
} catch {
|
|
155
|
+
// Legacy, retry and repair notifications may carry no run identity.
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
142
160
|
/**
|
|
143
161
|
* Start a bounded, work-conserving recovery controller from injected seams.
|
|
144
162
|
* Subscription happens before the first database scan, so a terminal commit
|
|
@@ -189,24 +207,50 @@ export function startRunLedgerOutboxRecovery(
|
|
|
189
207
|
let hubLease: PgNotifyHubLease | null = null;
|
|
190
208
|
let unsubscribeNotify: (() => void) | undefined;
|
|
191
209
|
let unsubscribeState: (() => void) | undefined;
|
|
192
|
-
let requestDrain: (reason: string) => void = () => undefined;
|
|
210
|
+
let requestDrain: (reason: string, runId?: string) => void = () => undefined;
|
|
211
|
+
const lifecycleRunHints = new Set<string>();
|
|
212
|
+
const observabilityRunHints = new Set<string>();
|
|
213
|
+
const hasRequests = () =>
|
|
214
|
+
requested ||
|
|
215
|
+
observabilityRequested ||
|
|
216
|
+
lifecycleRunHints.size > 0 ||
|
|
217
|
+
observabilityRunHints.size > 0;
|
|
218
|
+
const addRunHint = (hints: Set<string>, runId: string) => {
|
|
219
|
+
hints.add(runId);
|
|
220
|
+
if (hints.size <= MAX_NOTIFICATION_RUN_HINTS) return true;
|
|
221
|
+
hints.clear();
|
|
222
|
+
return false;
|
|
223
|
+
};
|
|
224
|
+
const takeRunHint = (hints: Set<string>): string | undefined => {
|
|
225
|
+
const runId = hints.values().next().value;
|
|
226
|
+
if (runId !== undefined) hints.delete(runId);
|
|
227
|
+
return runId;
|
|
228
|
+
};
|
|
193
229
|
const inFlightStartedAt = new Map<number, number>();
|
|
194
230
|
let nextDispatchId = 0;
|
|
195
231
|
let lastDispatchSettledAt: number | null = null;
|
|
196
232
|
let admissionTurns = 0;
|
|
197
233
|
const dispatchOnce = async (
|
|
198
234
|
observability = false,
|
|
235
|
+
runId?: string,
|
|
199
236
|
): Promise<PostgresSchedulerOutboxProjectionResult> => {
|
|
200
237
|
const oldestReady =
|
|
201
238
|
admissionTurns === ADMISSION_TURNS_PER_OLDEST_READY_TURN;
|
|
202
|
-
|
|
239
|
+
// Scoped claims cannot change which run wins, so they must not consume
|
|
240
|
+
// the global drain's starvation-prevention turns.
|
|
241
|
+
if (!observability && runId === undefined)
|
|
242
|
+
admissionTurns = oldestReady ? 0 : admissionTurns + 1;
|
|
203
243
|
const id = nextDispatchId++;
|
|
204
244
|
inFlightStartedAt.set(id, now());
|
|
205
245
|
try {
|
|
206
|
-
if (observability)
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
246
|
+
if (observability)
|
|
247
|
+
return await (runId === undefined
|
|
248
|
+
? deps.dispatchObservabilityOnce!()
|
|
249
|
+
: deps.dispatchObservabilityOnce!(runId));
|
|
250
|
+
const headPriority = oldestReady ? 'oldest_ready' : 'admission_first';
|
|
251
|
+
return await (runId === undefined
|
|
252
|
+
? deps.dispatchOnce(headPriority)
|
|
253
|
+
: deps.dispatchOnce(headPriority, runId));
|
|
210
254
|
} finally {
|
|
211
255
|
inFlightStartedAt.delete(id);
|
|
212
256
|
lastDispatchSettledAt = now();
|
|
@@ -226,8 +270,8 @@ export function startRunLedgerOutboxRecovery(
|
|
|
226
270
|
const lease = deps.acquireNotifyHub();
|
|
227
271
|
if (!lease) return;
|
|
228
272
|
hubLease = lease;
|
|
229
|
-
unsubscribeNotify = lease.hub.subscribe(() =>
|
|
230
|
-
requestDrain('postgres_notify'),
|
|
273
|
+
unsubscribeNotify = lease.hub.subscribe((payload) =>
|
|
274
|
+
requestDrain('postgres_notify', notificationRunId(payload)),
|
|
231
275
|
);
|
|
232
276
|
unsubscribeState = lease.hub.subscribeState((state) => {
|
|
233
277
|
// A reconnect is a durable catch-up boundary. The controller re-reads
|
|
@@ -239,10 +283,8 @@ export function startRunLedgerOutboxRecovery(
|
|
|
239
283
|
});
|
|
240
284
|
};
|
|
241
285
|
|
|
242
|
-
|
|
286
|
+
const startDrain = (reason: string) => {
|
|
243
287
|
if (closed) return;
|
|
244
|
-
requested = true;
|
|
245
|
-
observabilityRequested = !!deps.dispatchObservabilityOnce;
|
|
246
288
|
if (cooldownUntilMs !== null && now() < cooldownUntilMs) return;
|
|
247
289
|
clearWakeTimer();
|
|
248
290
|
if (drainPromise) {
|
|
@@ -260,10 +302,31 @@ export function startRunLedgerOutboxRecovery(
|
|
|
260
302
|
})
|
|
261
303
|
.finally(() => {
|
|
262
304
|
drainPromise = null;
|
|
263
|
-
if (
|
|
305
|
+
if (hasRequests() && !closed) startDrain('coalesced_wake');
|
|
264
306
|
});
|
|
265
307
|
};
|
|
266
308
|
|
|
309
|
+
requestDrain = (reason: string, runId?: string) => {
|
|
310
|
+
if (closed) return;
|
|
311
|
+
if (runId === undefined) {
|
|
312
|
+
requested = true;
|
|
313
|
+
observabilityRequested = !!deps.dispatchObservabilityOnce;
|
|
314
|
+
lifecycleRunHints.clear();
|
|
315
|
+
observabilityRunHints.clear();
|
|
316
|
+
} else {
|
|
317
|
+
// A pending global scan already covers these hints. Once that scan is
|
|
318
|
+
// launched its flag is consumed, so a later notification is retained.
|
|
319
|
+
if (!requested && !addRunHint(lifecycleRunHints, runId)) requested = true;
|
|
320
|
+
if (
|
|
321
|
+
deps.dispatchObservabilityOnce &&
|
|
322
|
+
!observabilityRequested &&
|
|
323
|
+
!addRunHint(observabilityRunHints, runId)
|
|
324
|
+
)
|
|
325
|
+
observabilityRequested = true;
|
|
326
|
+
}
|
|
327
|
+
startDrain(reason);
|
|
328
|
+
};
|
|
329
|
+
|
|
267
330
|
const recordProjectionRetry = (retrying: number) => {
|
|
268
331
|
logger.warn('[run-ledger-outbox.recovery_retry]', {
|
|
269
332
|
stage: 'projection',
|
|
@@ -311,7 +374,7 @@ export function startRunLedgerOutboxRecovery(
|
|
|
311
374
|
scheduleFailureRetry('next_due', error);
|
|
312
375
|
return;
|
|
313
376
|
}
|
|
314
|
-
if (closed ||
|
|
377
|
+
if (closed || hasRequests()) return;
|
|
315
378
|
// When LISTEN is healthy the exact durable due time is sufficient. If the
|
|
316
379
|
// optional listener is absent, down, or lost an election, an indexed
|
|
317
380
|
// short safety wake discovers a fresh row that otherwise had no
|
|
@@ -337,6 +400,7 @@ export function startRunLedgerOutboxRecovery(
|
|
|
337
400
|
type Delivery = {
|
|
338
401
|
id: symbol;
|
|
339
402
|
observability: boolean;
|
|
403
|
+
runId?: string;
|
|
340
404
|
result:
|
|
341
405
|
| PromiseSettledResult<PostgresSchedulerOutboxProjectionResult>
|
|
342
406
|
| { status: 'skipped' };
|
|
@@ -371,7 +435,7 @@ export function startRunLedgerOutboxRecovery(
|
|
|
371
435
|
completed = attempted = projected = retrying = blocked = 0;
|
|
372
436
|
};
|
|
373
437
|
|
|
374
|
-
const launch = (observability = false) => {
|
|
438
|
+
const launch = (observability = false, runId?: string) => {
|
|
375
439
|
const id = Symbol();
|
|
376
440
|
pending.add(id);
|
|
377
441
|
if (observability) pendingObservability.add(id);
|
|
@@ -380,7 +444,7 @@ export function startRunLedgerOutboxRecovery(
|
|
|
380
444
|
dispatchFailed = true;
|
|
381
445
|
scheduleFailureRetry('dispatch', result.reason);
|
|
382
446
|
}
|
|
383
|
-
deliveries.push({ id, observability, result });
|
|
447
|
+
deliveries.push({ id, observability, runId, result });
|
|
384
448
|
wakeActiveDrain?.();
|
|
385
449
|
};
|
|
386
450
|
// Subscribe once per delivery. Repeated Promise.race calls would retain
|
|
@@ -393,7 +457,7 @@ export function startRunLedgerOutboxRecovery(
|
|
|
393
457
|
try {
|
|
394
458
|
settle({
|
|
395
459
|
status: 'fulfilled',
|
|
396
|
-
value: await dispatchOnce(observability),
|
|
460
|
+
value: await dispatchOnce(observability, runId),
|
|
397
461
|
});
|
|
398
462
|
} catch (reason) {
|
|
399
463
|
settle({ status: 'rejected', reason });
|
|
@@ -404,13 +468,13 @@ export function startRunLedgerOutboxRecovery(
|
|
|
404
468
|
const canProbeLifecycle = () =>
|
|
405
469
|
!closed &&
|
|
406
470
|
!dispatchFailed &&
|
|
407
|
-
requested &&
|
|
471
|
+
(requested || lifecycleRunHints.size > 0) &&
|
|
408
472
|
pending.size < parallelism &&
|
|
409
473
|
pending.size - pendingObservability.size < lifecycleParallelism;
|
|
410
474
|
const canProbeObservability = () =>
|
|
411
475
|
!closed &&
|
|
412
476
|
!dispatchFailed &&
|
|
413
|
-
observabilityRequested &&
|
|
477
|
+
(observabilityRequested || observabilityRunHints.size > 0) &&
|
|
414
478
|
pending.size < parallelism &&
|
|
415
479
|
pendingObservability.size < observabilityParallelism;
|
|
416
480
|
|
|
@@ -420,12 +484,16 @@ export function startRunLedgerOutboxRecovery(
|
|
|
420
484
|
// still delivering. Keep it pending when full so an empty result
|
|
421
485
|
// cannot consume a notification that arrived after its query began.
|
|
422
486
|
if (canProbeObservability()) {
|
|
487
|
+
const runId = observabilityRequested
|
|
488
|
+
? undefined
|
|
489
|
+
: takeRunHint(observabilityRunHints);
|
|
423
490
|
observabilityRequested = false;
|
|
424
|
-
launch(true);
|
|
491
|
+
launch(true, runId);
|
|
425
492
|
}
|
|
426
493
|
if (canProbeLifecycle()) {
|
|
494
|
+
const runId = requested ? undefined : takeRunHint(lifecycleRunHints);
|
|
427
495
|
requested = false;
|
|
428
|
-
launch();
|
|
496
|
+
launch(false, runId);
|
|
429
497
|
}
|
|
430
498
|
// Productive completion permits immediate use of every free slot.
|
|
431
499
|
// Empty results grant no refill, so idle work cannot spin on scans.
|
|
@@ -481,14 +549,31 @@ export function startRunLedgerOutboxRecovery(
|
|
|
481
549
|
if (completed >= parallelism) reportDeliveries();
|
|
482
550
|
// Retryable rows own durable retry clocks. The SQL claim keeps their
|
|
483
551
|
// run ordered while successful slots keep unrelated runs moving.
|
|
484
|
-
if (delivery.
|
|
552
|
+
if (delivery.runId !== undefined) {
|
|
553
|
+
// Continue only the notified run. Refilling with global claims here
|
|
554
|
+
// would turn one productive scoped claim back into whole-outbox scans.
|
|
555
|
+
if (result.attempted > 0) {
|
|
556
|
+
if (delivery.observability) {
|
|
557
|
+
if (
|
|
558
|
+
!observabilityRequested &&
|
|
559
|
+
!addRunHint(observabilityRunHints, delivery.runId)
|
|
560
|
+
)
|
|
561
|
+
observabilityRequested = true;
|
|
562
|
+
} else if (
|
|
563
|
+
!requested &&
|
|
564
|
+
!addRunHint(lifecycleRunHints, delivery.runId)
|
|
565
|
+
)
|
|
566
|
+
requested = true;
|
|
567
|
+
}
|
|
568
|
+
} else if (delivery.observability) {
|
|
485
569
|
observabilityRefill = result.attempted > 0;
|
|
486
570
|
} else refill = result.attempted > 0;
|
|
487
571
|
}
|
|
488
572
|
};
|
|
489
573
|
|
|
490
574
|
// Subscribe first, then scan. Notifications are lossy by design, so every
|
|
491
|
-
// wake re-reads durable
|
|
575
|
+
// named wake re-reads that run's durable prefix. Startup, reconnect, retries
|
|
576
|
+
// and identity-less notifications retain the global recovery scan.
|
|
492
577
|
attachNotifyHub();
|
|
493
578
|
requestDrain('startup');
|
|
494
579
|
|
|
@@ -512,6 +597,8 @@ export function startRunLedgerOutboxRecovery(
|
|
|
512
597
|
wakeActiveDrain?.();
|
|
513
598
|
clearWakeTimer();
|
|
514
599
|
cooldownUntilMs = null;
|
|
600
|
+
lifecycleRunHints.clear();
|
|
601
|
+
observabilityRunHints.clear();
|
|
515
602
|
unsubscribeNotify?.();
|
|
516
603
|
unsubscribeState?.();
|
|
517
604
|
await drainPromise;
|
|
@@ -769,6 +856,7 @@ export function startRunLedgerOutboxRecoveryController(input: {
|
|
|
769
856
|
const dispatch = (
|
|
770
857
|
headPriority: RunLedgerOutboxHeadPriority,
|
|
771
858
|
searchOnly: boolean,
|
|
859
|
+
runId?: string,
|
|
772
860
|
) =>
|
|
773
861
|
dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
|
|
774
862
|
(operation) =>
|
|
@@ -780,6 +868,7 @@ export function startRunLedgerOutboxRecoveryController(input: {
|
|
|
780
868
|
tokenAuthority: 'gateway',
|
|
781
869
|
materializeObservability: true,
|
|
782
870
|
searchOnly,
|
|
871
|
+
...(runId === undefined ? {} : { runId }),
|
|
783
872
|
withObservabilityClient: (operation) =>
|
|
784
873
|
withRuntimeSchedulerClient(observabilityPoolOptions, operation),
|
|
785
874
|
limit: 8,
|
|
@@ -801,8 +890,8 @@ export function startRunLedgerOutboxRecoveryController(input: {
|
|
|
801
890
|
schedulerOptions,
|
|
802
891
|
}),
|
|
803
892
|
),
|
|
804
|
-
dispatchOnce: (headPriority) => dispatch(headPriority, false),
|
|
805
|
-
dispatchObservabilityOnce: () => dispatch('oldest_ready', true),
|
|
893
|
+
dispatchOnce: (headPriority, runId) => dispatch(headPriority, false, runId),
|
|
894
|
+
dispatchObservabilityOnce: (runId) => dispatch('oldest_ready', true, runId),
|
|
806
895
|
observabilityParallelism: RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT,
|
|
807
896
|
// `dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope` releases each
|
|
808
897
|
// short scheduler checkout before it calls the Vercel gateway. The two
|
|
@@ -9,11 +9,11 @@ import {
|
|
|
9
9
|
registerAuthCommands,
|
|
10
10
|
reportSetupCompleted,
|
|
11
11
|
resolvePendingAuthClaimForResume
|
|
12
|
-
} from "./chunk-
|
|
13
|
-
import "./chunk-
|
|
14
|
-
import "./chunk-
|
|
12
|
+
} from "./chunk-QOYD2THS.mjs";
|
|
13
|
+
import "./chunk-GT5HPOM3.mjs";
|
|
14
|
+
import "./chunk-722274AA.mjs";
|
|
15
15
|
import "./chunk-DLIRSHV6.mjs";
|
|
16
|
-
import "./chunk-
|
|
16
|
+
import "./chunk-XFTGWGZ5.mjs";
|
|
17
17
|
import "./chunk-NXGCO4JB.mjs";
|
|
18
18
|
import "./chunk-WZYKRD7T.mjs";
|
|
19
19
|
import "./chunk-KRD6H754.mjs";
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
var _chunkBWFUBETDjs = require('./chunk-BWFUBETD.js');
|
|
13
|
+
require('./chunk-7ZAG2XG5.js');
|
|
14
|
+
require('./chunk-NUD2WO3E.js');
|
|
15
|
+
require('./chunk-XNYIZMNQ.js');
|
|
16
|
+
require('./chunk-L7YXXM5W.js');
|
|
17
|
+
require('./chunk-7P5AA3SN.js');
|
|
18
|
+
require('./chunk-OG64RBAC.js');
|
|
19
|
+
require('./chunk-COGTRVWW.js');
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
exports.EXIT_WORKSPACE_MISMATCH = _chunkBWFUBETDjs.EXIT_WORKSPACE_MISMATCH; exports.clearPendingClaim = _chunkBWFUBETDjs.clearPendingClaim; exports.clearSatisfiedPendingClaim = _chunkBWFUBETDjs.clearSatisfiedPendingClaim; exports.handleRegister = _chunkBWFUBETDjs.handleRegister; exports.handleStatus = _chunkBWFUBETDjs.handleStatus; exports.handleWait = _chunkBWFUBETDjs.handleWait; exports.readPendingAuthClaim = _chunkBWFUBETDjs.readPendingAuthClaim; exports.registerAuthCommands = _chunkBWFUBETDjs.registerAuthCommands; exports.reportSetupCompleted = _chunkBWFUBETDjs.reportSetupCompleted; exports.resolvePendingAuthClaimForResume = _chunkBWFUBETDjs.resolvePendingAuthClaimForResume;
|
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
DeeplineClient,
|
|
3
3
|
iterateCursorPages,
|
|
4
4
|
requireBillingIdempotencyKey
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-IKC7EPDS.mjs";
|
|
6
6
|
import "./chunk-CBVIJSHF.mjs";
|
|
7
7
|
import {
|
|
8
8
|
createTimestampedName,
|
|
@@ -10,13 +10,13 @@ import {
|
|
|
10
10
|
openInBrowser,
|
|
11
11
|
printCommandEnvelope,
|
|
12
12
|
shouldEmitJson
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-GT5HPOM3.mjs";
|
|
14
14
|
import {
|
|
15
15
|
AuthError,
|
|
16
16
|
ConfigError
|
|
17
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-722274AA.mjs";
|
|
18
18
|
import "./chunk-DLIRSHV6.mjs";
|
|
19
|
-
import "./chunk-
|
|
19
|
+
import "./chunk-XFTGWGZ5.mjs";
|
|
20
20
|
import "./chunk-6IKARAYD.mjs";
|
|
21
21
|
import {
|
|
22
22
|
DeeplineError
|