@rebasepro/server 0.21.2-canary.g1ea48be → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/{GCSStorageController-CLIJXwGS.js → GCSStorageController-CjrA4PMo.js} +37 -23
  2. package/dist/GCSStorageController-CjrA4PMo.js.map +1 -0
  3. package/dist/{S3StorageController-Dcuf8lMA.js → S3StorageController-B6pKDNVj.js} +2 -2
  4. package/dist/{S3StorageController-Dcuf8lMA.js.map → S3StorageController-B6pKDNVj.js.map} +1 -1
  5. package/dist/api/ast-schema-editor.d.ts +35 -0
  6. package/dist/api/live-schema-routes.d.ts +14 -0
  7. package/dist/api/rest/api-generator.d.ts +117 -35
  8. package/dist/api/rest/auth-collection-writes.d.ts +85 -0
  9. package/dist/api/rest/field-access-query.d.ts +6 -2
  10. package/dist/api/rest/idempotency.d.ts +7 -1
  11. package/dist/api/rest/nested-write-access.d.ts +46 -0
  12. package/dist/api/rest/write-validation.d.ts +32 -0
  13. package/dist/{ast-schema-editor-CslO8Oje.js → ast-schema-editor-Mvr50v_S.js} +104 -4
  14. package/dist/ast-schema-editor-Mvr50v_S.js.map +1 -0
  15. package/dist/auth/address-ownership.d.ts +53 -0
  16. package/dist/auth/admin-user-ops.d.ts +35 -2
  17. package/dist/auth/auth-hooks.d.ts +4 -0
  18. package/dist/auth/captcha.d.ts +5 -0
  19. package/dist/auth/interfaces.d.ts +36 -6
  20. package/dist/auth/jwt.d.ts +17 -0
  21. package/dist/auth/oauth-signin-policy.d.ts +25 -8
  22. package/dist/auth/rate-limiter.d.ts +31 -1
  23. package/dist/auth/session-routes.d.ts +7 -0
  24. package/dist/auth/token-revocation.d.ts +4 -1
  25. package/dist/{auth-DZXQRmYD.js → auth-B-GIMpDG.js} +643 -230
  26. package/dist/auth-B-GIMpDG.js.map +1 -0
  27. package/dist/backend-DTAOsLQc.js +30 -0
  28. package/dist/backend-DTAOsLQc.js.map +1 -0
  29. package/dist/backup/backup-common.d.ts +19 -0
  30. package/dist/{backup-DzI9jLwc.js → backup-D7YR94N3.js} +69 -8
  31. package/dist/backup-D7YR94N3.js.map +1 -0
  32. package/dist/boot/driver.d.ts +10 -0
  33. package/dist/boot/env.d.ts +2 -2
  34. package/dist/boot/fetch-bundle.d.ts +18 -1
  35. package/dist/boot/rls-audit-option.d.ts +26 -0
  36. package/dist/boot/sources.d.ts +1 -0
  37. package/dist/{contract-routes-eLxV0le1.js → contract-routes-CbFjuBwa.js} +2 -2
  38. package/dist/{contract-routes-eLxV0le1.js.map → contract-routes-CbFjuBwa.js.map} +1 -1
  39. package/dist/cron/cron-routes.d.ts +7 -2
  40. package/dist/cron/cron-scheduler.d.ts +128 -8
  41. package/dist/cron/cron-store.d.ts +72 -8
  42. package/dist/cron/index.d.ts +1 -1
  43. package/dist/{cron-loader-CQjvjpEw.js → cron-loader-DfTj2Hbi.js} +2 -2
  44. package/dist/{cron-loader-CQjvjpEw.js.map → cron-loader-DfTj2Hbi.js.map} +1 -1
  45. package/dist/{cron-routes-cv0hSd-O.js → cron-routes-eE8nif_b.js} +33 -12
  46. package/dist/cron-routes-eE8nif_b.js.map +1 -0
  47. package/dist/{cron-scheduler-COPQxlEq.js → cron-scheduler-B0pLfAix.js} +394 -68
  48. package/dist/cron-scheduler-B0pLfAix.js.map +1 -0
  49. package/dist/{cron-store-Bdm7JqFB.js → cron-store-TcoGz-xS.js} +133 -12
  50. package/dist/cron-store-TcoGz-xS.js.map +1 -0
  51. package/dist/{ddl-bootstrap-CfNvxMuK.js → ddl-bootstrap-C6mo0Kmz.js} +2 -25
  52. package/dist/ddl-bootstrap-C6mo0Kmz.js.map +1 -0
  53. package/dist/email/link-base.d.ts +5 -4
  54. package/dist/email/smtp-email-service.d.ts +13 -1
  55. package/dist/email/templates.d.ts +9 -0
  56. package/dist/email/types.d.ts +3 -2
  57. package/dist/env.d.ts +24 -5
  58. package/dist/{history-recorder-CYfkqP2X.js → history-recorder-B4MpJfJK.js} +8 -6
  59. package/dist/history-recorder-B4MpJfJK.js.map +1 -0
  60. package/dist/{history-store-B1CBwLfi.js → history-store-BhxWOuz9.js} +2 -2
  61. package/dist/{history-store-B1CBwLfi.js.map → history-store-BhxWOuz9.js.map} +1 -1
  62. package/dist/index.d.ts +6 -2
  63. package/dist/index.es.js +2769 -878
  64. package/dist/index.es.js.map +1 -1
  65. package/dist/init/docs.d.ts +5 -2
  66. package/dist/init/global-callbacks.d.ts +21 -0
  67. package/dist/init/shutdown.d.ts +8 -3
  68. package/dist/jobs/index.d.ts +2 -2
  69. package/dist/jobs/job-queue.d.ts +23 -2
  70. package/dist/jobs/job-store.d.ts +37 -5
  71. package/dist/jobs/types.d.ts +8 -6
  72. package/dist/{jobs-2fO2BI8P.js → jobs-CazMYhyy.js} +305 -166
  73. package/dist/jobs-CazMYhyy.js.map +1 -0
  74. package/dist/{jwt-C4OW-DNq.js → jwt-DnQHNFCl.js} +77 -25
  75. package/dist/{jwt-C4OW-DNq.js.map → jwt-DnQHNFCl.js.map} +1 -1
  76. package/dist/{keys-Qfc4XieN.js → keys-CogCQpxG.js} +18 -3
  77. package/dist/{keys-Qfc4XieN.js.map → keys-CogCQpxG.js.map} +1 -1
  78. package/dist/{logs-routes-3EEzPjhl.js → logs-routes-Bj4TYYUl.js} +7 -4
  79. package/dist/{logs-routes-3EEzPjhl.js.map → logs-routes-Bj4TYYUl.js.map} +1 -1
  80. package/dist/mcp/mcp-routes.d.ts +38 -2
  81. package/dist/mcp/mcp-tools.d.ts +7 -1
  82. package/dist/mcp/oauth-routes.d.ts +27 -0
  83. package/dist/mcp/oauth-store.d.ts +29 -13
  84. package/dist/metrics/history-recorder.d.ts +1 -1
  85. package/dist/{openapi-generator-D8ZIN_ss.js → openapi-generator-O_O24MAT.js} +34 -12
  86. package/dist/openapi-generator-O_O24MAT.js.map +1 -0
  87. package/dist/{query-parser-CoIOmflr.js → query-parser-DGRVFNM3.js} +39 -30
  88. package/dist/query-parser-DGRVFNM3.js.map +1 -0
  89. package/dist/rls-audit/index.d.ts +4 -0
  90. package/dist/{schema-editor-routes-DdLihzp0.js → schema-editor-routes-C5-lh_jO.js} +2 -2
  91. package/dist/{schema-editor-routes-DdLihzp0.js.map → schema-editor-routes-C5-lh_jO.js.map} +1 -1
  92. package/dist/{src-BYax9_rm.js → src-pmvW7BFx.js} +189 -14
  93. package/dist/src-pmvW7BFx.js.map +1 -0
  94. package/dist/{src-Br6ARbs6.js → src-vkcwKXbT.js} +2 -27
  95. package/dist/{src-Br6ARbs6.js.map → src-vkcwKXbT.js.map} +1 -1
  96. package/dist/storage/GCSStorageController.d.ts +11 -1
  97. package/dist/storage/keys.d.ts +12 -0
  98. package/dist/storage/rendition-cache.d.ts +11 -1
  99. package/dist/storage/request-keys.d.ts +67 -0
  100. package/dist/storage/types.d.ts +17 -1
  101. package/dist/types-BfKcm9do.js.map +1 -1
  102. package/package.json +9 -9
  103. package/dist/GCSStorageController-CLIJXwGS.js.map +0 -1
  104. package/dist/ast-schema-editor-CslO8Oje.js.map +0 -1
  105. package/dist/auth-DZXQRmYD.js.map +0 -1
  106. package/dist/backup-DzI9jLwc.js.map +0 -1
  107. package/dist/cron-routes-cv0hSd-O.js.map +0 -1
  108. package/dist/cron-scheduler-COPQxlEq.js.map +0 -1
  109. package/dist/cron-store-Bdm7JqFB.js.map +0 -1
  110. package/dist/ddl-bootstrap-CfNvxMuK.js.map +0 -1
  111. package/dist/history-recorder-CYfkqP2X.js.map +0 -1
  112. package/dist/jobs-2fO2BI8P.js.map +0 -1
  113. package/dist/openapi-generator-D8ZIN_ss.js.map +0 -1
  114. package/dist/query-parser-CoIOmflr.js.map +0 -1
  115. package/dist/src-BYax9_rm.js.map +0 -1
@@ -3,11 +3,12 @@ import __rebaseProcess from "process";
3
3
  globalThis.process ??= __rebaseProcess;
4
4
  __rebaseCreateRequire(import.meta.url);
5
5
  import { n as __exportAll } from "./rolldown-runtime-dW7B1o5h.js";
6
- import { N as REBASE_USER_ROLE, P as revokeInternalTableSql } from "./src-BYax9_rm.js";
7
- import "./src-Br6ARbs6.js";
8
- import { n as createDdlBootstrapper, o as isSQLAdmin, r as hasInCauseChain } from "./ddl-bootstrap-CfNvxMuK.js";
6
+ import { F as revokeInternalTableSql, P as REBASE_USER_ROLE } from "./src-pmvW7BFx.js";
7
+ import "./src-vkcwKXbT.js";
8
+ import { t as isSQLAdmin } from "./backend-DTAOsLQc.js";
9
9
  import { n as sqlRows, t as firstSqlRow } from "./sql-rows-C6GEc2oE.js";
10
10
  import { r as logger } from "./logger-DO2PZc4i.js";
11
+ import { n as createDdlBootstrapper, r as hasInCauseChain } from "./ddl-bootstrap-C6mo0Kmz.js";
11
12
  import { randomUUID } from "crypto";
12
13
  //#region src/db/ambient-transaction.ts
13
14
  /** Frozen — see above. `server-postgres/src/services/write-transaction-scope.ts` writes it. */
@@ -17,6 +18,255 @@ function currentAmbientTransaction() {
17
18
  return globalThis[RESOLVER_SLOT]?.();
18
19
  }
19
20
  //#endregion
21
+ //#region src/jobs/job-queue.ts
22
+ /**
23
+ * The worker: claim, run, record, repeat.
24
+ *
25
+ * Everything difficult about running jobs concurrently is in `job-store.ts`,
26
+ * where one `UPDATE … FOR UPDATE SKIP LOCKED` does the arbitration. What is
27
+ * left here is a loop and the decisions around a handler that throws.
28
+ */
29
+ var DEFAULT_CONCURRENCY = 5;
30
+ var DEFAULT_POLL_INTERVAL_MS = 2e3;
31
+ var DEFAULT_VISIBILITY_TIMEOUT_MS = 5 * 6e4;
32
+ var DEFAULT_MAX_ATTEMPTS = 3;
33
+ /** 1s, 5s, 25s, 125s … capped at an hour. */
34
+ function defaultBackoff(attempt) {
35
+ return Math.min(1e3 * Math.pow(5, Math.max(0, attempt - 1)), 60 * 6e4);
36
+ }
37
+ /**
38
+ * How often the reaper runs, relative to the visibility timeout.
39
+ *
40
+ * A quarter of it, so a stranded job waits at most 1.25× the timeout rather
41
+ * than 2× — and so the sweep is not itself a per-poll query against a table
42
+ * whose interesting rows are, almost always, none.
43
+ */
44
+ var REAP_INTERVAL_FACTOR = .25;
45
+ /**
46
+ * Thrown by a handler for a failure no retry can change — a destination that
47
+ * is refused, a receiver that redirects. The job is dead-lettered on this
48
+ * attempt, with the message as its `last_error`, instead of spending its
49
+ * remaining attempts proving the same thing. Returning normally is not the
50
+ * alternative: that records the job as succeeded.
51
+ */
52
+ var PermanentJobError = class extends Error {
53
+ constructor(message, options) {
54
+ super(message, options);
55
+ this.name = "PermanentJobError";
56
+ }
57
+ };
58
+ function createJobQueue(store, options = {}) {
59
+ const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
60
+ const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
61
+ const visibilityTimeoutMs = options.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS;
62
+ const defaultMaxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
63
+ const backoff = options.backoff ?? defaultBackoff;
64
+ const handlers = /* @__PURE__ */ new Map();
65
+ for (const [task, handler] of Object.entries(options.tasks ?? {})) handlers.set(task, handler);
66
+ const workerId = `${process.pid}-${randomUUID().slice(0, 8)}`;
67
+ let timer = null;
68
+ let reapTimer = null;
69
+ let running = false;
70
+ /**
71
+ * Every job this worker is running, tracked one by one rather than per
72
+ * claimed batch: a batch would let its slowest job decide when the next
73
+ * claim happens, so one handler that never settled would idle every other
74
+ * slot — and hold `stop()` with it.
75
+ */
76
+ const active = /* @__PURE__ */ new Set();
77
+ /** The claim in progress, so `stop()` also waits for jobs it is about to hand out. */
78
+ let claiming = null;
79
+ /** The last claim filled every free slot, so more work is probably waiting. */
80
+ let backlog = false;
81
+ let reaping = false;
82
+ let draining = null;
83
+ /**
84
+ * Renew `claim` while its handler runs, at a third of the visibility
85
+ * timeout, so the reaper only ever takes a job from a worker that stopped
86
+ * renewing — a dead one — and never from one that is merely slow. Returns
87
+ * the stop.
88
+ */
89
+ function keepClaim(job, claim) {
90
+ if (!store.heartbeat) return () => void 0;
91
+ const renew = store.heartbeat.bind(store);
92
+ const interval = setInterval(() => {
93
+ renew(job.id, claim).then((held) => {
94
+ if (held) return;
95
+ clearInterval(interval);
96
+ logger.warn(`[jobs] "${job.task}" lost its claim on attempt ${claim.attempt} while running; its outcome will not be recorded`, { jobId: job.id });
97
+ }).catch((error) => {
98
+ logger.warn(`[jobs] Could not renew the claim on "${job.task}"`, {
99
+ jobId: job.id,
100
+ error
101
+ });
102
+ });
103
+ }, Math.max(1, Math.floor(visibilityTimeoutMs / 3)));
104
+ interval.unref?.();
105
+ return () => clearInterval(interval);
106
+ }
107
+ async function runJob(job) {
108
+ const handler = handlers.get(job.task);
109
+ const claim = {
110
+ workerId,
111
+ attempt: job.attempts
112
+ };
113
+ if (!handler) {
114
+ logger.warn(`[jobs] No handler registered for task "${job.task}" — returning the job to the queue`);
115
+ await store.fail(job.id, `No handler registered for task "${job.task}"`, job.attempts < job.maxAttempts ? new Date(Date.now() + backoff(job.attempts)) : null, claim);
116
+ return;
117
+ }
118
+ const releaseClaim = keepClaim(job, claim);
119
+ try {
120
+ try {
121
+ await handler({
122
+ id: job.id,
123
+ task: job.task,
124
+ payload: job.payload,
125
+ attempt: job.attempts,
126
+ maxAttempts: job.maxAttempts
127
+ });
128
+ } finally {
129
+ releaseClaim();
130
+ }
131
+ await store.complete(job.id, claim);
132
+ } catch (error) {
133
+ const message = error instanceof Error ? error.stack ?? error.message : String(error);
134
+ const permanent = error instanceof PermanentJobError;
135
+ const willRetry = !permanent && job.attempts < job.maxAttempts;
136
+ await store.fail(job.id, message.slice(0, 4e3), willRetry ? new Date(Date.now() + backoff(job.attempts)) : null, claim);
137
+ if (willRetry) logger.warn(`[jobs] "${job.task}" failed on attempt ${job.attempts}/${job.maxAttempts}; retrying`, { jobId: job.id });
138
+ else if (permanent) logger.error(`[jobs] "${job.task}" failed permanently on attempt ${job.attempts}; not retrying`, {
139
+ jobId: job.id,
140
+ error: message
141
+ });
142
+ else logger.error(`[jobs] "${job.task}" failed permanently after ${job.attempts} attempts`, {
143
+ jobId: job.id,
144
+ error: message
145
+ });
146
+ }
147
+ }
148
+ /**
149
+ * Return jobs stranded by a dead worker. On its own timer, not inside the
150
+ * poll: a poll can be skipped for as long as every slot is busy, and that
151
+ * is exactly when a stranded job most needs a peer to take it back.
152
+ */
153
+ async function reap() {
154
+ if (reaping) return;
155
+ reaping = true;
156
+ try {
157
+ await store.reapExpired(visibilityTimeoutMs);
158
+ } catch (error) {
159
+ logger.error("[jobs] Failed to reclaim expired jobs", { error });
160
+ } finally {
161
+ reaping = false;
162
+ }
163
+ }
164
+ /** Claim as many jobs as there are free slots and start each one on its own. */
165
+ async function claimAndStart() {
166
+ const free = concurrency - active.size;
167
+ if (free <= 0) return [];
168
+ if (handlers.size === 0) return [];
169
+ const jobs = await store.claim(free, workerId, [...handlers.keys()]);
170
+ backlog = jobs.length >= free;
171
+ return jobs.map((job) => {
172
+ const run = runJob(job).catch((error) => {
173
+ logger.error(`[jobs] Could not record the outcome of "${job.task}"`, {
174
+ jobId: job.id,
175
+ error
176
+ });
177
+ }).finally(() => {
178
+ active.delete(run);
179
+ if (backlog) schedule(0);
180
+ });
181
+ active.add(run);
182
+ return run;
183
+ });
184
+ }
185
+ function schedule(delayMs) {
186
+ if (!running) return;
187
+ if (timer) clearTimeout(timer);
188
+ timer = setTimeout(() => {
189
+ timer = null;
190
+ tick();
191
+ }, delayMs);
192
+ timer.unref?.();
193
+ }
194
+ async function tick() {
195
+ if (!running || claiming) return;
196
+ const work = claimAndStart().catch((error) => {
197
+ logger.error("[jobs] Poll failed", { error });
198
+ backlog = false;
199
+ });
200
+ claiming = work;
201
+ await work;
202
+ claiming = null;
203
+ schedule(backlog && active.size < concurrency ? 0 : pollIntervalMs);
204
+ }
205
+ async function drain() {
206
+ await claiming;
207
+ await Promise.allSettled([...active]);
208
+ }
209
+ return {
210
+ start() {
211
+ if (running) return;
212
+ running = true;
213
+ logger.info(`[jobs] Worker started (concurrency ${concurrency}, poll ${pollIntervalMs}ms)`);
214
+ reap();
215
+ reapTimer = setInterval(() => {
216
+ reap();
217
+ }, visibilityTimeoutMs * REAP_INTERVAL_FACTOR);
218
+ reapTimer.unref?.();
219
+ schedule(0);
220
+ },
221
+ async stop(timeoutMs) {
222
+ running = false;
223
+ if (timer) {
224
+ clearTimeout(timer);
225
+ timer = null;
226
+ }
227
+ if (reapTimer) {
228
+ clearInterval(reapTimer);
229
+ reapTimer = null;
230
+ }
231
+ draining ??= drain().finally(() => {
232
+ draining = null;
233
+ });
234
+ if (timeoutMs === void 0) {
235
+ await draining;
236
+ return;
237
+ }
238
+ let budget;
239
+ const finished = await Promise.race([draining.then(() => true), new Promise((resolve) => {
240
+ budget = setTimeout(() => resolve(false), timeoutMs);
241
+ budget.unref?.();
242
+ })]);
243
+ clearTimeout(budget);
244
+ if (!finished) logger.warn(`[jobs] ${active.size} job(s) still running after ${timeoutMs}ms; stopping without them. They keep their claim and are retried once the visibility timeout reclaims it.`);
245
+ },
246
+ async runOnce() {
247
+ const started = await claimAndStart();
248
+ await Promise.allSettled(started);
249
+ return started.length;
250
+ },
251
+ register(task, handler) {
252
+ if (handlers.has(task)) logger.warn(`[jobs] Task "${task}" was already registered; the later handler wins`);
253
+ handlers.set(task, handler);
254
+ },
255
+ isRunning() {
256
+ return running;
257
+ },
258
+ async enqueue(task, payload, enqueueOptions = {}) {
259
+ return store.insert({
260
+ task,
261
+ payload: payload ?? null,
262
+ runAt: new Date(Date.now() + (enqueueOptions.delayMs ?? 0)),
263
+ maxAttempts: enqueueOptions.maxAttempts ?? defaultMaxAttempts,
264
+ idempotencyKey: enqueueOptions.idempotencyKey
265
+ });
266
+ }
267
+ };
268
+ }
269
+ //#endregion
20
270
  //#region src/jobs/job-store.ts
21
271
  /**
22
272
  * The queue's storage, as SQL against `rebase.jobs`.
@@ -77,6 +327,11 @@ function isUniqueViolation(err) {
77
327
  */
78
328
  var ENQUEUE_FUNCTION = "rebase.enqueue_job";
79
329
  var ENQUEUE_SIGNATURE = `${ENQUEUE_FUNCTION}(text, jsonb, timestamptz, integer, text)`;
330
+ /** The rows a claim's writes may touch: `$1` the job, `$2` the worker, `$3` the attempt. */
331
+ var HELD_BY_CLAIM = "id = $1 AND status = 'running' AND locked_by = $2 AND attempts = $3";
332
+ function warnStaleOutcome(id, claim, outcome) {
333
+ logger.warn(`[jobs] Ignored the ${outcome} of attempt ${claim.attempt} of job ${id}: the job was reclaimed from this worker while it ran, so that attempt's outcome is no longer the job's.`);
334
+ }
80
335
  function createJobStore(driver) {
81
336
  const admin = driver.admin;
82
337
  if (!isSQLAdmin(admin)) {
@@ -184,7 +439,9 @@ function createJobStore(driver) {
184
439
  throw error;
185
440
  }
186
441
  },
187
- async claim(limit, workerId) {
442
+ async claim(limit, workerId, tasks) {
443
+ if (tasks && tasks.length === 0) return [];
444
+ const taskFilter = tasks ? `AND task IN (${tasks.map((_, i) => `$${i + 3}`).join(", ")})` : "";
188
445
  return sqlRows(await exec(`UPDATE ${TABLE} SET
189
446
  status = 'running',
190
447
  attempts = attempts + 1,
@@ -193,32 +450,55 @@ function createJobStore(driver) {
193
450
  updated_at = now()
194
451
  WHERE id IN (
195
452
  SELECT id FROM ${TABLE}
196
- WHERE status = 'pending' AND run_at <= now()
453
+ WHERE status = 'pending' AND run_at <= now() ${taskFilter}
197
454
  ORDER BY run_at, created_at
198
455
  LIMIT $1
199
456
  FOR UPDATE SKIP LOCKED
200
457
  )
201
- RETURNING *`, [limit, workerId])).map(toRecord);
458
+ RETURNING *`, [
459
+ limit,
460
+ workerId,
461
+ ...tasks ?? []
462
+ ])).map(toRecord);
202
463
  },
203
- async complete(id) {
204
- await exec(`UPDATE ${TABLE} SET status = 'succeeded', locked_at = NULL, locked_by = NULL,
464
+ async complete(id, claim) {
465
+ if (sqlRows(await exec(`UPDATE ${TABLE} SET status = 'succeeded', locked_at = NULL, locked_by = NULL,
205
466
  last_error = NULL, updated_at = now()
206
- WHERE id = $1`, [id]);
467
+ WHERE ${HELD_BY_CLAIM}
468
+ RETURNING id`, [
469
+ id,
470
+ claim.workerId,
471
+ claim.attempt
472
+ ])).length === 0) warnStaleOutcome(id, claim, "success");
207
473
  },
208
- async fail(id, error, retryAt) {
209
- if (retryAt) {
210
- await exec(`UPDATE ${TABLE} SET status = 'pending', run_at = $2, locked_at = NULL,
211
- locked_by = NULL, last_error = $3, updated_at = now()
212
- WHERE id = $1`, [
213
- id,
214
- retryAt.toISOString(),
215
- error
216
- ]);
217
- return;
218
- }
219
- await exec(`UPDATE ${TABLE} SET status = 'failed', locked_at = NULL, locked_by = NULL,
220
- last_error = $2, updated_at = now()
221
- WHERE id = $1`, [id, error]);
474
+ async fail(id, error, retryAt, claim) {
475
+ if (sqlRows(retryAt ? await exec(`UPDATE ${TABLE} SET status = 'pending', run_at = $4, locked_at = NULL,
476
+ locked_by = NULL, last_error = $5, updated_at = now()
477
+ WHERE ${HELD_BY_CLAIM}
478
+ RETURNING id`, [
479
+ id,
480
+ claim.workerId,
481
+ claim.attempt,
482
+ retryAt.toISOString(),
483
+ error
484
+ ]) : await exec(`UPDATE ${TABLE} SET status = 'failed', locked_at = NULL, locked_by = NULL,
485
+ last_error = $4, updated_at = now()
486
+ WHERE ${HELD_BY_CLAIM}
487
+ RETURNING id`, [
488
+ id,
489
+ claim.workerId,
490
+ claim.attempt,
491
+ error
492
+ ])).length === 0) warnStaleOutcome(id, claim, "failure");
493
+ },
494
+ async heartbeat(id, claim) {
495
+ return sqlRows(await exec(`UPDATE ${TABLE} SET locked_at = now()
496
+ WHERE ${HELD_BY_CLAIM}
497
+ RETURNING id`, [
498
+ id,
499
+ claim.workerId,
500
+ claim.attempt
501
+ ])).length > 0;
222
502
  },
223
503
  async reapExpired(visibilityTimeoutMs) {
224
504
  const seconds = Math.max(1, Math.round(visibilityTimeoutMs / 1e3));
@@ -245,153 +525,12 @@ function createJobStore(driver) {
245
525
  };
246
526
  }
247
527
  //#endregion
248
- //#region src/jobs/job-queue.ts
249
- /**
250
- * The worker: claim, run, record, repeat.
251
- *
252
- * Everything difficult about running jobs concurrently is in `job-store.ts`,
253
- * where one `UPDATE … FOR UPDATE SKIP LOCKED` does the arbitration. What is
254
- * left here is a loop and the decisions around a handler that throws.
255
- */
256
- var DEFAULT_CONCURRENCY = 5;
257
- var DEFAULT_POLL_INTERVAL_MS = 2e3;
258
- var DEFAULT_VISIBILITY_TIMEOUT_MS = 5 * 6e4;
259
- var DEFAULT_MAX_ATTEMPTS = 3;
260
- /** 1s, 5s, 25s, 125s … capped at an hour. */
261
- function defaultBackoff(attempt) {
262
- return Math.min(1e3 * Math.pow(5, Math.max(0, attempt - 1)), 60 * 6e4);
263
- }
264
- /**
265
- * How often the reaper runs, relative to the visibility timeout.
266
- *
267
- * A quarter of it, so a stranded job waits at most 1.25× the timeout rather
268
- * than 2× — and so the sweep is not itself a per-poll query against a table
269
- * whose interesting rows are, almost always, none.
270
- */
271
- var REAP_INTERVAL_FACTOR = .25;
272
- function createJobQueue(store, options = {}) {
273
- const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
274
- const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
275
- const visibilityTimeoutMs = options.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS;
276
- const defaultMaxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
277
- const backoff = options.backoff ?? defaultBackoff;
278
- const handlers = /* @__PURE__ */ new Map();
279
- for (const [task, handler] of Object.entries(options.tasks ?? {})) handlers.set(task, handler);
280
- const workerId = `${process.pid}-${randomUUID().slice(0, 8)}`;
281
- let timer = null;
282
- let running = false;
283
- let draining = false;
284
- /** Resolves when the in-flight poll finishes, so `stop()` can wait for it. */
285
- let inFlight = Promise.resolve();
286
- let lastReapAt = 0;
287
- async function runJob(job) {
288
- const handler = handlers.get(job.task);
289
- if (!handler) {
290
- logger.warn(`[jobs] No handler registered for task "${job.task}" — returning the job to the queue`);
291
- await store.fail(job.id, `No handler registered for task "${job.task}"`, job.attempts < job.maxAttempts ? new Date(Date.now() + backoff(job.attempts)) : null);
292
- return;
293
- }
294
- try {
295
- await handler({
296
- id: job.id,
297
- task: job.task,
298
- payload: job.payload,
299
- attempt: job.attempts,
300
- maxAttempts: job.maxAttempts
301
- });
302
- await store.complete(job.id);
303
- } catch (error) {
304
- const message = error instanceof Error ? error.stack ?? error.message : String(error);
305
- const willRetry = job.attempts < job.maxAttempts;
306
- await store.fail(job.id, message.slice(0, 4e3), willRetry ? new Date(Date.now() + backoff(job.attempts)) : null);
307
- if (willRetry) logger.warn(`[jobs] "${job.task}" failed on attempt ${job.attempts}/${job.maxAttempts}; retrying`, { jobId: job.id });
308
- else logger.error(`[jobs] "${job.task}" failed permanently after ${job.attempts} attempts`, {
309
- jobId: job.id,
310
- error: message
311
- });
312
- }
313
- }
314
- async function poll() {
315
- const now = Date.now();
316
- if (now - lastReapAt > visibilityTimeoutMs * REAP_INTERVAL_FACTOR) {
317
- lastReapAt = now;
318
- try {
319
- await store.reapExpired(visibilityTimeoutMs);
320
- } catch (error) {
321
- logger.error("[jobs] Failed to reclaim expired jobs", { error });
322
- }
323
- }
324
- const jobs = await store.claim(concurrency, workerId);
325
- if (jobs.length === 0) return 0;
326
- await Promise.allSettled(jobs.map(runJob));
327
- return jobs.length;
328
- }
329
- function schedule(delayMs) {
330
- if (!running) return;
331
- timer = setTimeout(() => {
332
- tick();
333
- }, delayMs);
334
- timer.unref?.();
335
- }
336
- async function tick() {
337
- if (!running) return;
338
- const work = (async () => {
339
- try {
340
- return await poll();
341
- } catch (error) {
342
- logger.error("[jobs] Poll failed", { error });
343
- return 0;
344
- }
345
- })();
346
- inFlight = work;
347
- schedule(await work >= concurrency ? 0 : pollIntervalMs);
348
- }
349
- return {
350
- start() {
351
- if (running) return;
352
- running = true;
353
- logger.info(`[jobs] Worker started (concurrency ${concurrency}, poll ${pollIntervalMs}ms)`);
354
- schedule(0);
355
- },
356
- async stop() {
357
- running = false;
358
- if (timer) {
359
- clearTimeout(timer);
360
- timer = null;
361
- }
362
- if (draining) return;
363
- draining = true;
364
- await inFlight.catch(() => void 0);
365
- draining = false;
366
- },
367
- runOnce() {
368
- return poll();
369
- },
370
- register(task, handler) {
371
- if (handlers.has(task)) logger.warn(`[jobs] Task "${task}" was already registered; the later handler wins`);
372
- handlers.set(task, handler);
373
- },
374
- isRunning() {
375
- return running;
376
- },
377
- async enqueue(task, payload, enqueueOptions = {}) {
378
- return store.insert({
379
- task,
380
- payload: payload ?? null,
381
- runAt: new Date(Date.now() + (enqueueOptions.delayMs ?? 0)),
382
- maxAttempts: enqueueOptions.maxAttempts ?? defaultMaxAttempts,
383
- idempotencyKey: enqueueOptions.idempotencyKey
384
- });
385
- }
386
- };
387
- }
388
- //#endregion
389
528
  //#region src/jobs/index.ts
390
529
  var jobs_exports = /* @__PURE__ */ __exportAll({
391
530
  createJobQueue: () => createJobQueue,
392
531
  createJobStore: () => createJobStore
393
532
  });
394
533
  //#endregion
395
- export { currentAmbientTransaction as a, createJobStore as i, createJobQueue as n, defaultBackoff as r, jobs_exports as t };
534
+ export { defaultBackoff as a, createJobQueue as i, createJobStore as n, currentAmbientTransaction as o, PermanentJobError as r, jobs_exports as t };
396
535
 
397
- //# sourceMappingURL=jobs-2fO2BI8P.js.map
536
+ //# sourceMappingURL=jobs-CazMYhyy.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jobs-CazMYhyy.js","names":[],"sources":["../src/db/ambient-transaction.ts","../src/jobs/job-queue.ts","../src/jobs/job-store.ts","../src/jobs/index.ts"],"sourcesContent":["/**\n * The write transaction the current code is running inside, when there is one.\n *\n * A collection callback runs inside the transaction that carries its write.\n * Work it hands off — a job, a queue message, a topic event, a webhook\n * delivery — has to share that write's fate: queued when it commits, never\n * queued when it rolls back, and invisible to a worker until the commit. The\n * code that queues it lives in this package and knows no driver; the\n * transaction belongs to the driver. So the driver publishes the transaction it\n * opened for a write, and the queueing code asks for it here.\n *\n * Until this existed the job store wrote through the default driver's own\n * connection, in autocommit, whatever called it. A job enqueued from an\n * `afterSave` that then threw stayed queued for a row that was never written,\n * and a worker could claim it before the row it was about committed.\n *\n * The resolver lives on `globalThis` under a registered symbol, like the\n * `rebase` singleton: a managed runtime can load two copies of this package,\n * and the copy a driver registers with must be the one the job store reads.\n * No Node imports — the store is reachable from code that has to stay portable.\n *\n * **The symbol's key is a contract between packages, and frozen.** The\n * Postgres driver writes the slot itself rather than importing a setter from\n * here: the managed runtime supplies this package from the image while a\n * bundle brings its own driver, and a driver importing a function an older\n * image does not export would fail to link and take the boot down. Written\n * through the slot, any pairing of versions degrades to the old behaviour —\n * jobs committing on their own — instead.\n */\nexport interface AmbientTransaction {\n /**\n * Run one statement on the transaction, `$1…` placeholders as\n * `executeSql` takes them. It runs as the transaction's current role — on\n * Postgres the request's restricted one — so a statement that needs more\n * goes through a function granted to that role.\n */\n exec(sqlText: string, params?: unknown[]): Promise<Record<string, unknown>[]>;\n /** Run `fn` after the transaction commits. Never runs if it rolls back. */\n afterCommit(fn: () => void): void;\n}\n\n/** Frozen — see above. `server-postgres/src/services/write-transaction-scope.ts` writes it. */\nconst RESOLVER_SLOT = Symbol.for(\"rebase.server.ambientTransactionResolver\");\n\ntype ResolverSlot = { [RESOLVER_SLOT]?: () => AmbientTransaction | undefined };\n\n/**\n * Register how to find the write transaction the caller is inside; pass\n * `undefined` to unregister. For tests and for drivers in this repository's\n * lockstep — a driver shipped separately writes the slot, as above.\n */\nexport function setAmbientTransactionResolver(resolve: (() => AmbientTransaction | undefined) | undefined): void {\n (globalThis as ResolverSlot)[RESOLVER_SLOT] = resolve;\n}\n\n/** The write transaction the caller is inside, or `undefined` outside one. */\nexport function currentAmbientTransaction(): AmbientTransaction | undefined {\n return (globalThis as ResolverSlot)[RESOLVER_SLOT]?.();\n}\n","import { randomUUID } from \"crypto\";\nimport { logger } from \"../utils/logger.js\";\nimport type { JobClaim, JobStore } from \"./job-store.js\";\nimport type { EnqueueOptions, JobHandler, JobQueueClient, JobQueueOptions, JobRecord } from \"./types.js\";\n\n/**\n * The worker: claim, run, record, repeat.\n *\n * Everything difficult about running jobs concurrently is in `job-store.ts`,\n * where one `UPDATE … FOR UPDATE SKIP LOCKED` does the arbitration. What is\n * left here is a loop and the decisions around a handler that throws.\n */\n\nconst DEFAULT_CONCURRENCY = 5;\nconst DEFAULT_POLL_INTERVAL_MS = 2_000;\nconst DEFAULT_VISIBILITY_TIMEOUT_MS = 5 * 60_000;\nconst DEFAULT_MAX_ATTEMPTS = 3;\n\n/** 1s, 5s, 25s, 125s … capped at an hour. */\nexport function defaultBackoff(attempt: number): number {\n return Math.min(1_000 * Math.pow(5, Math.max(0, attempt - 1)), 60 * 60_000);\n}\n\n/**\n * How often the reaper runs, relative to the visibility timeout.\n *\n * A quarter of it, so a stranded job waits at most 1.25× the timeout rather\n * than 2× — and so the sweep is not itself a per-poll query against a table\n * whose interesting rows are, almost always, none.\n */\nconst REAP_INTERVAL_FACTOR = 0.25;\n\n/**\n * Thrown by a handler for a failure no retry can change — a destination that\n * is refused, a receiver that redirects. The job is dead-lettered on this\n * attempt, with the message as its `last_error`, instead of spending its\n * remaining attempts proving the same thing. Returning normally is not the\n * alternative: that records the job as succeeded.\n */\nexport class PermanentJobError extends Error {\n constructor(message: string, options?: { cause?: unknown }) {\n super(message, options);\n this.name = \"PermanentJobError\";\n }\n}\n\nexport interface JobQueue extends JobQueueClient {\n start(): void;\n /**\n * Stop claiming, and wait for the jobs in flight — for at most `timeoutMs`\n * when given. A job still running when the budget runs out keeps its claim\n * and is recovered by the visibility timeout; waiting on it without a bound\n * would let one handler that never settles hold the whole shutdown.\n */\n stop(timeoutMs?: number): Promise<void>;\n /**\n * Claim what fits in the free slots, run it, and resolve with how many jobs\n * ran. For tests and for `/jobs/drain`; not meant to run beside `start()`.\n */\n runOnce(): Promise<number>;\n /** Registered after construction — how `tasks` from config and internal producers meet. */\n register<P = unknown>(task: string, handler: JobHandler<P>): void;\n isRunning(): boolean;\n}\n\nexport function createJobQueue(store: JobStore, options: JobQueueOptions = {}): JobQueue {\n const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;\n const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;\n const visibilityTimeoutMs = options.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS;\n const defaultMaxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;\n const backoff = options.backoff ?? defaultBackoff;\n\n const handlers = new Map<string, JobHandler<never>>();\n for (const [task, handler] of Object.entries(options.tasks ?? {})) {\n handlers.set(task, handler);\n }\n\n // Identifies this process in `locked_by`: \"which pod had it when it\n // stopped\" is the first question anyone asks of a stuck job, and with the\n // attempt it is the claim an outcome is written under (see `JobClaim`).\n const workerId = `${process.pid}-${randomUUID().slice(0, 8)}`;\n\n let timer: NodeJS.Timeout | null = null;\n let reapTimer: NodeJS.Timeout | null = null;\n let running = false;\n /**\n * Every job this worker is running, tracked one by one rather than per\n * claimed batch: a batch would let its slowest job decide when the next\n * claim happens, so one handler that never settled would idle every other\n * slot — and hold `stop()` with it.\n */\n const active = new Set<Promise<void>>();\n /** The claim in progress, so `stop()` also waits for jobs it is about to hand out. */\n let claiming: Promise<unknown> | null = null;\n /** The last claim filled every free slot, so more work is probably waiting. */\n let backlog = false;\n let reaping = false;\n let draining: Promise<void> | null = null;\n\n /**\n * Renew `claim` while its handler runs, at a third of the visibility\n * timeout, so the reaper only ever takes a job from a worker that stopped\n * renewing — a dead one — and never from one that is merely slow. Returns\n * the stop.\n */\n function keepClaim(job: JobRecord, claim: JobClaim): () => void {\n if (!store.heartbeat) return () => undefined;\n const renew = store.heartbeat.bind(store);\n const interval = setInterval(() => {\n renew(job.id, claim).then((held) => {\n if (held) return;\n clearInterval(interval);\n logger.warn(\n `[jobs] \"${job.task}\" lost its claim on attempt ${claim.attempt} while running; ` +\n \"its outcome will not be recorded\", { jobId: job.id }\n );\n }).catch((error: unknown) => {\n logger.warn(`[jobs] Could not renew the claim on \"${job.task}\"`, { jobId: job.id, error });\n });\n }, Math.max(1, Math.floor(visibilityTimeoutMs / 3)));\n interval.unref?.();\n return () => clearInterval(interval);\n }\n\n async function runJob(job: JobRecord): Promise<void> {\n const handler = handlers.get(job.task);\n const claim: JobClaim = { workerId, attempt: job.attempts };\n\n if (!handler) {\n // Only reachable through a store that ignores the `tasks` filter on\n // `claim` — the one that ships never hands this worker a task it\n // has no handler for. Give the row back for a peer that has one.\n // The attempt the claim spent is not undone, so a task nobody in\n // the fleet implements dead-letters with an error naming it rather\n // than cycling through such a store forever.\n logger.warn(`[jobs] No handler registered for task \"${job.task}\" — returning the job to the queue`);\n await store.fail(\n job.id,\n `No handler registered for task \"${job.task}\"`,\n job.attempts < job.maxAttempts ? new Date(Date.now() + backoff(job.attempts)) : null,\n claim\n );\n return;\n }\n\n const releaseClaim = keepClaim(job, claim);\n try {\n try {\n await handler({\n id: job.id,\n task: job.task,\n payload: job.payload as never,\n attempt: job.attempts,\n maxAttempts: job.maxAttempts\n } as never);\n } finally {\n releaseClaim();\n }\n await store.complete(job.id, claim);\n } catch (error) {\n const message = error instanceof Error ? (error.stack ?? error.message) : String(error);\n const permanent = error instanceof PermanentJobError;\n const willRetry = !permanent && job.attempts < job.maxAttempts;\n\n // Truncated, because `last_error` holds a stack and a queue that\n // accumulates megabytes of them is its own outage.\n await store.fail(job.id, message.slice(0, 4_000), willRetry ? new Date(Date.now() + backoff(job.attempts)) : null, claim);\n\n if (willRetry) {\n logger.warn(`[jobs] \"${job.task}\" failed on attempt ${job.attempts}/${job.maxAttempts}; retrying`, { jobId: job.id });\n } else if (permanent) {\n logger.error(`[jobs] \"${job.task}\" failed permanently on attempt ${job.attempts}; not retrying`, { jobId: job.id, error: message });\n } else {\n // The last attempt is an error, not a warning: nothing else will\n // touch this job, and if nobody looks at the table it is simply\n // lost work.\n logger.error(`[jobs] \"${job.task}\" failed permanently after ${job.attempts} attempts`, { jobId: job.id, error: message });\n }\n }\n }\n\n /**\n * Return jobs stranded by a dead worker. On its own timer, not inside the\n * poll: a poll can be skipped for as long as every slot is busy, and that\n * is exactly when a stranded job most needs a peer to take it back.\n */\n async function reap(): Promise<void> {\n if (reaping) return;\n reaping = true;\n try {\n await store.reapExpired(visibilityTimeoutMs);\n } catch (error) {\n logger.error(\"[jobs] Failed to reclaim expired jobs\", { error });\n } finally {\n reaping = false;\n }\n }\n\n /** Claim as many jobs as there are free slots and start each one on its own. */\n async function claimAndStart(): Promise<Promise<void>[]> {\n const free = concurrency - active.size;\n if (free <= 0) return [];\n\n // Only the tasks this worker can run. During a rolling deploy an\n // instance on older code shares the table with jobs whose task only\n // the newer code implements, and a claim spends an attempt: claiming\n // them here would dead-letter them before an updated peer got to them.\n // A handler registered after `start()` joins the next claim.\n if (handlers.size === 0) return [];\n const jobs = await store.claim(free, workerId, [...handlers.keys()]);\n backlog = jobs.length >= free;\n\n return jobs.map((job) => {\n const run: Promise<void> = runJob(job)\n .catch((error) => {\n // `runJob` handles a throwing handler; this is a store\n // write failing after it. The job keeps its claim, so the\n // visibility timeout recovers it.\n logger.error(`[jobs] Could not record the outcome of \"${job.task}\"`, { jobId: job.id, error });\n })\n .finally(() => {\n active.delete(run);\n // A slot just came free and there is work waiting: take the\n // next job now, not a poll interval from now.\n if (backlog) schedule(0);\n });\n active.add(run);\n return run;\n });\n }\n\n function schedule(delayMs: number): void {\n if (!running) return;\n if (timer) clearTimeout(timer);\n timer = setTimeout(() => {\n timer = null;\n void tick();\n }, delayMs);\n // Never hold the process open. A queue with nothing to do should not be\n // the reason `rebase dev` will not exit.\n timer.unref?.();\n }\n\n async function tick(): Promise<void> {\n // A claim already in progress reschedules when it finishes.\n if (!running || claiming) return;\n const work = claimAndStart().catch((error) => {\n logger.error(\"[jobs] Poll failed\", { error });\n backlog = false;\n });\n claiming = work;\n await work;\n claiming = null;\n\n // A full claim with a slot still free — a job already finished — means\n // go straight back. Otherwise sleep; a job finishing during a backlog\n // wakes the loop early.\n schedule(backlog && active.size < concurrency ? 0 : pollIntervalMs);\n }\n\n async function drain(): Promise<void> {\n await claiming;\n await Promise.allSettled([...active]);\n }\n\n return {\n start(): void {\n if (running) return;\n running = true;\n logger.info(`[jobs] Worker started (concurrency ${concurrency}, poll ${pollIntervalMs}ms)`);\n void reap();\n reapTimer = setInterval(() => { void reap(); }, visibilityTimeoutMs * REAP_INTERVAL_FACTOR);\n reapTimer.unref?.();\n schedule(0);\n },\n\n async stop(timeoutMs?: number): Promise<void> {\n running = false;\n if (timer) {\n clearTimeout(timer);\n timer = null;\n }\n if (reapTimer) {\n clearInterval(reapTimer);\n reapTimer = null;\n }\n // Jobs in flight keep their claim until they finish or the\n // visibility timeout expires, so waiting here is what turns a\n // graceful shutdown into \"no job runs twice\".\n draining ??= drain().finally(() => { draining = null; });\n if (timeoutMs === undefined) {\n await draining;\n return;\n }\n let budget: NodeJS.Timeout | undefined;\n const finished = await Promise.race([\n draining.then(() => true),\n new Promise<false>((resolve) => {\n budget = setTimeout(() => resolve(false), timeoutMs);\n budget.unref?.();\n })\n ]);\n clearTimeout(budget);\n if (!finished) {\n logger.warn(\n `[jobs] ${active.size} job(s) still running after ${timeoutMs}ms; stopping without them. ` +\n \"They keep their claim and are retried once the visibility timeout reclaims it.\"\n );\n }\n },\n\n async runOnce(): Promise<number> {\n const started = await claimAndStart();\n await Promise.allSettled(started);\n return started.length;\n },\n\n register<P = unknown>(task: string, handler: JobHandler<P>): void {\n if (handlers.has(task)) {\n logger.warn(`[jobs] Task \"${task}\" was already registered; the later handler wins`);\n }\n handlers.set(task, handler as JobHandler<never>);\n },\n\n isRunning(): boolean {\n return running;\n },\n\n async enqueue<P = unknown>(task: string, payload?: P, enqueueOptions: EnqueueOptions = {}): Promise<string | null> {\n return store.insert({\n task,\n payload: payload ?? null,\n runAt: new Date(Date.now() + (enqueueOptions.delayMs ?? 0)),\n maxAttempts: enqueueOptions.maxAttempts ?? defaultMaxAttempts,\n idempotencyKey: enqueueOptions.idempotencyKey\n });\n }\n };\n}\n","import type { DataDriver } from \"@rebasepro/types\";\nimport { isSQLAdmin } from \"@rebasepro/types\";\nimport { REBASE_USER_ROLE, revokeInternalTableSql, sqlRows, firstSqlRow } from \"@rebasepro/common\";\nimport { logger } from \"../utils/logger.js\";\nimport { createDdlBootstrapper, hasInCauseChain, type SqlExec } from \"../boot/ddl-bootstrap.js\";\nimport { currentAmbientTransaction } from \"../db/ambient-transaction.js\";\nimport type { JobRecord } from \"./types.js\";\n\n/**\n * The queue's storage, as SQL against `rebase.jobs`.\n *\n * Split from the worker for the same reason `cron-store` is split from\n * `cron-scheduler`: the interesting parts here are three statements that have\n * to be exactly right under concurrency, and they are much easier to reason\n * about — and to test — away from a polling loop.\n */\n\nconst TABLE = \"rebase.jobs\";\n\n/** How long finished jobs are kept before the boot sweep removes them. */\nconst SUCCEEDED_RETENTION_DAYS = 3;\n/**\n * Failures outlive successes by a lot. A dead-lettered job is evidence, and the\n * person who needs it is usually looking on Monday for something that happened\n * on Friday night.\n */\nconst FAILED_RETENTION_DAYS = 30;\n\n/** A row as Postgres returns it. */\ninterface JobRow {\n id: string;\n task: string;\n payload: unknown;\n status: string;\n run_at: string;\n attempts: number;\n max_attempts: number;\n last_error: string | null;\n created_at: string;\n updated_at: string;\n}\n\nfunction toRecord(row: JobRow): JobRecord {\n return {\n id: row.id,\n task: row.task,\n payload: row.payload,\n status: row.status as JobRecord[\"status\"],\n runAt: new Date(row.run_at).toISOString(),\n attempts: Number(row.attempts),\n maxAttempts: Number(row.max_attempts),\n lastError: row.last_error,\n createdAt: new Date(row.created_at).toISOString(),\n updatedAt: new Date(row.updated_at).toISOString()\n };\n}\n\n/** Same rule as `cron-store`: match the SQLSTATE, never the message. */\nfunction isUniqueViolation(err: unknown): boolean {\n return hasInCauseChain(err, (e) => e.code === \"23505\");\n}\n\n/**\n * How a job gets into `rebase.jobs` from inside somebody else's transaction.\n *\n * An enqueue from a collection callback has to ride the write's transaction —\n * committed with it, discarded with it, invisible to workers until then — and\n * that transaction runs as the request's restricted role, which has no access\n * to `rebase.jobs` at all (see the revoke in `ensureTable`). This function is\n * the one door: `SECURITY DEFINER`, so the insert runs with the owner's rights\n * for that one statement and the transaction's role never changes. Switching\n * the role for the insert instead would elevate the whole connection for as\n * long as the insert took, and a callback's un-awaited statement would run\n * in that window as the owner.\n *\n * The door is narrow on purpose. It takes the five columns an enqueue sets and\n * nothing else, so it cannot claim, complete or rewrite a job — and only the\n * request role is granted it, not PUBLIC. That role runs statements the server\n * writes; nobody outside gets SQL as it.\n *\n * `ON CONFLICT DO NOTHING` rather than a caught unique violation: an error\n * inside a transaction aborts the whole transaction, so an idempotency clash\n * would have failed the customer's write. A clash returns no row, hence NULL.\n */\nconst ENQUEUE_FUNCTION = \"rebase.enqueue_job\";\nconst ENQUEUE_SIGNATURE = `${ENQUEUE_FUNCTION}(text, jsonb, timestamptz, integer, text)`;\n\n/**\n * Which claim an outcome belongs to: the worker holding the job, and the\n * attempt that claim spent.\n *\n * The visibility timeout hands a job to a second worker when the first stops\n * renewing its claim, and the first may still be running — slow, not dead. Its\n * outcome is then about an attempt the table has moved past, and writing it\n * would mark a running job `succeeded`, or send a finished one back to\n * `pending` for another run. Both writes therefore apply only while the claim\n * they name still holds the row. The attempt is part of the identity because a\n * worker's own reaper can hand a job back to the same worker in another slot.\n */\nexport interface JobClaim {\n workerId: string;\n attempt: number;\n}\n\nexport interface JobStore {\n ensureTable(): Promise<void>;\n /** Returns the new job's id, or `null` if an idempotency key matched unfinished work. */\n insert(job: {\n task: string;\n payload: unknown;\n runAt: Date;\n maxAttempts: number;\n idempotencyKey?: string;\n }): Promise<string | null>;\n /**\n * Atomically take up to `limit` runnable jobs for this worker — only jobs\n * whose task is in `tasks`, when it is given. A worker never claims work it\n * cannot run: a claim spends an attempt, so an instance running older code\n * during a rollout would otherwise dead-letter the newer code's jobs.\n */\n claim(limit: number, workerId: string, tasks?: readonly string[]): Promise<JobRecord[]>;\n /** Mark the job `succeeded` — a no-op once `claim` no longer holds it. */\n complete(id: string, claim: JobClaim): Promise<void>;\n /**\n * Back to `pending` with a later `runAt`, or `failed` when out of attempts —\n * a no-op once `claim` no longer holds it.\n */\n fail(id: string, error: string, retryAt: Date | null, claim: JobClaim): Promise<void>;\n /**\n * Renew `claim` on a job whose handler is still running, so the visibility\n * timeout only ever reclaims a job from a worker that stopped renewing.\n * Resolves with whether the claim still held. A store without it leaves a\n * handler that outlives the timeout to be run a second time beside itself.\n */\n heartbeat?(id: string, claim: JobClaim): Promise<boolean>;\n /** Return jobs stranded by a worker that died holding them. Resolves with how many. */\n reapExpired(visibilityTimeoutMs: number): Promise<number>;\n fetch(id: string): Promise<JobRecord | null>;\n}\n\n/** The rows a claim's writes may touch: `$1` the job, `$2` the worker, `$3` the attempt. */\nconst HELD_BY_CLAIM = \"id = $1 AND status = 'running' AND locked_by = $2 AND attempts = $3\";\n\nfunction warnStaleOutcome(id: string, claim: JobClaim, outcome: \"success\" | \"failure\"): void {\n logger.warn(\n `[jobs] Ignored the ${outcome} of attempt ${claim.attempt} of job ${id}: the job was reclaimed ` +\n \"from this worker while it ran, so that attempt's outcome is no longer the job's.\"\n );\n}\n\nexport function createJobStore(driver: DataDriver): JobStore | undefined {\n const admin = driver.admin;\n if (!isSQLAdmin(admin)) {\n logger.warn(\n \"⚠️ [jobs] DataDriver does not support SQL admin — the durable job queue is unavailable. \" +\n \"Work that would have been queued runs inline instead.\"\n );\n return undefined;\n }\n\n // Two shapes of the same call: the bootstrapper's `SqlExec` takes an\n // options object, while everything below reads better with a positional\n // parameter array.\n const execRaw: SqlExec = (sqlText, options) =>\n admin.executeSql(sqlText, options?.params ? { params: options.params } : undefined);\n const exec = (sqlText: string, params?: unknown[]) =>\n execRaw(sqlText, params ? { params } : undefined);\n\n const ddl = createDdlBootstrapper(execRaw, \"jobs\");\n\n /**\n * Whether {@link ENQUEUE_FUNCTION} exists, so an enqueue inside a write can\n * ride its transaction. Decided at boot rather than assumed: a database\n * that refused the function would otherwise fail every write whose callback\n * enqueues, where falling back to the old behaviour only loses atomicity.\n */\n let enqueueInTransaction = false;\n\n return {\n async ensureTable(): Promise<void> {\n await ddl.ensureObject(\"Creating schema rebase\", \"CREATE SCHEMA IF NOT EXISTS rebase\");\n\n await ddl.ensureObject(`Creating ${TABLE}`, `\n CREATE TABLE IF NOT EXISTS ${TABLE} (\n id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,\n task TEXT NOT NULL,\n payload JSONB,\n status TEXT NOT NULL DEFAULT 'pending',\n run_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n attempts INTEGER NOT NULL DEFAULT 0,\n max_attempts INTEGER NOT NULL DEFAULT 3,\n locked_at TIMESTAMPTZ,\n locked_by TEXT,\n idempotency_key TEXT,\n last_error TEXT,\n created_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n updated_at TIMESTAMPTZ NOT NULL DEFAULT now()\n )\n `);\n\n // The claim query's index. Partial, because the rows it has to find\n // fast are a shrinking minority of a table that also holds every\n // success and every dead letter.\n await ddl.ensureObject(\"Creating idx_jobs_runnable\", `\n CREATE INDEX IF NOT EXISTS idx_jobs_runnable\n ON ${TABLE}(run_at, created_at)\n WHERE status = 'pending'\n `);\n\n // Finds jobs stranded by a dead worker.\n await ddl.ensureObject(\"Creating idx_jobs_running\", `\n CREATE INDEX IF NOT EXISTS idx_jobs_running\n ON ${TABLE}(locked_at)\n WHERE status = 'running'\n `);\n\n // What makes `idempotencyKey` a guarantee rather than a\n // check-then-act race: two instances reacting to one event both\n // reach the INSERT, and the index decides. Partial on unfinished\n // work — see `EnqueueOptions.idempotencyKey` for why a key must be\n // reusable once its job is done.\n await ddl.ensureObject(\"Creating idx_jobs_idempotency\", `\n CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_idempotency\n ON ${TABLE}(idempotency_key)\n WHERE idempotency_key IS NOT NULL AND status IN ('pending', 'running')\n `);\n\n const ready = await ddl.isReadable(TABLE);\n\n if (ready) {\n await ddl.step(\"Job retention sweep\", async () => {\n await exec(\n `DELETE FROM ${TABLE}\n WHERE (status = 'succeeded' AND updated_at < now() - make_interval(days => $1))\n OR (status = 'failed' AND updated_at < now() - make_interval(days => $2))`,\n [SUCCEEDED_RETENTION_DAYS, FAILED_RETENTION_DAYS]\n );\n });\n\n // Payloads are arbitrary application data — a webhook body, a\n // user id, whatever was passed — and a writable queue lets any\n // signed-in user schedule work of their choosing under the\n // server's own authority. Same reasoning, and the same\n // unconditional re-application, as `cron_claims`.\n await ddl.step(\"Revoking end-user access to jobs\", () =>\n exec(revokeInternalTableSql(\"rebase\", \"jobs\")));\n\n await ddl.ensureObject(`Creating ${ENQUEUE_FUNCTION}`, `\n CREATE OR REPLACE FUNCTION ${ENQUEUE_FUNCTION}(\n p_task text, p_payload jsonb, p_run_at timestamptz,\n p_max_attempts integer, p_idempotency_key text\n ) RETURNS text\n LANGUAGE sql\n SECURITY DEFINER\n SET search_path = pg_catalog, pg_temp\n AS $fn$\n INSERT INTO ${TABLE} (task, payload, run_at, max_attempts, idempotency_key)\n VALUES (p_task, p_payload, p_run_at, p_max_attempts, p_idempotency_key)\n ON CONFLICT DO NOTHING\n RETURNING id\n $fn$\n `);\n // Re-applied every boot: a new function is executable by PUBLIC\n // until told otherwise, and the request role may be provisioned\n // after the function was first made.\n await ddl.step(`Granting ${ENQUEUE_FUNCTION} to the request role`, async () => {\n await exec(`REVOKE ALL ON FUNCTION ${ENQUEUE_SIGNATURE} FROM PUBLIC`);\n await exec(`\n DO $grant$ BEGIN\n IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${REBASE_USER_ROLE}') THEN\n GRANT EXECUTE ON FUNCTION ${ENQUEUE_SIGNATURE} TO ${REBASE_USER_ROLE};\n END IF;\n END $grant$\n `);\n });\n enqueueInTransaction = firstSqlRow<{ present: boolean }>(await exec(\n `SELECT to_regprocedure('${ENQUEUE_SIGNATURE}') IS NOT NULL AS present`\n ))?.present === true;\n if (!enqueueInTransaction) {\n logger.warn(\n `⚠️ [jobs] ${ENQUEUE_FUNCTION} could not be created, so a job enqueued inside a write ` +\n \"commits on its own: it stays queued if the write rolls back.\"\n );\n }\n\n logger.info(\"✅ Job queue table ready\");\n } else {\n logger.error(\n `❌ [jobs] ${TABLE} is unavailable — nothing can be queued and nothing queued earlier ` +\n \"will run. Callers fall back to running the work inline.\"\n );\n }\n },\n\n async insert(job): Promise<string | null> {\n const params = [\n job.task,\n JSON.stringify(job.payload ?? null),\n job.runAt.toISOString(),\n job.maxAttempts,\n job.idempotencyKey ?? null\n ];\n // Inside a write: on its transaction, through the definer function\n // — see `ENQUEUE_FUNCTION`. Checked and used with no `await`\n // between, so the transaction cannot close in the gap.\n const ambient = enqueueInTransaction ? currentAmbientTransaction() : undefined;\n if (ambient) {\n const rows = await ambient.exec(\n `SELECT ${ENQUEUE_FUNCTION}($1::text, $2::jsonb, $3::timestamptz, $4::integer, $5::text) AS id`,\n params\n );\n return (rows?.[0]?.id as string | null | undefined) ?? null;\n }\n try {\n const rows = await exec(\n `INSERT INTO ${TABLE} (task, payload, run_at, max_attempts, idempotency_key)\n VALUES ($1, $2::jsonb, $3, $4, $5)\n RETURNING id`,\n params\n );\n return (rows?.[0]?.id as string) ?? null;\n } catch (error) {\n // The losing side of an idempotency race. Not an error: the\n // work the caller wanted is already queued, which is the\n // outcome they asked for.\n if (isUniqueViolation(error)) return null;\n throw error;\n }\n },\n\n async claim(limit: number, workerId: string, tasks?: readonly string[]): Promise<JobRecord[]> {\n if (tasks && tasks.length === 0) return [];\n // One placeholder per task rather than an array parameter: every\n // driver binds a string, and not every driver binds an array.\n const taskFilter = tasks\n ? `AND task IN (${tasks.map((_, i) => `$${i + 3}`).join(\", \")})`\n : \"\";\n // `FOR UPDATE SKIP LOCKED` is the whole design. The inner select\n // takes row locks on the jobs it picks and *skips* any a concurrent\n // worker already holds, so N workers polling the same table divide\n // the work instead of contending for the head of it — and no job is\n // ever handed to two of them.\n //\n // `attempts` is incremented here, on claim, rather than on failure.\n // A worker that is killed mid-job never reports anything, so\n // counting on failure would let a job that crashes the process be\n // retried forever, once per restart, taking the process down each\n // time.\n const rows = await exec(\n `UPDATE ${TABLE} SET\n status = 'running',\n attempts = attempts + 1,\n locked_at = now(),\n locked_by = $2,\n updated_at = now()\n WHERE id IN (\n SELECT id FROM ${TABLE}\n WHERE status = 'pending' AND run_at <= now() ${taskFilter}\n ORDER BY run_at, created_at\n LIMIT $1\n FOR UPDATE SKIP LOCKED\n )\n RETURNING *`,\n [limit, workerId, ...(tasks ?? [])]\n );\n return sqlRows<JobRow>(rows).map(toRecord);\n },\n\n async complete(id: string, claim: JobClaim): Promise<void> {\n const rows = await exec(\n `UPDATE ${TABLE} SET status = 'succeeded', locked_at = NULL, locked_by = NULL,\n last_error = NULL, updated_at = now()\n WHERE ${HELD_BY_CLAIM}\n RETURNING id`,\n [id, claim.workerId, claim.attempt]\n );\n if (sqlRows(rows).length === 0) warnStaleOutcome(id, claim, \"success\");\n },\n\n async fail(id: string, error: string, retryAt: Date | null, claim: JobClaim): Promise<void> {\n const rows = retryAt\n ? await exec(\n `UPDATE ${TABLE} SET status = 'pending', run_at = $4, locked_at = NULL,\n locked_by = NULL, last_error = $5, updated_at = now()\n WHERE ${HELD_BY_CLAIM}\n RETURNING id`,\n [id, claim.workerId, claim.attempt, retryAt.toISOString(), error]\n )\n : await exec(\n `UPDATE ${TABLE} SET status = 'failed', locked_at = NULL, locked_by = NULL,\n last_error = $4, updated_at = now()\n WHERE ${HELD_BY_CLAIM}\n RETURNING id`,\n [id, claim.workerId, claim.attempt, error]\n );\n if (sqlRows(rows).length === 0) warnStaleOutcome(id, claim, \"failure\");\n },\n\n async heartbeat(id: string, claim: JobClaim): Promise<boolean> {\n const rows = await exec(\n `UPDATE ${TABLE} SET locked_at = now()\n WHERE ${HELD_BY_CLAIM}\n RETURNING id`,\n [id, claim.workerId, claim.attempt]\n );\n return sqlRows(rows).length > 0;\n },\n\n async reapExpired(visibilityTimeoutMs: number): Promise<number> {\n // A worker killed while holding a job cannot release it, so nothing\n // but a timeout will ever free the row. Jobs that still have\n // attempts left go back to `pending`; the rest are dead-lettered\n // with an error that says what happened, because \"attempts: 3,\n // lastError: null\" is otherwise a genuinely baffling row to find.\n const seconds = Math.max(1, Math.round(visibilityTimeoutMs / 1000));\n\n const revived = await exec(\n `UPDATE ${TABLE} SET status = 'pending', locked_at = NULL, locked_by = NULL,\n last_error = 'Worker stopped responding; the job was reclaimed', updated_at = now()\n WHERE status = 'running'\n AND locked_at < now() - make_interval(secs => $1)\n AND attempts < max_attempts\n RETURNING id`,\n [seconds]\n );\n\n const buried = await exec(\n `UPDATE ${TABLE} SET status = 'failed', locked_at = NULL, locked_by = NULL,\n last_error = 'Worker stopped responding on the final attempt', updated_at = now()\n WHERE status = 'running'\n AND locked_at < now() - make_interval(secs => $1)\n AND attempts >= max_attempts\n RETURNING id`,\n [seconds]\n );\n\n const count = (revived?.length ?? 0) + (buried?.length ?? 0);\n if (count > 0) {\n logger.warn(\n `[jobs] Reclaimed ${count} job(s) from a worker that stopped responding ` +\n `(${revived?.length ?? 0} retryable, ${buried?.length ?? 0} dead-lettered)`\n );\n }\n return count;\n },\n\n async fetch(id: string): Promise<JobRecord | null> {\n const rows = await exec(`SELECT * FROM ${TABLE} WHERE id = $1`, [id]);\n const row = firstSqlRow<JobRow>(rows);\n return row ? toRecord(row) : null;\n }\n };\n}\n","export { createJobStore } from \"./job-store\";\nexport type { JobClaim, JobStore } from \"./job-store\";\nexport { createJobQueue, defaultBackoff, PermanentJobError } from \"./job-queue\";\nexport type { JobQueue } from \"./job-queue\";\nexport type {\n EnqueueOptions,\n JobContext,\n JobHandler,\n JobQueueClient,\n JobQueueOptions,\n JobRecord,\n JobStatus\n} from \"./types\";\n"],"mappings":";;;;;;;;;;;;;;AA0CA,IAAM,gBAAgB,OAAO,IAAI,0CAA0C;;AAc3E,SAAgB,4BAA4D;CACxE,OAAQ,WAA4B,cAAc,GAAG;AACzD;;;;;;;;;;AC7CA,IAAM,sBAAsB;AAC5B,IAAM,2BAA2B;AACjC,IAAM,gCAAgC,IAAI;AAC1C,IAAM,uBAAuB;;AAG7B,SAAgB,eAAe,SAAyB;CACpD,OAAO,KAAK,IAAI,MAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC,GAAG,KAAK,GAAM;AAC9E;;;;;;;;AASA,IAAM,uBAAuB;;;;;;;;AAS7B,IAAa,oBAAb,cAAuC,MAAM;CACzC,YAAY,SAAiB,SAA+B;EACxD,MAAM,SAAS,OAAO;EACtB,KAAK,OAAO;CAChB;AACJ;AAqBA,SAAgB,eAAe,OAAiB,UAA2B,CAAC,GAAa;CACrF,MAAM,cAAc,QAAQ,eAAe;CAC3C,MAAM,iBAAiB,QAAQ,kBAAkB;CACjD,MAAM,sBAAsB,QAAQ,uBAAuB;CAC3D,MAAM,qBAAqB,QAAQ,eAAe;CAClD,MAAM,UAAU,QAAQ,WAAW;CAEnC,MAAM,2BAAW,IAAI,IAA+B;CACpD,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,SAAS,CAAC,CAAC,GAC5D,SAAS,IAAI,MAAM,OAAO;CAM9B,MAAM,WAAW,GAAG,QAAQ,IAAI,GAAG,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC;CAE1D,IAAI,QAA+B;CACnC,IAAI,YAAmC;CACvC,IAAI,UAAU;;;;;;;CAOd,MAAM,yBAAS,IAAI,IAAmB;;CAEtC,IAAI,WAAoC;;CAExC,IAAI,UAAU;CACd,IAAI,UAAU;CACd,IAAI,WAAiC;;;;;;;CAQrC,SAAS,UAAU,KAAgB,OAA6B;EAC5D,IAAI,CAAC,MAAM,WAAW,aAAa,KAAA;EACnC,MAAM,QAAQ,MAAM,UAAU,KAAK,KAAK;EACxC,MAAM,WAAW,kBAAkB;GAC/B,MAAM,IAAI,IAAI,KAAK,CAAC,CAAC,MAAM,SAAS;IAChC,IAAI,MAAM;IACV,cAAc,QAAQ;IACtB,OAAO,KACH,WAAW,IAAI,KAAK,8BAA8B,MAAM,QAAQ,mDAC5B,EAAE,OAAO,IAAI,GAAG,CACxD;GACJ,CAAC,CAAC,CAAC,OAAO,UAAmB;IACzB,OAAO,KAAK,wCAAwC,IAAI,KAAK,IAAI;KAAE,OAAO,IAAI;KAAI;IAAM,CAAC;GAC7F,CAAC;EACL,GAAG,KAAK,IAAI,GAAG,KAAK,MAAM,sBAAsB,CAAC,CAAC,CAAC;EACnD,SAAS,QAAQ;EACjB,aAAa,cAAc,QAAQ;CACvC;CAEA,eAAe,OAAO,KAA+B;EACjD,MAAM,UAAU,SAAS,IAAI,IAAI,IAAI;EACrC,MAAM,QAAkB;GAAE;GAAU,SAAS,IAAI;EAAS;EAE1D,IAAI,CAAC,SAAS;GAOV,OAAO,KAAK,0CAA0C,IAAI,KAAK,mCAAmC;GAClG,MAAM,MAAM,KACR,IAAI,IACJ,mCAAmC,IAAI,KAAK,IAC5C,IAAI,WAAW,IAAI,cAAc,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,MAChF,KACJ;GACA;EACJ;EAEA,MAAM,eAAe,UAAU,KAAK,KAAK;EACzC,IAAI;GACA,IAAI;IACA,MAAM,QAAQ;KACV,IAAI,IAAI;KACR,MAAM,IAAI;KACV,SAAS,IAAI;KACb,SAAS,IAAI;KACb,aAAa,IAAI;IACrB,CAAU;GACd,UAAU;IACN,aAAa;GACjB;GACA,MAAM,MAAM,SAAS,IAAI,IAAI,KAAK;EACtC,SAAS,OAAO;GACZ,MAAM,UAAU,iBAAiB,QAAS,MAAM,SAAS,MAAM,UAAW,OAAO,KAAK;GACtF,MAAM,YAAY,iBAAiB;GACnC,MAAM,YAAY,CAAC,aAAa,IAAI,WAAW,IAAI;GAInD,MAAM,MAAM,KAAK,IAAI,IAAI,QAAQ,MAAM,GAAG,GAAK,GAAG,YAAY,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,MAAM,KAAK;GAExH,IAAI,WACA,OAAO,KAAK,WAAW,IAAI,KAAK,sBAAsB,IAAI,SAAS,GAAG,IAAI,YAAY,aAAa,EAAE,OAAO,IAAI,GAAG,CAAC;QACjH,IAAI,WACP,OAAO,MAAM,WAAW,IAAI,KAAK,kCAAkC,IAAI,SAAS,iBAAiB;IAAE,OAAO,IAAI;IAAI,OAAO;GAAQ,CAAC;QAKlI,OAAO,MAAM,WAAW,IAAI,KAAK,6BAA6B,IAAI,SAAS,YAAY;IAAE,OAAO,IAAI;IAAI,OAAO;GAAQ,CAAC;EAEhI;CACJ;;;;;;CAOA,eAAe,OAAsB;EACjC,IAAI,SAAS;EACb,UAAU;EACV,IAAI;GACA,MAAM,MAAM,YAAY,mBAAmB;EAC/C,SAAS,OAAO;GACZ,OAAO,MAAM,yCAAyC,EAAE,MAAM,CAAC;EACnE,UAAU;GACN,UAAU;EACd;CACJ;;CAGA,eAAe,gBAA0C;EACrD,MAAM,OAAO,cAAc,OAAO;EAClC,IAAI,QAAQ,GAAG,OAAO,CAAC;EAOvB,IAAI,SAAS,SAAS,GAAG,OAAO,CAAC;EACjC,MAAM,OAAO,MAAM,MAAM,MAAM,MAAM,UAAU,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC;EACnE,UAAU,KAAK,UAAU;EAEzB,OAAO,KAAK,KAAK,QAAQ;GACrB,MAAM,MAAqB,OAAO,GAAG,CAAC,CACjC,OAAO,UAAU;IAId,OAAO,MAAM,2CAA2C,IAAI,KAAK,IAAI;KAAE,OAAO,IAAI;KAAI;IAAM,CAAC;GACjG,CAAC,CAAC,CACD,cAAc;IACX,OAAO,OAAO,GAAG;IAGjB,IAAI,SAAS,SAAS,CAAC;GAC3B,CAAC;GACL,OAAO,IAAI,GAAG;GACd,OAAO;EACX,CAAC;CACL;CAEA,SAAS,SAAS,SAAuB;EACrC,IAAI,CAAC,SAAS;EACd,IAAI,OAAO,aAAa,KAAK;EAC7B,QAAQ,iBAAiB;GACrB,QAAQ;GACR,KAAU;EACd,GAAG,OAAO;EAGV,MAAM,QAAQ;CAClB;CAEA,eAAe,OAAsB;EAEjC,IAAI,CAAC,WAAW,UAAU;EAC1B,MAAM,OAAO,cAAc,CAAC,CAAC,OAAO,UAAU;GAC1C,OAAO,MAAM,sBAAsB,EAAE,MAAM,CAAC;GAC5C,UAAU;EACd,CAAC;EACD,WAAW;EACX,MAAM;EACN,WAAW;EAKX,SAAS,WAAW,OAAO,OAAO,cAAc,IAAI,cAAc;CACtE;CAEA,eAAe,QAAuB;EAClC,MAAM;EACN,MAAM,QAAQ,WAAW,CAAC,GAAG,MAAM,CAAC;CACxC;CAEA,OAAO;EACH,QAAc;GACV,IAAI,SAAS;GACb,UAAU;GACV,OAAO,KAAK,sCAAsC,YAAY,SAAS,eAAe,IAAI;GAC1F,KAAU;GACV,YAAY,kBAAkB;IAAE,KAAU;GAAG,GAAG,sBAAsB,oBAAoB;GAC1F,UAAU,QAAQ;GAClB,SAAS,CAAC;EACd;EAEA,MAAM,KAAK,WAAmC;GAC1C,UAAU;GACV,IAAI,OAAO;IACP,aAAa,KAAK;IAClB,QAAQ;GACZ;GACA,IAAI,WAAW;IACX,cAAc,SAAS;IACvB,YAAY;GAChB;GAIA,aAAa,MAAM,CAAC,CAAC,cAAc;IAAE,WAAW;GAAM,CAAC;GACvD,IAAI,cAAc,KAAA,GAAW;IACzB,MAAM;IACN;GACJ;GACA,IAAI;GACJ,MAAM,WAAW,MAAM,QAAQ,KAAK,CAChC,SAAS,WAAW,IAAI,GACxB,IAAI,SAAgB,YAAY;IAC5B,SAAS,iBAAiB,QAAQ,KAAK,GAAG,SAAS;IACnD,OAAO,QAAQ;GACnB,CAAC,CACL,CAAC;GACD,aAAa,MAAM;GACnB,IAAI,CAAC,UACD,OAAO,KACH,UAAU,OAAO,KAAK,8BAA8B,UAAU,0GAElE;EAER;EAEA,MAAM,UAA2B;GAC7B,MAAM,UAAU,MAAM,cAAc;GACpC,MAAM,QAAQ,WAAW,OAAO;GAChC,OAAO,QAAQ;EACnB;EAEA,SAAsB,MAAc,SAA8B;GAC9D,IAAI,SAAS,IAAI,IAAI,GACjB,OAAO,KAAK,gBAAgB,KAAK,iDAAiD;GAEtF,SAAS,IAAI,MAAM,OAA4B;EACnD;EAEA,YAAqB;GACjB,OAAO;EACX;EAEA,MAAM,QAAqB,MAAc,SAAa,iBAAiC,CAAC,GAA2B;GAC/G,OAAO,MAAM,OAAO;IAChB;IACA,SAAS,WAAW;IACpB,OAAO,IAAI,KAAK,KAAK,IAAI,KAAK,eAAe,WAAW,EAAE;IAC1D,aAAa,eAAe,eAAe;IAC3C,gBAAgB,eAAe;GACnC,CAAC;EACL;CACJ;AACJ;;;;;;;;;;;ACjUA,IAAM,QAAQ;;AAGd,IAAM,2BAA2B;;;;;;AAMjC,IAAM,wBAAwB;AAgB9B,SAAS,SAAS,KAAwB;CACtC,OAAO;EACH,IAAI,IAAI;EACR,MAAM,IAAI;EACV,SAAS,IAAI;EACb,QAAQ,IAAI;EACZ,OAAO,IAAI,KAAK,IAAI,MAAM,CAAC,CAAC,YAAY;EACxC,UAAU,OAAO,IAAI,QAAQ;EAC7B,aAAa,OAAO,IAAI,YAAY;EACpC,WAAW,IAAI;EACf,WAAW,IAAI,KAAK,IAAI,UAAU,CAAC,CAAC,YAAY;EAChD,WAAW,IAAI,KAAK,IAAI,UAAU,CAAC,CAAC,YAAY;CACpD;AACJ;;AAGA,SAAS,kBAAkB,KAAuB;CAC9C,OAAO,gBAAgB,MAAM,MAAM,EAAE,SAAS,OAAO;AACzD;;;;;;;;;;;;;;;;;;;;;;;AAwBA,IAAM,mBAAmB;AACzB,IAAM,oBAAoB,GAAG,iBAAiB;;AAwD9C,IAAM,gBAAgB;AAEtB,SAAS,iBAAiB,IAAY,OAAiB,SAAsC;CACzF,OAAO,KACH,sBAAsB,QAAQ,cAAc,MAAM,QAAQ,UAAU,GAAG,yGAE3E;AACJ;AAEA,SAAgB,eAAe,QAA0C;CACrE,MAAM,QAAQ,OAAO;CACrB,IAAI,CAAC,WAAW,KAAK,GAAG;EACpB,OAAO,KACH,+IAEJ;EACA;CACJ;CAKA,MAAM,WAAoB,SAAS,YAC/B,MAAM,WAAW,SAAS,SAAS,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,KAAA,CAAS;CACtF,MAAM,QAAQ,SAAiB,WAC3B,QAAQ,SAAS,SAAS,EAAE,OAAO,IAAI,KAAA,CAAS;CAEpD,MAAM,MAAM,sBAAsB,SAAS,MAAM;;;;;;;CAQjD,IAAI,uBAAuB;CAE3B,OAAO;EACH,MAAM,cAA6B;GAC/B,MAAM,IAAI,aAAa,0BAA0B,oCAAoC;GAErF,MAAM,IAAI,aAAa,YAAY,SAAS;6CACX,MAAM;;;;;;;;;;;;;;;aAetC;GAKD,MAAM,IAAI,aAAa,8BAA8B;;qBAE5C,MAAM;;aAEd;GAGD,MAAM,IAAI,aAAa,6BAA6B;;qBAE3C,MAAM;;aAEd;GAOD,MAAM,IAAI,aAAa,iCAAiC;;qBAE/C,MAAM;;aAEd;GAID,IAAI,MAFgB,IAAI,WAAW,KAAK,GAE7B;IACP,MAAM,IAAI,KAAK,uBAAuB,YAAY;KAC9C,MAAM,KACF,eAAe,MAAM;;2GAGrB,CAAC,0BAA0B,qBAAqB,CACpD;IACJ,CAAC;IAOD,MAAM,IAAI,KAAK,0CACX,KAAK,uBAAuB,UAAU,MAAM,CAAC,CAAC;IAElD,MAAM,IAAI,aAAa,YAAY,oBAAoB;iDACtB,iBAAiB;;;;;;;;sCAQ5B,MAAM;;;;;iBAK3B;IAID,MAAM,IAAI,KAAK,YAAY,iBAAiB,uBAAuB,YAAY;KAC3E,MAAM,KAAK,0BAA0B,kBAAkB,aAAa;KACpE,MAAM,KAAK;;iFAEkD,iBAAiB;4DACtC,kBAAkB,MAAM,iBAAiB;;;qBAGhF;IACL,CAAC;IACD,uBAAuB,YAAkC,MAAM,KAC3D,2BAA2B,kBAAkB,0BACjD,CAAC,CAAC,EAAE,YAAY;IAChB,IAAI,CAAC,sBACD,OAAO,KACH,aAAa,iBAAiB,qHAElC;IAGJ,OAAO,KAAK,yBAAyB;GACzC,OACI,OAAO,MACH,YAAY,MAAM,2HAEtB;EAER;EAEA,MAAM,OAAO,KAA6B;GACtC,MAAM,SAAS;IACX,IAAI;IACJ,KAAK,UAAU,IAAI,WAAW,IAAI;IAClC,IAAI,MAAM,YAAY;IACtB,IAAI;IACJ,IAAI,kBAAkB;GAC1B;GAIA,MAAM,UAAU,uBAAuB,0BAA0B,IAAI,KAAA;GACrE,IAAI,SAKA,QAAQ,MAJW,QAAQ,KACvB,UAAU,iBAAiB,sEAC3B,MACJ,EAAA,GACe,EAAE,EAAE,MAAoC;GAE3D,IAAI;IAOA,QAAQ,MANW,KACf,eAAe,MAAM;;oCAGrB,MACJ,EAAA,GACe,EAAE,EAAE,MAAiB;GACxC,SAAS,OAAO;IAIZ,IAAI,kBAAkB,KAAK,GAAG,OAAO;IACrC,MAAM;GACV;EACJ;EAEA,MAAM,MAAM,OAAe,UAAkB,OAAiD;GAC1F,IAAI,SAAS,MAAM,WAAW,GAAG,OAAO,CAAC;GAGzC,MAAM,aAAa,QACb,gBAAgB,MAAM,KAAK,GAAG,MAAM,IAAI,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,KAC5D;GA6BN,OAAO,QAAgB,MAjBJ,KACf,UAAU,MAAM;;;;;;;sCAOM,MAAM;oEACwB,WAAW;;;;;+BAM/D;IAAC;IAAO;IAAU,GAAI,SAAS,CAAC;GAAE,CACtC,CAC2B,CAAC,CAAC,IAAI,QAAQ;EAC7C;EAEA,MAAM,SAAS,IAAY,OAAgC;GAQvD,IAAI,QAAQ,MAPO,KACf,UAAU,MAAM;;yBAEP,cAAc;gCAEvB;IAAC;IAAI,MAAM;IAAU,MAAM;GAAO,CACtC,CACgB,CAAC,CAAC,WAAW,GAAG,iBAAiB,IAAI,OAAO,SAAS;EACzE;EAEA,MAAM,KAAK,IAAY,OAAe,SAAsB,OAAgC;GAgBxF,IAAI,QAfS,UACP,MAAM,KACJ,UAAU,MAAM;;6BAEP,cAAc;oCAEvB;IAAC;IAAI,MAAM;IAAU,MAAM;IAAS,QAAQ,YAAY;IAAG;GAAK,CACpE,IACE,MAAM,KACJ,UAAU,MAAM;;6BAEP,cAAc;oCAEvB;IAAC;IAAI,MAAM;IAAU,MAAM;IAAS;GAAK,CAC7C,CACY,CAAC,CAAC,WAAW,GAAG,iBAAiB,IAAI,OAAO,SAAS;EACzE;EAEA,MAAM,UAAU,IAAY,OAAmC;GAO3D,OAAO,QAAQ,MANI,KACf,UAAU,MAAM;yBACP,cAAc;gCAEvB;IAAC;IAAI,MAAM;IAAU,MAAM;GAAO,CACtC,CACmB,CAAC,CAAC,SAAS;EAClC;EAEA,MAAM,YAAY,qBAA8C;GAM5D,MAAM,UAAU,KAAK,IAAI,GAAG,KAAK,MAAM,sBAAsB,GAAI,CAAC;GAElE,MAAM,UAAU,MAAM,KAClB,UAAU,MAAM;;;;;gCAMhB,CAAC,OAAO,CACZ;GAEA,MAAM,SAAS,MAAM,KACjB,UAAU,MAAM;;;;;gCAMhB,CAAC,OAAO,CACZ;GAEA,MAAM,SAAS,SAAS,UAAU,MAAM,QAAQ,UAAU;GAC1D,IAAI,QAAQ,GACR,OAAO,KACH,oBAAoB,MAAM,iDACtB,SAAS,UAAU,EAAE,cAAc,QAAQ,UAAU,EAAE,gBAC/D;GAEJ,OAAO;EACX;EAEA,MAAM,MAAM,IAAuC;GAE/C,MAAM,MAAM,YAAoB,MADb,KAAK,iBAAiB,MAAM,iBAAiB,CAAC,EAAE,CAAC,CAChC;GACpC,OAAO,MAAM,SAAS,GAAG,IAAI;EACjC;CACJ;AACJ"}