@withbuddi/buddi 0.1.0-pre.36 → 0.1.0-pre.37

package/packages/core/dist/queue/jobs.d.ts

@@ -9,8 +9,10 @@
  *    `lease_owner = $worker and state = 'leased'`. That is the fence: a worker
  *    whose lease expired and was reclaimed elsewhere updates nothing and is
  *    told so (`false` / `null`), instead of writing over the new owner's run.
- *  - `releaseStaleLeases` is startup recovery *and* the running sweep — a lease
- *    that outlived its process goes back to `pending` with its attempt spent.
+ *  - `interruptLeases` is startup recovery, the running sweep and the
+ *    shutdown release — a lease that outlived its run is settled: requeued
+ *    once when the run had done nothing yet, failed with the reason otherwise.
+ *    (`releaseStaleLeases` is the older, unconditional put-back.)
  */
 import type { Pool } from 'pg';
 import { type FailureClass } from './retry-policy.js';
@@ -114,6 +116,59 @@ export declare function resumeJob(pool: Pool, jobId: string, input?: ResumeInput
  * process that dies mid-job cannot loop forever on the installation's behalf.
  */
 export declare function releaseStaleLeases(pool: Pool, now: Date): Promise<number>;
+/** Why a lease is being taken back from a run that never finished. */
+export declare const INTERRUPTED_REASON = "interrupted by a restart";
+export type InterruptScope = 
+/** Every lease whose expiry passed: the holder stopped heartbeating. */
+{
+    expiredBy: Date;
+}
+/**
+ * Start-up recovery: every lease that expired, and every lease held by a
+ * previous boot of this worker family (`holderPrefix`), whatever its expiry —
+ * a restart takes seconds, the lease lasts minutes, and the process that
+ * held it is gone either way.
+ */
+ | {
+    expiredBy: Date;
+    self: string;
+    holderPrefix: string;
+}
+/** Graceful shutdown: the leases this worker holds. */
+ | {
+    heldBy: string;
+};
+export type InterruptedJob = Job & {
+    /** True when the job went back on the queue; false when it failed for the owner. */
+    requeued: boolean;
+    /** Non-auto tools this attempt had already called (its side effects). */
+    acted: string[];
+};
+/**
+ * Take back the leases of runs that will never report, and settle each one.
+ *
+ * The retry policy is the run's own record: an attempt that had not yet
+ * called a tool with an effect (any tier but `auto`, or an `auto` tool marked
+ * `sideEffect`, read from the `tool.called` events carrying this job's id
+ * since its latest claim) and whose report was not delivered (a
+ * `mission.delivered` event for its occurrence or conversation) goes
+ * back on the queue — once. One that had acted, or that was already requeued
+ * after an interruption, or that has no attempts left, fails with the reason in
+ * `last_error`, where Activity → Jobs shows it; a mission occurrence it was
+ * running is closed as failed with the same reason. Nothing is re-run that
+ * might send, buy or write a second time without the owner deciding.
+ *
+ * Every write is fenced on the holder the candidate query saw, so a lease
+ * reclaimed in between is left alone. The attempt stays spent.
+ */
+export declare function interruptLeases(pool: Pool, scope: InterruptScope, detail?: string, opts?: {
+    /**
+     * False when the attempt may still be running in this process (its
+     * handler ignored the stop): putting it back would let the next start run
+     * it beside the one still going, so it fails instead.
+     */
+    requeue?: boolean;
+}): Promise<InterruptedJob[]>;
 /**
  * Stop a job for good. Anything not already finished can be cancelled, and so
  * can a failed one: "failed" is the queue waiting for a human, and giving up

package/packages/core/dist/queue/jobs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jobs.d.ts","sourceRoot":"","sources":["../../src/queue/jobs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAG/B,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAOL,KAAK,GAAG,EAER,KAAK,QAAQ,EACd,MAAM,YAAY,CAAC;AAEpB,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,IAAI,CAAC;IAChB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,kEAAkE;AAClE,wBAAsB,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAiC3E;AAED,MAAM,MAAM,UAAU,GAAG;IACvB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,GAAG,EAAE,IAAI,CAAC;IACV,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAkCjF;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,OAAO,CAAC,CASlB;AAED,yDAAyD;AACzD,wBAAsB,WAAW,CAC/B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAsBrB;AAED,MAAM,MAAM,SAAS,GAAG;IACtB,qEAAqE;IACrE,KAAK,EAAE,OAAO,CAAC;IACf,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,cAAc,CAAC,EAAE;QAAE,KAAK,EAAE,YAAY,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,SAA2B,GACjC,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CA0CrB;AAED;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC,CAAC;AAEF,wBAAsB,UAAU,CAC9B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAuBrB;AAED,MAAM,MAAM,WAAW,GAAG;IACxB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,sEAAsE;IACtE,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AAEF,6CAA6C;AAC7C,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,WAAgB,GACtB,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAwBrB;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAqB/E;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAoB9E;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CA2B7E;AAED,wBAAsB,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAM3E;AAED,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAMxF;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9B,0FAA0F;IAC1F,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AAEF,kDAAkD;AAClD,wBAAsB,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,GAAE,aAAkB,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAcpF;AAED,MAAM,WAAW,iBAAiB;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,uDAAuD;IACvD,KAAK,EAAE,IAAI,CAAC;IACZ;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAuBvF;AAED;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,KAAK,GAAE;IACL,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,GAAG,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxB,qFAAqF;IACrF,MAAM,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;CAC1B,GACL,OAAO,CAAC,GAAG,EAAE,CAAC,CA6ChB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAiBrE;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE;IAAE,GAAG,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IAAC,EAAE,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D,OAAO,CAAC,GAAG,EAAE,CAAC,CAehB;AAED,sEAAsE;AACtE,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAYtF"}
+{"version":3,"file":"jobs.d.ts","sourceRoot":"","sources":["../../src/queue/jobs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAG/B,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAOL,KAAK,GAAG,EAER,KAAK,QAAQ,EACd,MAAM,YAAY,CAAC;AAEpB,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,IAAI,CAAC;IAChB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,kEAAkE;AAClE,wBAAsB,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAiC3E;AAED,MAAM,MAAM,UAAU,GAAG;IACvB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,GAAG,EAAE,IAAI,CAAC;IACV,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAkCjF;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,OAAO,CAAC,CASlB;AAED,yDAAyD;AACzD,wBAAsB,WAAW,CAC/B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAsBrB;AAED,MAAM,MAAM,SAAS,GAAG;IACtB,qEAAqE;IACrE,KAAK,EAAE,OAAO,CAAC;IACf,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,cAAc,CAAC,EAAE;QAAE,KAAK,EAAE,YAAY,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,SAA2B,GACjC,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CA0CrB;AAED;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC,CAAC;AAEF,wBAAsB,UAAU,CAC9B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAuBrB;AAED,MAAM,MAAM,WAAW,GAAG;IACxB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,sEAAsE;IACtE,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AAEF,6CAA6C;AAC7C,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,WAAgB,GACtB,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAwBrB;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAqB/E;AAED,sEAAsE;AACtE,eAAO,MAAM,kBAAkB,6BAA6B,CAAC;AAE7D,MAAM,MAAM,cAAc;AACxB,wEAAwE;AACtE;IAAE,SAAS,EAAE,IAAI,CAAA;CAAE;AACrB;;;;;GAKG;GACD;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE;AACzD,uDAAuD;GACrD;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvB,MAAM,MAAM,cAAc,GAAG,GAAG,GAAG;IACjC,oFAAoF;IACpF,QAAQ,EAAE,OAAO,CAAC;IAClB,yEAAyE;IACzE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,cAAc,EACrB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,GAAE;IACJ;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,GACL,OAAO,CAAC,cAAc,EAAE,CAAC,CAkH3B;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAoB9E;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CA2B7E;AAED,wBAAsB,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAM3E;AAED,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAMxF;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9B,0FAA0F;IAC1F,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AAEF,kDAAkD;AAClD,wBAAsB,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,GAAE,aAAkB,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAcpF;AAED,MAAM,WAAW,iBAAiB;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,uDAAuD;IACvD,KAAK,EAAE,IAAI,CAAC;IACZ;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAuBvF;AAED;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,KAAK,GAAE;IACL,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,GAAG,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxB,qFAAqF;IACrF,MAAM,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;CAC1B,GACL,OAAO,CAAC,GAAG,EAAE,CAAC,CA6ChB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAiBrE;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,IAAI,EACV,KAAK,EAAE;IAAE,GAAG,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IAAC,EAAE,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D,OAAO,CAAC,GAAG,EAAE,CAAC,CAehB;AAED,sEAAsE;AACtE,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAYtF"}

package/packages/core/dist/queue/jobs.js

@@ -193,6 +193,120 @@ export async function releaseStaleLeases(pool, now) {
     }
     return rows.length;
 }
+/** Why a lease is being taken back from a run that never finished. */
+export const INTERRUPTED_REASON = 'interrupted by a restart';
+/**
+ * Take back the leases of runs that will never report, and settle each one.
+ *
+ * The retry policy is the run's own record: an attempt that had not yet
+ * called a tool with an effect (any tier but `auto`, or an `auto` tool marked
+ * `sideEffect`, read from the `tool.called` events carrying this job's id
+ * since its latest claim) and whose report was not delivered (a
+ * `mission.delivered` event for its occurrence or conversation) goes
+ * back on the queue — once. One that had acted, or that was already requeued
+ * after an interruption, or that has no attempts left, fails with the reason in
+ * `last_error`, where Activity → Jobs shows it; a mission occurrence it was
+ * running is closed as failed with the same reason. Nothing is re-run that
+ * might send, buy or write a second time without the owner deciding.
+ *
+ * Every write is fenced on the holder the candidate query saw, so a lease
+ * reclaimed in between is left alone. The attempt stays spent.
+ */
+export async function interruptLeases(pool, scope, detail, opts = {}) {
+    const params = [];
+    let where;
+    if ('heldBy' in scope) {
+        params.push(scope.heldBy);
+        where = 'j.lease_owner = $1';
+    }
+    else if ('self' in scope) {
+        params.push(scope.expiredBy.toISOString(), scope.self, `${scope.holderPrefix}%`);
+        where = `j.lease_owner is distinct from $2 and (j.lease_until < $1::timestamptz or j.lease_owner like $3)`;
+    }
+    else {
+        params.push(scope.expiredBy.toISOString());
+        where = 'j.lease_until < $1::timestamptz';
+    }
+    const { rows: candidates } = await pool.query(`select j.id, j.lease_owner, j.attempts, j.max_attempts,
+       coalesce((select array_agg(distinct a.name order by a.name) from (
+          select e.payload->>'name' as name
+            from core.events e
+           where e.kind = 'tool.called'
+             and e.payload->>'jobId' = j.id::text
+             and (coalesce(e.payload->>'tier', 'gated') <> 'auto' or e.payload->>'sideEffect' = 'true')
+             and e.created_at >= coalesce((select max(c.created_at) from core.events c
+                  where c.kind = 'job.claimed' and c.payload->>'jobId' = j.id::text), j.created_at)
+          union all
+          -- An earlier version recorded calls without the job or the tier, so
+          -- one of those since this claim may be this run's: count it as acted.
+          select 'a tool call recorded by an earlier version'
+            from core.events e
+           where e.kind = 'tool.called'
+             and not (e.payload ? 'tier')
+             and (j.conversation_id is null or e.conversation_id is null or e.conversation_id = j.conversation_id)
+             and e.created_at >= coalesce((select max(c.created_at) from core.events c
+                  where c.kind = 'job.claimed' and c.payload->>'jobId' = j.id::text), j.created_at)
+          union all
+          select 'its report was delivered'
+            from core.events e
+           where e.kind = 'mission.delivered'
+             and ((j.payload->>'occurrenceId' is not null and e.payload->>'occurrenceId' = j.payload->>'occurrenceId')
+                  or (j.conversation_id is not null and e.conversation_id = j.conversation_id))
+             and e.created_at >= coalesce((select max(c.created_at) from core.events c
+                  where c.kind = 'job.claimed' and c.payload->>'jobId' = j.id::text), j.created_at)
+       ) a), '{}'::text[]) as acted,
+       (select count(*)::int from core.events e
+         where e.kind = 'job.interrupted' and e.payload->>'jobId' = j.id::text
+           and e.payload->>'requeued' = 'true') as requeues
+     from core.jobs j
+     where j.state = 'leased' and ${where}`, params);
+    const settled = [];
+    const why = detail ? `${INTERRUPTED_REASON} (${detail})` : INTERRUPTED_REASON;
+    for (const c of candidates) {
+        const acted = c.acted.filter((name) => typeof name === 'string' && name !== '');
+        const requeue = opts.requeue !== false && acted.length === 0 && c.requeues === 0 && c.attempts < c.max_attempts;
+        const message = requeue
+            ? `${why}; queued again`
+            : opts.requeue === false && acted.length === 0
+                ? `${why} while it was still running; not run again on its own, so it never runs twice at once. Retry it if it should run.`
+                : acted.length > 0
+                    ? `${why} after it had acted (${acted.join(', ')}); not run again on its own, so nothing happens twice. Retry it if it should run.`
+                    : c.requeues > 0
+                        ? `${why} a second time; not run again on its own. Retry it if it should run.`
+                        : `${why} on its last attempt.`;
+        const { rows } = await pool.query(`update core.jobs
+       set state = case when $3 then 'pending' else 'failed' end,
+           run_after = case when $3 then now() else run_after end,
+           last_error = $4,
+           lease_owner = null,
+           lease_until = null,
+           acknowledged_at = null,
+           acknowledged_by = null,
+           updated_at = now()
+       where id = $1::uuid and state = 'leased' and lease_owner is not distinct from $2
+       returning ${JOB_COLUMNS}`, [c.id, c.lease_owner, requeue, message]);
+        if (rows.length === 0)
+            continue;
+        const job = toJob(rows[0]);
+        await appendEvent(pool, 'job.interrupted', { jobId: job.id, kind: job.kind, holder: c.lease_owner, attempts: job.attempts, requeued: requeue, acted, reason: why }, job.conversationId ?? undefined);
+        if (!requeue) {
+            // A mission's occurrence is the run the owner sees on Missions; it closes
+            // with the job rather than staying claimed for a run that will not come.
+            const occurrenceId = job.payload?.occurrenceId;
+            if (typeof occurrenceId === 'string') {
+                await pool.query(`update core.occurrences set state = 'failed', finished_at = now(), error = $2
+            where id::text = $1 and state = 'claimed'`, [occurrenceId, message]);
+            }
+            await appendEvent(pool, 'job.failed', {
+                jobId: job.id, kind: job.kind, worker: c.lease_owner, attempts: job.attempts,
+                maxAttempts: job.maxAttempts, error: message, retrying: false,
+                failureClass: 'interrupted', failureReason: why,
+            }, job.conversationId ?? undefined);
+        }
+        settled.push({ ...job, requeued: requeue, acted });
+    }
+    return settled;
+}
 /**
  * Stop a job for good. Anything not already finished can be cancelled, and so
  * can a failed one: "failed" is the queue waiting for a human, and giving up

package/packages/core/dist/queue/jobs.js.map

@@ -1 +1 @@
-{"version":3,"file":"jobs.js","sourceRoot":"","sources":["../../src/queue/jobs.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EACL,eAAe,EACf,oBAAoB,EACpB,wBAAwB,GAEzB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,uBAAuB,EACvB,WAAW,EACX,yBAAyB,EACzB,aAAa,EACb,YAAY,EACZ,KAAK,GAIN,MAAM,YAAY,CAAC;AAsBpB,kEAAkE;AAClE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAU,EAAE,KAAmB;IAC3D,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC1F,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;iBAGa,WAAW,EAAE,EAC1B;QACE,KAAK,CAAC,IAAI;QACV,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC;QAClE,KAAK,CAAC,QAAQ,IAAI,CAAC;QACnB,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,IAAI;QACrC,KAAK,CAAC,WAAW,IAAI,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW;QAC5D,KAAK,CAAC,QAAQ,IAAI,IAAI;QACtB,KAAK,CAAC,cAAc,IAAI,IAAI;KAC7B,CACF,CAAC;IAEF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;QACrC,MAAM,WAAW,CACf,IAAI,EACJ,cAAc,EACd,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,EACjF,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;IAED,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,QAAkB,CAAC,CAAC;IACxE,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,QAAQ,cAAc,CAAC,CAAC;IAChG,OAAO,QAAQ,CAAC;AAClB,CAAC;AAWD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAU,EAAE,KAAiB;IAC1D,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1E,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9E,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;;;;mBAWe,UAAU;;;;;iBAKZ,WAAW,EAAE,EAC1B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAClC,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,aAAa,EACb,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,EAC9E,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,KAAa,EACb,MAAc,EACd,OAAe;IAEf,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CACnC;;;mEAG+D,EAC/D,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,CAChC,CAAC;IACF,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,yDAAyD;AACzD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAU,EACV,KAAa,EACb,MAAc,EACd,MAAgB;IAEhB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;iBAQa,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CACtE,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,eAAe,EACf,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,EACjE,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAeD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAU,EACV,KAAa,EACb,MAAc,EACd,KAAa,EACb,QAAmB,EAAE,KAAK,EAAE,IAAI,EAAE;IAElC,0EAA0E;IAC1E,wDAAwD;IACxD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;kBAKc,aAAa,GAAG,IAAI,MAAM,YAAY;;;;;;;;iBAQvC,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CACnG,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC;IACzC,MAAM,WAAW,CACf,IAAI,EACJ,YAAY,EACZ;QACE,KAAK,EAAE,GAAG,CAAC,EAAE;QACb,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM;QACN,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,KAAK;QACL,QAAQ;QACR,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,KAAK,CAAC,cAAc;YACtB,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE;YAC1F,CAAC,CAAC,EAAE,CAAC;KACR,EACD,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAiBD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAU,EACV,KAAa,EACb,MAAc,EACd,MAAc,EACd,OAAqB,EAAE;IAEvB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;IAChC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;iBAQa,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAC5E,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,eAAe,EACf,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,EACjD,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAaD,6CAA6C;AAC7C,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,KAAa,EACb,QAAqB,EAAE;IAEvB,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC;IACjC,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC;IAC7C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;iBAQa,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CACpE,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,aAAa,EACb,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EACpF,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAU,EAAE,GAAS;IAC5D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;iBAOa,WAAW,EAAE,EAC1B,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CACpB,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,WAAW,CAAC,IAAI,EAAE,mBAAmB,EAAE;YAC3C,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;SACvB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAU,EAAE,KAAa;IACvD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;iBAMa,WAAW,EAAE,EAC1B,CAAC,KAAK,CAAC,CACR,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,eAAe,EACf,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,EACjC,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAU,EAAE,KAAa;IACtD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;;;;;;iBAaa,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,CAAC,GAAG,oBAAoB,CAAC,EAAE,wBAAwB,CAAC,WAAW,CAAC,CACzE,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,cAAc,EACd,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EACjD,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAU,EAAE,KAAa;IACpD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,UAAU,WAAW,qCAAqC,EAC1D,CAAC,KAAK,CAAC,CACR,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAU,EAAE,QAAgB;IACjE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,UAAU,WAAW,sCAAsC,EAC3D,CAAC,QAAQ,CAAC,CACX,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3D,CAAC;AAkBD,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAU,EAAE,QAAuB,EAAE;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,yBAAyB;QAChE,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,uBAAuB;YACxD,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,uBAAuB,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,UAAU,WAAW;;;aAGZ,MAAM;;wBAEK,EACpB,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CACzH,CAAC;IACF,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC;AAmBD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,KAAwB;IACrE,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,oBAAoB,CAAC,CAAC,CAAC;IACzD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,UAAU,WAAW;;;;;;;;;;cAUX,EACV;QACE,KAAK;QACL,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;QACzB,KAAK,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,IAAI;QAClC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/C,KAAK,CAAC,OAAO,IAAI,IAAI;KACtB,CACF,CAAC;IACF,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,QAQI,EAAE;IAEN,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,MAAM,CAAC;IACrI,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;;;;;;;;;;;eAkBW,MAAM;;;;iBAIJ,WAAW,EAAE,EAC1B;QACE,KAAK,CAAC,KAAK,IAAI,IAAI;QACnB,KAAK,CAAC,IAAI,IAAI,IAAI;QAClB,CAAC,GAAG,oBAAoB,CAAC;QACzB,wBAAwB,CAAC,WAAW;QACpC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/C,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;KAClC,CACF,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,WAAW,CACf,IAAI,EACJ,cAAc,EACd,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAC7D,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAWD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAU;IAC/C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,oBAAoB,uBAAuB;;iCAEd,CAC9B,CAAC;IACF,MAAM,MAAM,GAAc;QACxB,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;KACb,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI;QAAE,MAAM,CAAC,GAAG,CAAC,KAAwB,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7E,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAU,EACV,KAA8D;IAE9D,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACpE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;cAEU,yBAAyB;;kBAErB,WAAW,EAAE,EAC3B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,IAAI,OAAO,CAAC,CACjE,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,WAAW,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,GAAsB;IACpE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;kBAGc,WAAW,EAAE,EAC3B,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CACX,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,WAAW,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACnG,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"jobs.js","sourceRoot":"","sources":["../../src/queue/jobs.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EACL,eAAe,EACf,oBAAoB,EACpB,wBAAwB,GAEzB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,uBAAuB,EACvB,WAAW,EACX,yBAAyB,EACzB,aAAa,EACb,YAAY,EACZ,KAAK,GAIN,MAAM,YAAY,CAAC;AAsBpB,kEAAkE;AAClE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAU,EAAE,KAAmB;IAC3D,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC1F,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;iBAGa,WAAW,EAAE,EAC1B;QACE,KAAK,CAAC,IAAI;QACV,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC;QAClE,KAAK,CAAC,QAAQ,IAAI,CAAC;QACnB,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,IAAI;QACrC,KAAK,CAAC,WAAW,IAAI,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW;QAC5D,KAAK,CAAC,QAAQ,IAAI,IAAI;QACtB,KAAK,CAAC,cAAc,IAAI,IAAI;KAC7B,CACF,CAAC;IAEF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;QACrC,MAAM,WAAW,CACf,IAAI,EACJ,cAAc,EACd,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,EACjF,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;IAED,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,QAAkB,CAAC,CAAC;IACxE,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,QAAQ,cAAc,CAAC,CAAC;IAChG,OAAO,QAAQ,CAAC;AAClB,CAAC;AAWD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAU,EAAE,KAAiB;IAC1D,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1E,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9E,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;;;;mBAWe,UAAU;;;;;iBAKZ,WAAW,EAAE,EAC1B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAClC,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,aAAa,EACb,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,EAC9E,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,KAAa,EACb,MAAc,EACd,OAAe;IAEf,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CACnC;;;mEAG+D,EAC/D,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,CAChC,CAAC;IACF,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,yDAAyD;AACzD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAU,EACV,KAAa,EACb,MAAc,EACd,MAAgB;IAEhB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;iBAQa,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CACtE,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,eAAe,EACf,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,EACjE,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAeD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAU,EACV,KAAa,EACb,MAAc,EACd,KAAa,EACb,QAAmB,EAAE,KAAK,EAAE,IAAI,EAAE;IAElC,0EAA0E;IAC1E,wDAAwD;IACxD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;kBAKc,aAAa,GAAG,IAAI,MAAM,YAAY;;;;;;;;iBAQvC,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CACnG,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC;IACzC,MAAM,WAAW,CACf,IAAI,EACJ,YAAY,EACZ;QACE,KAAK,EAAE,GAAG,CAAC,EAAE;QACb,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM;QACN,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,KAAK;QACL,QAAQ;QACR,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,KAAK,CAAC,cAAc;YACtB,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE;YAC1F,CAAC,CAAC,EAAE,CAAC;KACR,EACD,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAiBD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAU,EACV,KAAa,EACb,MAAc,EACd,MAAc,EACd,OAAqB,EAAE;IAEvB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;IAChC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;iBAQa,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAC5E,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,eAAe,EACf,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,EACjD,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAaD,6CAA6C;AAC7C,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,KAAa,EACb,QAAqB,EAAE;IAEvB,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC;IACjC,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC;IAC7C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;iBAQa,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CACpE,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,aAAa,EACb,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EACpF,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAU,EAAE,GAAS;IAC5D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;iBAOa,WAAW,EAAE,EAC1B,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CACpB,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,WAAW,CAAC,IAAI,EAAE,mBAAmB,EAAE;YAC3C,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;SACvB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,MAAM,kBAAkB,GAAG,0BAA0B,CAAC;AAsB7D;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAU,EACV,KAAqB,EACrB,MAAe,EACf,OAOI,EAAE;IAEN,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,IAAI,KAAa,CAAC;IAClB,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,KAAK,GAAG,oBAAoB,CAAC;IAC/B,CAAC;SAAM,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC;QACjF,KAAK,GAAG,kGAAkG,CAAC;IAC7G,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3C,KAAK,GAAG,iCAAiC,CAAC;IAC5C,CAAC;IACD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAI3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCAgCgC,KAAK,EAAE,EACvC,MAAM,CACP,CAAC;IAEF,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC9E,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC;QAChH,MAAM,OAAO,GAAG,OAAO;YACrB,CAAC,CAAC,GAAG,GAAG,gBAAgB;YACxB,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAC5C,CAAC,CAAC,GAAG,GAAG,mHAAmH;gBAC7H,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;oBAChB,CAAC,CAAC,GAAG,GAAG,wBAAwB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mFAAmF;oBACnI,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC;wBACd,CAAC,CAAC,GAAG,GAAG,sEAAsE;wBAC9E,CAAC,CAAC,GAAG,GAAG,uBAAuB,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;;;mBAUa,WAAW,EAAE,EAC1B,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,CACxC,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;QACrC,MAAM,WAAW,CACf,IAAI,EACJ,iBAAiB,EACjB,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EACvH,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,0EAA0E;YAC1E,yEAAyE;YACzE,MAAM,YAAY,GAAI,GAAG,CAAC,OAA6C,EAAE,YAAY,CAAC;YACtF,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,KAAK,CACd;sDAC4C,EAC5C,CAAC,YAAY,EAAE,OAAO,CAAC,CACxB,CAAC;YACJ,CAAC;YACD,MAAM,WAAW,CACf,IAAI,EACJ,YAAY,EACZ;gBACE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBAC5E,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK;gBAC7D,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG;aAChD,EACD,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAU,EAAE,KAAa;IACvD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;iBAMa,WAAW,EAAE,EAC1B,CAAC,KAAK,CAAC,CACR,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,eAAe,EACf,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,EACjC,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAU,EAAE,KAAa;IACtD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;;;;;;iBAaa,WAAW,EAAE,EAC1B,CAAC,KAAK,EAAE,CAAC,GAAG,oBAAoB,CAAC,EAAE,wBAAwB,CAAC,WAAW,CAAC,CACzE,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;IACrC,MAAM,WAAW,CACf,IAAI,EACJ,cAAc,EACd,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EACjD,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAU,EAAE,KAAa;IACpD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,UAAU,WAAW,qCAAqC,EAC1D,CAAC,KAAK,CAAC,CACR,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAU,EAAE,QAAgB;IACjE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,UAAU,WAAW,sCAAsC,EAC3D,CAAC,QAAQ,CAAC,CACX,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3D,CAAC;AAkBD,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAU,EAAE,QAAuB,EAAE;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,yBAAyB;QAChE,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,uBAAuB;YACxD,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,uBAAuB,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,UAAU,WAAW;;;aAGZ,MAAM;;wBAEK,EACpB,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CACzH,CAAC;IACF,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC;AAmBD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,KAAwB;IACrE,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,oBAAoB,CAAC,CAAC,CAAC;IACzD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,UAAU,WAAW;;;;;;;;;;cAUX,EACV;QACE,KAAK;QACL,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;QACzB,KAAK,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,IAAI;QAClC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/C,KAAK,CAAC,OAAO,IAAI,IAAI;KACtB,CACF,CAAC;IACF,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,QAQI,EAAE;IAEN,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,MAAM,CAAC;IACrI,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;;;;;;;;;;;;;;;;eAkBW,MAAM;;;;iBAIJ,WAAW,EAAE,EAC1B;QACE,KAAK,CAAC,KAAK,IAAI,IAAI;QACnB,KAAK,CAAC,IAAI,IAAI,IAAI;QAClB,CAAC,GAAG,oBAAoB,CAAC;QACzB,wBAAwB,CAAC,WAAW;QACpC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/C,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;KAClC,CACF,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,WAAW,CACf,IAAI,EACJ,cAAc,EACd,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAC7D,GAAG,CAAC,cAAc,IAAI,SAAS,CAChC,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAWD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAU;IAC/C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B,oBAAoB,uBAAuB;;iCAEd,CAC9B,CAAC;IACF,MAAM,MAAM,GAAc;QACxB,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;KACb,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI;QAAE,MAAM,CAAC,GAAG,CAAC,KAAwB,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7E,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAU,EACV,KAA8D;IAE9D,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACpE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;cAEU,yBAAyB;;kBAErB,WAAW,EAAE,EAC3B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,IAAI,OAAO,CAAC,CACjE,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,WAAW,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,GAAsB;IACpE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAC/B;;;kBAGc,WAAW,EAAE,EAC3B,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CACX,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,WAAW,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACnG,OAAO,IAAI,CAAC;AACd,CAAC"}

package/packages/core/dist/queue/worker.d.ts

@@ -5,8 +5,16 @@
  * name:
  *
  *  - **Startup recovery is Phase 1.** Before the first claim, every lease that
- *    outlived its process is released. A machine that lost power mid-mission
- *    picks the work back up rather than leaving it leased forever.
+ *    outlived its process is settled (`interruptLeases`): the expired ones,
+ *    and every one held by a previous boot of this worker (`holderPrefix`),
+ *    however long its lease still had to run. A restart takes seconds and a
+ *    lease lasts minutes, so expiry alone would leave the run stranded until
+ *    some later restart happened to come after it lapsed.
+ *  - **Leases are swept while running.** Every `sweepMs` a lease whose
+ *    heartbeat stopped longer than the lease ago is settled the same way.
+ *  - **Shutdown hands its leases back.** `stop()` aborts the runs, gives them
+ *    `stopGraceMs` to unwind, and settles whatever this worker still holds, so
+ *    the next boot finds nothing leased.
  *  - **Leases are heartbeated, and the heartbeat is a fence.** When a heartbeat
  *    says the lease is gone, the run is abandoned *immediately* — another
  *    worker owns the job now, and writing anything further would be writing
@@ -21,6 +29,7 @@
  * to be nearby.
  */
 import type { Pool } from 'pg';
+import { type InterruptedJob } from './jobs.js';
 import type { Job } from './types.js';
 /** What a handler may return instead of a result: park the job, durably. */
 export type Suspension = {
@@ -66,6 +75,19 @@ export interface RunWorkerOptions {
     heartbeatMs?: number;
     /** Release stale leases before the first claim (default true). */
     recoverOnStart?: boolean;
+    /**
+     * Worker ids of the same family as this one (`serve:` for every boot of
+     * `buddi serve`). At start, a lease held by any other id with this prefix
+     * belongs to a process that is gone, and is settled whatever its expiry.
+     * Omitted: only expired leases are.
+     */
+    holderPrefix?: string;
+    /** Cadence of the running sweep for expired leases (default one minute; 0 turns it off). */
+    sweepMs?: number;
+    /** How long `stop()` lets aborted runs unwind before settling their leases (default 5s). */
+    stopGraceMs?: number;
+    /** Told about every lease settled at start, by the sweep, or at stop. */
+    onInterrupted?: (jobs: InterruptedJob[], when: 'start' | 'sweep' | 'stop') => void;
     onError?: (err: unknown, job?: Job) => void;
 }
 export interface WorkerHandle {

package/packages/core/dist/queue/worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/queue/worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAU/B,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,4EAA4E;AAC5E,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC,CAAC;AAEF,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,UAAU,CAMhE;AAED,MAAM,WAAW,UAAU;IACzB,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;OAGG;IACH,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9B,wEAAwE;IACxE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEzE,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,CAAC;IACX,6EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACrC,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,YAAY;IAC3B,2EAA2E;IAC3E,IAAI,IAAI,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;IAC5B,qEAAqE;IACrE,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,YAAY,CA+K9D"}
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/queue/worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,EAOL,KAAK,cAAc,EACpB,MAAM,WAAW,CAAC;AAEnB,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,4EAA4E;AAC5E,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC,CAAC;AAEF,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,UAAU,CAMhE;AAED,MAAM,WAAW,UAAU;IACzB,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;OAGG;IACH,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9B,wEAAwE;IACxE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEzE,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,CAAC;IACX,6EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACrC,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4FAA4F;IAC5F,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4FAA4F;IAC5F,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,KAAK,IAAI,CAAC;IACnF,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,YAAY;IAC3B,2EAA2E;IAC3E,IAAI,IAAI,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;IAC5B,qEAAqE;IACrE,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,YAAY,CAiO9D"}

package/packages/core/dist/queue/worker.js

@@ -1,4 +1,4 @@
-import { claimJob, completeJob, failJob, heartbeat, releaseStaleLeases, suspendJob, } from './jobs.js';
+import { claimJob, completeJob, failJob, heartbeat, interruptLeases, suspendJob, } from './jobs.js';
 import { decideRetry } from './retry-policy.js';
 export function isSuspension(value) {
     return (typeof value === 'object' &&
@@ -16,10 +16,40 @@ export function runWorker(opts) {
     const done = new Promise((resolve) => {
         resolveDone = resolve;
     });
+    let resolveLoop = () => { };
+    const loopExited = new Promise((resolve) => {
+        resolveLoop = resolve;
+    });
+    let stopping = null;
     let timer = null;
     let wake = null;
     const active = new Set();
-    const recover = async () => releaseStaleLeases(pool, now());
+    const onInterrupted = opts.onInterrupted ?? (() => { });
+    const recover = async () => {
+        const scope = opts.holderPrefix
+            ? { expiredBy: now(), self: worker, holderPrefix: opts.holderPrefix }
+            : { expiredBy: now() };
+        const jobs = await interruptLeases(pool, scope);
+        if (jobs.length > 0)
+            onInterrupted(jobs, 'start');
+        return jobs.length;
+    };
+    // The running sweep: a run whose holder stopped heartbeating (a process that
+    // died without being told, a worker wedged past its lease) is settled within
+    // a minute of its lease lapsing, rather than at whichever restart comes next.
+    let sweeping = false;
+    const sweepMs = opts.sweepMs ?? 60_000;
+    const sweeper = sweepMs > 0 ? setInterval(() => {
+        if (!running || sweeping)
+            return;
+        sweeping = true;
+        void interruptLeases(pool, { expiredBy: now() }, 'its lease lapsed without a heartbeat')
+            .then((jobs) => { if (jobs.length > 0)
+            onInterrupted(jobs, 'sweep'); })
+            .catch(onError)
+            .finally(() => { sweeping = false; });
+    }, sweepMs) : null;
+    sweeper?.unref?.();
     const tick = async () => {
         const claimStartedAt = Date.now();
         const job = await claimJob(pool, {
@@ -169,21 +199,46 @@ export function runWorker(opts) {
             });
             wake = null;
         }
-        resolveDone();
+        resolveLoop();
     };
     void loop();
     return {
         tick,
         recover,
         done,
-        async stop() {
-            running = false;
-            for (const controller of active)
-                controller.abort(new Error('worker stopped'));
-            if (timer)
-                clearTimeout(timer);
-            wake?.();
-            await done;
+        stop() {
+            stopping ??= (async () => {
+                running = false;
+                if (sweeper)
+                    clearInterval(sweeper);
+                for (const controller of active)
+                    controller.abort(new Error('worker stopped'));
+                if (timer)
+                    clearTimeout(timer);
+                wake?.();
+                // Bounded: a handler that ignores its signal does not hold the stop
+                // past the service manager's grace (launchd's is 20s).
+                let grace;
+                const exited = await Promise.race([
+                    loopExited.then(() => true),
+                    new Promise((resolve) => { grace = setTimeout(() => resolve(false), opts.stopGraceMs ?? 5_000); grace.unref?.(); }),
+                ]);
+                clearTimeout(grace);
+                // The runs this process was in the middle of are settled now, not left
+                // leased for the next boot to find. A handler that ignored the stop is
+                // still running here: its job fails rather than going back on the
+                // queue, or the next start would run it a second time beside it.
+                try {
+                    const jobs = await interruptLeases(pool, { heldBy: worker }, 'buddi was stopping', exited ? {} : { requeue: false });
+                    if (jobs.length > 0)
+                        onInterrupted(jobs, 'stop');
+                }
+                catch (err) {
+                    onError(err);
+                }
+                resolveDone();
+            })();
+            return stopping;
         },
     };
 }

package/packages/core/dist/queue/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/queue/worker.ts"],"names":[],"mappings":"AAuBA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,OAAO,EACP,SAAS,EACT,kBAAkB,EAClB,UAAU,GACX,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAehD,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAAoB,CAAC,SAAS,KAAK,QAAQ,CACpD,CAAC;AACJ,CAAC;AA8CD,MAAM,UAAU,SAAS,CAAC,IAAsB;IAC9C,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,GAAS,EAAE,GAAE,CAAC,CAAC,CAAC;IAEjD,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,WAAW,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACzC,WAAW,GAAG,OAAO,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,IAAI,KAAK,GAA0B,IAAI,CAAC;IACxC,IAAI,IAAI,GAAwB,IAAI,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmB,CAAC;IAE1C,MAAM,OAAO,GAAG,KAAK,IAAqB,EAAE,CAAC,kBAAkB,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IAE7E,MAAM,IAAI,GAAG,KAAK,IAAyB,EAAE;QAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE;YAC/B,MAAM;YACN,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,GAAG,EAAE,GAAG,EAAE;YACV,OAAO;SACR,CAAC,CAAC;QACH,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,0EAA0E;QAC1E,0DAA0D;QAC1D,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,cAAc,GAAG,OAAO;YAAE,OAAO,GAAG,CAAC;QAEnE,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,yEAAyE;YACzE,sEAAsE;YACtE,8CAA8C;YAC9C,MAAM,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,4BAA4B,GAAG,CAAC,IAAI,GAAG,EAAE;gBAC3E,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC;QAED,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvB,MAAM,SAAS,GAAG,GAAS,EAAE;YAC3B,IAAI,GAAG,IAAI,CAAC;YACZ,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC;QACF,IAAI,UAA0B,CAAC;QAC/B,MAAM,UAAU,GAAG,CAAC,QAAgB,EAAQ,EAAE;YAC5C,YAAY,CAAC,UAAU,CAAC,CAAC;YACzB,UAAU,GAAG,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACvE,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC;QACvB,CAAC,CAAC;QACF,UAAU,CAAC,cAAc,GAAG,OAAO,CAAC,CAAC;QACrC,IAAI,WAAW,GAAkB,IAAI,CAAC;QACtC,IAAI,YAAiD,CAAC;QAEtD,MAAM,IAAI,GAAG,KAAK,IAAsB,EAAE;YACxC,IAAI,IAAI,IAAI,QAAQ;gBAAE,OAAO,KAAK,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,KAAc,CAAC;YACnB,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YACzD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBAClB,uEAAuE;gBACvE,OAAO,CAAC,IAAI,CAAC;YACf,CAAC;YACD,IAAI,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAC3B,IAAI,CAAC,KAAK;gBAAE,SAAS,EAAE,CAAC;iBACnB,IAAI,CAAC,IAAI;gBAAE,UAAU,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC;YAChD,OAAO,KAAK,IAAI,CAAC,IAAI,CAAC;QACxB,CAAC,CAAC;QAEF,MAAM,GAAG,GAAe;YACtB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS,EAAE,IAAI;YACf,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI;gBACxB,WAAW,GAAG,MAAM,CAAC;gBACrB,IAAI,IAAI,EAAE,YAAY;oBAAE,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;YAC3D,CAAC;YACD,IAAI,IAAI;gBACN,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;YAC9B,KAAK,IAAI,EAAE,CAAC;QACd,CAAC,EAAE,WAAW,CAAC,CAAC;QAChB,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU;YAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAEvD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACvC,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,IAAI,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,GAAG,CAAC,CAAC,qCAAqC;YAEvE,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;YACzE,MAAM,KAAK,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,YAAY,CAAC;YACvF,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBACpD,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzF,OAAO,GAAG,CAAC;YACb,CAAC;YACD,MAAM,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAClB,IAAI,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,GAAG,CAAC;YACjC,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,sEAAsE;YACtE,uEAAuE;YACvE,wEAAwE;YACxE,mEAAmE;YACnE,MAAM,QAAQ,GAAG,WAAW,CAAC;gBAC3B,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,GAAG,EAAE,GAAG,EAAE;gBACV,KAAK,EAAE,GAAG;aACX,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;gBAC3C,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,GAAG,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;gBAC9E,cAAc,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;aAC1E,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;gBAAS,CAAC;YACT,QAAQ,GAAG,IAAI,CAAC;YAChB,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,YAAY,CAAC,UAAW,CAAC,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACrC,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;YAClC,MAAM,OAAO,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,OAAO,EAAE,CAAC;YACf,IAAI,GAAG,GAAe,IAAI,CAAC;YAC3B,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,IAAI,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,CAAC;YACf,CAAC;YACD,IAAI,CAAC,OAAO;gBAAE,MAAM;YACpB,gEAAgE;YAChE,IAAI,GAAG;gBAAE,SAAS;YAClB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,IAAI,GAAG,OAAO,CAAC;gBACf,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,UAAU;oBAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YACvD,CAAC,CAAC,CAAC;YACH,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;QACD,WAAW,EAAE,CAAC;IAChB,CAAC,CAAC;IAEF,KAAK,IAAI,EAAE,CAAC;IAEZ,OAAO;QACL,IAAI;QACJ,OAAO;QACP,IAAI;QACJ,KAAK,CAAC,IAAI;YACR,OAAO,GAAG,KAAK,CAAC;YAChB,KAAK,MAAM,UAAU,IAAI,MAAM;gBAAE,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAC/E,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,EAAE,EAAE,CAAC;YACT,MAAM,IAAI,CAAC;QACb,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/queue/worker.ts"],"names":[],"mappings":"AA+BA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,OAAO,EACP,SAAS,EACT,eAAe,EACf,UAAU,GAEX,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAehD,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAAoB,CAAC,SAAS,KAAK,QAAQ,CACpD,CAAC;AACJ,CAAC;AA2DD,MAAM,UAAU,SAAS,CAAC,IAAsB;IAC9C,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,GAAS,EAAE,GAAE,CAAC,CAAC,CAAC;IAEjD,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,WAAW,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACzC,WAAW,GAAG,OAAO,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,IAAI,WAAW,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAC/C,WAAW,GAAG,OAAO,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,IAAI,QAAQ,GAAyB,IAAI,CAAC;IAC1C,IAAI,KAAK,GAA0B,IAAI,CAAC;IACxC,IAAI,IAAI,GAAwB,IAAI,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmB,CAAC;IAE1C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,GAAS,EAAE,GAAE,CAAC,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,KAAK,IAAqB,EAAE;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY;YAC7B,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE;YACrE,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC,CAAC;IAEF,6EAA6E;IAC7E,6EAA6E;IAC7E,8EAA8E;IAC9E,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC;IACvC,MAAM,OAAO,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE;QAC7C,IAAI,CAAC,OAAO,IAAI,QAAQ;YAAE,OAAO;QACjC,QAAQ,GAAG,IAAI,CAAC;QAChB,KAAK,eAAe,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,EAAE,sCAAsC,CAAC;aACrF,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;aACtE,KAAK,CAAC,OAAO,CAAC;aACd,OAAO,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACnB,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC;IAEnB,MAAM,IAAI,GAAG,KAAK,IAAyB,EAAE;QAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE;YAC/B,MAAM;YACN,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,GAAG,EAAE,GAAG,EAAE;YACV,OAAO;SACR,CAAC,CAAC;QACH,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,0EAA0E;QAC1E,0DAA0D;QAC1D,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,cAAc,GAAG,OAAO;YAAE,OAAO,GAAG,CAAC;QAEnE,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,yEAAyE;YACzE,sEAAsE;YACtE,8CAA8C;YAC9C,MAAM,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,4BAA4B,GAAG,CAAC,IAAI,GAAG,EAAE;gBAC3E,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC;QAED,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvB,MAAM,SAAS,GAAG,GAAS,EAAE;YAC3B,IAAI,GAAG,IAAI,CAAC;YACZ,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC;QACF,IAAI,UAA0B,CAAC;QAC/B,MAAM,UAAU,GAAG,CAAC,QAAgB,EAAQ,EAAE;YAC5C,YAAY,CAAC,UAAU,CAAC,CAAC;YACzB,UAAU,GAAG,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACvE,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC;QACvB,CAAC,CAAC;QACF,UAAU,CAAC,cAAc,GAAG,OAAO,CAAC,CAAC;QACrC,IAAI,WAAW,GAAkB,IAAI,CAAC;QACtC,IAAI,YAAiD,CAAC;QAEtD,MAAM,IAAI,GAAG,KAAK,IAAsB,EAAE;YACxC,IAAI,IAAI,IAAI,QAAQ;gBAAE,OAAO,KAAK,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,KAAc,CAAC;YACnB,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YACzD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBAClB,uEAAuE;gBACvE,OAAO,CAAC,IAAI,CAAC;YACf,CAAC;YACD,IAAI,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAC3B,IAAI,CAAC,KAAK;gBAAE,SAAS,EAAE,CAAC;iBACnB,IAAI,CAAC,IAAI;gBAAE,UAAU,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC;YAChD,OAAO,KAAK,IAAI,CAAC,IAAI,CAAC;QACxB,CAAC,CAAC;QAEF,MAAM,GAAG,GAAe;YACtB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS,EAAE,IAAI;YACf,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI;gBACxB,WAAW,GAAG,MAAM,CAAC;gBACrB,IAAI,IAAI,EAAE,YAAY;oBAAE,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;YAC3D,CAAC;YACD,IAAI,IAAI;gBACN,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;YAC9B,KAAK,IAAI,EAAE,CAAC;QACd,CAAC,EAAE,WAAW,CAAC,CAAC;QAChB,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU;YAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAEvD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACvC,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,IAAI,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,GAAG,CAAC,CAAC,qCAAqC;YAEvE,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;YACzE,MAAM,KAAK,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,YAAY,CAAC;YACvF,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBACpD,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzF,OAAO,GAAG,CAAC;YACb,CAAC;YACD,MAAM,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAClB,IAAI,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,GAAG,CAAC;YACjC,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,sEAAsE;YACtE,uEAAuE;YACvE,wEAAwE;YACxE,mEAAmE;YACnE,MAAM,QAAQ,GAAG,WAAW,CAAC;gBAC3B,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,GAAG,EAAE,GAAG,EAAE;gBACV,KAAK,EAAE,GAAG;aACX,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;gBAC3C,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,GAAG,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;gBAC9E,cAAc,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;aAC1E,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;gBAAS,CAAC;YACT,QAAQ,GAAG,IAAI,CAAC;YAChB,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,YAAY,CAAC,UAAW,CAAC,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACrC,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;YAClC,MAAM,OAAO,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,OAAO,EAAE,CAAC;YACf,IAAI,GAAG,GAAe,IAAI,CAAC;YAC3B,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,IAAI,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,CAAC;YACf,CAAC;YACD,IAAI,CAAC,OAAO;gBAAE,MAAM;YACpB,gEAAgE;YAChE,IAAI,GAAG;gBAAE,SAAS;YAClB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,IAAI,GAAG,OAAO,CAAC;gBACf,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,UAAU;oBAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YACvD,CAAC,CAAC,CAAC;YACH,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;QACD,WAAW,EAAE,CAAC;IAChB,CAAC,CAAC;IAEF,KAAK,IAAI,EAAE,CAAC;IAEZ,OAAO;QACL,IAAI;QACJ,OAAO;QACP,IAAI;QACJ,IAAI;YACF,QAAQ,KAAK,CAAC,KAAK,IAAI,EAAE;gBACvB,OAAO,GAAG,KAAK,CAAC;gBAChB,IAAI,OAAO;oBAAE,aAAa,CAAC,OAAO,CAAC,CAAC;gBACpC,KAAK,MAAM,UAAU,IAAI,MAAM;oBAAE,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBAC/E,IAAI,KAAK;oBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;gBAC/B,IAAI,EAAE,EAAE,CAAC;gBACT,oEAAoE;gBACpE,uDAAuD;gBACvD,IAAI,KAAiC,CAAC;gBACtC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;oBAChC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;oBAC3B,IAAI,OAAO,CAAU,CAAC,OAAO,EAAE,EAAE,GAAG,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;iBAC7H,CAAC,CAAC;gBACH,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,uEAAuE;gBACvE,uEAAuE;gBACvE,kEAAkE;gBAClE,iEAAiE;gBACjE,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;oBACrH,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;wBAAE,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACnD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;gBACD,WAAW,EAAE,CAAC;YAChB,CAAC,CAAC,EAAE,CAAC;YACL,OAAO,QAAQ,CAAC;QAClB,CAAC;KACF,CAAC;AACJ,CAAC"}

package/packages/core/dist/registry.d.ts

@@ -249,6 +249,14 @@ export declare class ToolRegistry {
     has(name: string): boolean;
     isSequential(name: string): boolean;
     waitsForOwner(name: string): boolean;
+    /**
+     * The tier a tool always runs at. Undefined for an unknown tool and for one
+     * that decides per call (`tierFor`): a caller that needs certainty, like the
+     * restart recovery deciding whether a run had acted, reads that as "may have".
+     */
+    tierOf(name: string): Tier | undefined;
+    /** Whether a call to this tool reaches outside the run without an approval (`sideEffect`). */
+    hasSideEffect(name: string): boolean;
     /** What kind of untrusted text this tool's output is, when it declares one. */
     untrustedKind(name: string): UntrustedKind | undefined;
     image(name: string, output: unknown, ctx: CoreToolContext): Promise<{

package/packages/core/dist/registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAE,gBAAgB,EAAqB,cAAc,EAAE,eAAe,EAAE,IAAI,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE9I,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AACvE,OAAO,EAA0C,KAAK,cAAc,EAAE,KAAK,SAAS,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AAC9H,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EAA+B,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAClF,OAAO,EAAgB,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACnE,OAAO,EAAmB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAoG,KAAK,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAKrJ,OAAO,EAAkD,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAE5G,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,qEAAqE;AACrE,eAAO,MAAM,gBAAgB,EAAE,SAAS,IAAI,EAAa,CAAC;AAE1D,0DAA0D;AAC1D,eAAO,MAAM,WAAW,EAAE,SAAS,IAAI,EAAc,CAAC;AAEtD;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,EAAE,SAAS,IAAI,EAAiC,CAAC;AAoB5E,oEAAoE;AACpE,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEjE,8DAA8D;AAC9D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7D,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,IAAI,CAAC;IACX,+FAA+F;IAC/F,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC,CAAC;AAEF,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,KAAK,EAAE,OAAO,GACpB;QAAE,OAAO,EAAE,IAAI,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,GAChC;QAAE,OAAO,EAAE,KAAK,CAAC;QAAC,KAAK,EAAE;YAAE,MAAM,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAA;aAAE,CAAC,CAAA;SAAE,CAAA;KAAE,CAAC;CACrG;AAED,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,OAAO,IAChC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,CAAC,CAAA;CAAE,GACvB;IACE;;;;OAIG;IACH,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB,GACD;IACE,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,cAAc,GAAG,cAAc,GAAG,qBAAqB,GAAG,YAAY,GAAG,wBAAwB,GAAG,UAAU,CAAC;IACvH,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAkBN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AA4F1C;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,CAkC1F;AAED,mEAAmE;AACnE,eAAO,MAAM,sBAAsB,OAAQ,CAAC;AAU5C,qBAAa,YAAY;;IAwBvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IA4IxC;;;;;;;OAOG;IACH,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAqInC,gDAAgD;IAChD,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED;;;;;;OAMG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAqB1C,SAAS,IAAI,cAAc,EAAE;IAI7B;;;;;OAKG;IACH,KAAK,IAAI,cAAc,EAAE;IAIzB;;;;OAIG;IACH,KAAK,IAAI,cAAc,EAAE;IAIzB;;;OAGG;IACH,OAAO,IAAI,eAAe,EAAE;IAI5B;;;OAGG;IACH,KAAK,IAAI,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAInD;;;;;;;;OAQG;IACH;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,GAAG,SAAS;IAK7F,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE;IAInC,gFAAgF;IAChF,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI1C;;;;;;;;OAQG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS;IAMrD;;;;OAIG;IACH,UAAU,IAAI,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;KAAE,CAAC;IAQtH,gFAAgF;IAChF,IAAI,IAAI,gBAAgB,EAAE;IAU1B,oFAAoF;IACpF,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAK1C;;;;;;OAMG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,eAAe,EAAE,SAAS,SAAyB,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAa/H;;;;OAIG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAIpD,2FAA2F;IAC3F,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA0B5D;;;;;;OAMG;IACG,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAgCvG,kGAAkG;IAClG,OAAO,IAAI,gBAAgB,EAAE;IAI7B,0CAA0C;IAC1C,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAQhD;;;;;OAKG;IACH,OAAO,IAAI,gBAAgB,EAAE;IAI7B,gFAAgF;IAChF,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAQhD,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAI1B,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAInC,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIpC,+EAA+E;IAC/E,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAIhD,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC;IAKrH;;;;;OAKG;IACH,IAAI,IAAI,QAAQ,EAAE;IAWlB;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS;IAuB1C,MAAM,CACV,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,eAAe;IACvB;;;;;;OAMG;IACH,IAAI,GAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAO,GAC/B,OAAO,CAAC,YAAY,CAAC;CAqRzB"}
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAE,gBAAgB,EAAqB,cAAc,EAAE,eAAe,EAAE,IAAI,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE9I,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AACvE,OAAO,EAA0C,KAAK,cAAc,EAAE,KAAK,SAAS,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AAC9H,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EAA+B,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAClF,OAAO,EAAgB,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACnE,OAAO,EAAmB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAoG,KAAK,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAKrJ,OAAO,EAAkD,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAE5G,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,qEAAqE;AACrE,eAAO,MAAM,gBAAgB,EAAE,SAAS,IAAI,EAAa,CAAC;AAE1D,0DAA0D;AAC1D,eAAO,MAAM,WAAW,EAAE,SAAS,IAAI,EAAc,CAAC;AAEtD;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,EAAE,SAAS,IAAI,EAAiC,CAAC;AAoB5E,oEAAoE;AACpE,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEjE,8DAA8D;AAC9D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7D,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,IAAI,CAAC;IACX,+FAA+F;IAC/F,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC,CAAC;AAEF,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,KAAK,EAAE,OAAO,GACpB;QAAE,OAAO,EAAE,IAAI,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,GAChC;QAAE,OAAO,EAAE,KAAK,CAAC;QAAC,KAAK,EAAE;YAAE,MAAM,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAA;aAAE,CAAC,CAAA;SAAE,CAAA;KAAE,CAAC;CACrG;AAED,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,OAAO,IAChC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,CAAC,CAAA;CAAE,GACvB;IACE;;;;OAIG;IACH,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB,GACD;IACE,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,cAAc,GAAG,cAAc,GAAG,qBAAqB,GAAG,YAAY,GAAG,wBAAwB,GAAG,UAAU,CAAC;IACvH,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAkBN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AA4F1C;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,CAkC1F;AAED,mEAAmE;AACnE,eAAO,MAAM,sBAAsB,OAAQ,CAAC;AAU5C,qBAAa,YAAY;;IAwBvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IA4IxC;;;;;;;OAOG;IACH,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAqInC,gDAAgD;IAChD,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED;;;;;;OAMG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAqB1C,SAAS,IAAI,cAAc,EAAE;IAI7B;;;;;OAKG;IACH,KAAK,IAAI,cAAc,EAAE;IAIzB;;;;OAIG;IACH,KAAK,IAAI,cAAc,EAAE;IAIzB;;;OAGG;IACH,OAAO,IAAI,eAAe,EAAE;IAI5B;;;OAGG;IACH,KAAK,IAAI,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAInD;;;;;;;;OAQG;IACH;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,GAAG,SAAS;IAK7F,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE;IAInC,gFAAgF;IAChF,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI1C;;;;;;;;OAQG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS;IAMrD;;;;OAIG;IACH,UAAU,IAAI,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;KAAE,CAAC;IAQtH,gFAAgF;IAChF,IAAI,IAAI,gBAAgB,EAAE;IAU1B,oFAAoF;IACpF,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAK1C;;;;;;OAMG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,eAAe,EAAE,SAAS,SAAyB,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAa/H;;;;OAIG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAIpD,2FAA2F;IAC3F,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA0B5D;;;;;;OAMG;IACG,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAgCvG,kGAAkG;IAClG,OAAO,IAAI,gBAAgB,EAAE;IAI7B,0CAA0C;IAC1C,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAQhD;;;;;OAKG;IACH,OAAO,IAAI,gBAAgB,EAAE;IAI7B,gFAAgF;IAChF,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAQhD,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAI1B,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAInC,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIpC;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAKtC,8FAA8F;IAC9F,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIpC,+EAA+E;IAC/E,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAIhD,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC;IAKrH;;;;;OAKG;IACH,IAAI,IAAI,QAAQ,EAAE;IAWlB;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS;IAuB1C,MAAM,CACV,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,eAAe;IACvB;;;;;;OAMG;IACH,IAAI,GAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAO,GAC/B,OAAO,CAAC,YAAY,CAAC;CAqRzB"}

package/packages/core/dist/registry.js

@@ -14,7 +14,7 @@
  *  - Tool threw            -> 'tool-error'; defects never surface as success.
  */
 import { zodToJsonSchema } from 'zod-to-json-schema';
-import { decideApproval } from './actions/approvals.js';
+import { asksEachTime, decideApproval } from './actions/approvals.js';
 import { executeApproved } from './actions/execute.js';
 import { findToolPermission } from './actions/permissions.js';
 import { missionExtrasProblem } from './tools.js';
@@ -749,6 +749,19 @@ export class ToolRegistry {
     waitsForOwner(name) {
         return this.#tools.get(name)?.tool.waitsForOwner === true;
     }
+    /**
+     * The tier a tool always runs at. Undefined for an unknown tool and for one
+     * that decides per call (`tierFor`): a caller that needs certainty, like the
+     * restart recovery deciding whether a run had acted, reads that as "may have".
+     */
+    tierOf(name) {
+        const tool = this.#tools.get(name)?.tool;
+        return tool && tool.tierFor === undefined ? tool.tier : undefined;
+    }
+    /** Whether a call to this tool reaches outside the run without an approval (`sideEffect`). */
+    hasSideEffect(name) {
+        return this.#tools.get(name)?.tool.sideEffect === true;
+    }
     /** What kind of untrusted text this tool's output is, when it declares one. */
     untrustedKind(name) {
         return this.#tools.get(name)?.tool.untrusted;
@@ -1039,7 +1052,7 @@ export class ToolRegistry {
              * permission row that could tell them apart, so a call whose tier was
              * decided per call is always put to the owner.
              */
-            const permission = tool.reusableApproval && !decidedPerCall
+            const permission = tool.reusableApproval && !decidedPerCall && !asksEachTime(described.envelope)
                 ? await findToolPermission(ctx.db, ctx, tool.name, version) : undefined;
             if (permission) {
                 const decision = await decideApproval(ctx.db, { actionId: action.id, decision: 'approved',