@whittlelabs/sifter 0.36.0 → 0.37.0

package/README.md

@@ -122,7 +122,9 @@ A running Sifter deals with Sift or `claude` going away on its own, and says wha
 - **The pairing is revoked** (removed in Sift, or replaced by a newer pairing): it exits with code 3 and shows the two ways to pair again. A review it was running goes back to the queue for another Sifter.
 - **It is decommissioned from Sift's settings page**: it finishes the review in hand and exits with code 0.
 
-Ctrl-C stops it at any time, including while it waits.
+Ctrl-C stops it at any time, including while it waits. A review in progress goes back to the queue for the next Sifter that checks in, and one that had already finished is submitted first; it quits within 20 seconds, or at once on a second Ctrl-C. Closing the terminal and `kill` do the same. If it stops without that chance (a crash, the machine asleep, the network gone), it stops checking in, and Jobs puts its reviews back in the queue once two minutes pass without a check-in (whittlelabs ADR 0129).
+
+Each run is logged to `~/.sifter/sifter.log` as well as the terminal (`~/.sifter/<profile>/sifter.log` for a named profile), rotated to `sifter.log.1` past 5 MB. A hosted Sifter logs to stdout only.
 
 ## Check
 

package/bin.js

@@ -22263,8 +22263,20 @@ var require_client2 = __commonJS({
        * callers should be rare. Returns the dispatched-job context the
        * subscriber needs to execute and submit.
        */
-      async acceptDispatch(jobId) {
-        return this.request("POST", `/api/jobs/${jobId}/attendance`);
+      async acceptDispatch(jobId, options) {
+        return this.request("POST", `/api/jobs/${jobId}/attendance`, options?.shiftId ? { shiftId: options.shiftId } : void 0);
+      }
+      // ── Shifts (whittlelabs ADR 0129) ────────────────────────────────
+      /**
+       * Start a shift for this process to claim on. A Jobs that predates shifts
+       * answers 404; the subscribe loop then claims without one.
+       */
+      async startShift() {
+        return this.request("POST", "/api/my/shifts");
+      }
+      /** End a shift, releasing every claim still taken on it back to the queue. */
+      async endShift(shiftId) {
+        await this.request("DELETE", `/api/my/shifts/${shiftId}`);
       }
       async submit(jobId, options) {
         return this.request("PUT", `/api/jobs/${jobId}/attendance`, options);
@@ -22286,12 +22298,15 @@ var require_client2 = __commonJS({
           params.set("offset", String(options.offset));
         if (options?.sort)
           params.set("sort", options.sort);
+        if (options?.shift)
+          params.set("shift", options.shift);
         const qs = params.toString();
-        const { jobs, total, directives } = await this.request("GET", `/api/my/jobs${qs ? `?${qs}` : ""}`);
+        const { jobs, total, directives, shift } = await this.request("GET", `/api/my/jobs${qs ? `?${qs}` : ""}`);
         return {
           data: jobs,
           pagination: { total, limit: options?.limit ?? 25, offset: options?.offset ?? 0 },
-          directives: directives ?? null
+          directives: directives ?? null,
+          shift: shift ?? null
         };
       }
       // ── HTTP plumbing ────────────────────────────────────────────────
@@ -22382,6 +22397,17 @@ var require_subscribe = __commonJS({
       const rest = seconds % 60;
       return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`;
     }
+    function statusOf(err) {
+      const code = err?.statusCode;
+      return typeof code === "number" ? code : void 0;
+    }
+    var HandBack = class {
+      reason;
+      constructor(reason) {
+        this.reason = reason;
+      }
+    };
+    var RELEASED = Symbol("released");
     function newOutcome() {
       let resolve;
       let reject;
@@ -22422,7 +22448,10 @@ var require_subscribe = __commonJS({
       let pollTimer = null;
       let inflight = 0;
       let resolvedPoolIds = [];
-      const controllers = /* @__PURE__ */ new Set();
+      const controllers = /* @__PURE__ */ new Map();
+      let shift = null;
+      let shiftsSupported = true;
+      let draining = null;
       const holds = /* @__PURE__ */ new Set();
       let fatalError = null;
       let outcome = newOutcome();
@@ -22551,13 +22580,74 @@ var require_subscribe = __commonJS({
         }
         if (stopped || mySession !== session)
           return;
-        logger.info("subscribe loop started", { pools: resolvedPoolIds, maxConcurrency });
+        try {
+          const opened = await untilAvailable(() => startShift(), mySession);
+          if (opened === STOPPED)
+            return;
+        } catch (err) {
+          if (mySession === session) {
+            stopped = true;
+            outcome.reject(err);
+          }
+          throw err;
+        }
+        if (stopped || mySession !== session)
+          return;
+        logger.info("subscribe loop started", {
+          pools: resolvedPoolIds,
+          maxConcurrency,
+          ...shift ? { shiftId: shift.id } : {}
+        });
         schedulePoll(pollIntervalMs);
       }
+      async function startShift() {
+        if (!shiftsSupported)
+          return;
+        try {
+          shift = await opts.jobsClient.startShift();
+        } catch (err) {
+          if (isFatal(err) || isUnavailable(err) || isRateLimited(err))
+            throw err;
+          if (statusOf(err) === 404) {
+            shiftsSupported = false;
+            logger.debug?.("Jobs predates shifts; claims are held until each job's deadline");
+            return;
+          }
+          logger.warn(`Could not start a shift (${describeOutage(err)}). Claims taken meanwhile are held until each job's deadline; the next poll tries again.`);
+        }
+      }
+      async function endShift() {
+        const ending = shift;
+        shift = null;
+        if (!ending)
+          return;
+        try {
+          await opts.jobsClient.endShift(ending.id);
+        } catch (err) {
+          logger.debug?.("could not end the shift; it will lapse", {
+            shiftId: ending.id,
+            error: err instanceof Error ? err.message : String(err)
+          });
+        }
+      }
+      async function shiftReleased(ended) {
+        const mine = [...controllers].filter(([, held]) => held.shiftId === ended.id);
+        logger.warn(`Jobs ended this Sifter's shift (${ended.endReason ?? "ended"}) and released its claims` + (mine.length > 0 ? `; ${mine.length} job(s) running on it were handed back to the queue and stop here` : "") + ". Starting a new shift.", { shiftId: ended.id });
+        for (const [controller] of mine)
+          controller.abort(RELEASED);
+        shift = null;
+        try {
+          await startShift();
+        } catch (err) {
+          if (isFatal(err))
+            throw err;
+        }
+      }
       async function finish() {
         while (inflight > 0) {
           await sleep(100);
         }
+        await endShift();
         logger.info("subscribe loop stopped");
         if (fatalError !== null)
           outcome.reject(fatalError.error);
@@ -22572,45 +22662,61 @@ var require_subscribe = __commonJS({
         }
         wakeStartup?.();
       }
-      async function stop() {
+      async function stop(options = {}) {
+        if (options.handBack !== void 0) {
+          for (const [controller] of controllers)
+            controller.abort(new HandBack(options.handBack));
+        }
+        if (draining)
+          return draining;
         if (stopped)
           return;
         halt();
-        await finish();
+        draining = finish().finally(() => {
+          draining = null;
+        });
+        return draining;
       }
       async function fail(err) {
         if (fatalError === null)
           fatalError = { error: err };
-        for (const controller of controllers)
+        for (const [controller] of controllers)
           controller.abort(err);
+        if (draining)
+          return draining;
         if (stopped)
           return;
         halt();
-        await finish();
+        draining = finish().finally(() => {
+          draining = null;
+        });
+        return draining;
       }
       function schedulePoll(delayMs) {
         const mySession = session;
+        const checkInMs = shift ? shift.lapseAfterSeconds * 1e3 / 4 : Infinity;
         pollTimer = setTimeout(() => {
           void poll(mySession);
-        }, delayMs);
+        }, Math.min(delayMs, checkInMs));
       }
       async function poll(mySession) {
         if (stopped || mySession !== session)
           return;
         let nextPollMs = pollIntervalMs;
-        if (inflight >= maxConcurrency) {
-          schedulePoll(pollIntervalMs);
-          return;
-        }
         try {
           const paused = holds.size > 0;
           const free = maxConcurrency - inflight;
-          const { data, directives } = await opts.jobsClient.listDispatchableJobs({
+          if (shift === null && shiftsSupported)
+            await startShift();
+          const { data, directives, shift: standing } = await opts.jobsClient.listDispatchableJobs({
             pools: resolvedPoolIds,
-            limit: paused ? 1 : accepts ? Math.max(free, DECLINE_WINDOW) : free,
-            sort: "priority:desc,queued_at:asc"
+            limit: paused || free <= 0 ? 1 : accepts ? Math.max(free, DECLINE_WINDOW) : free,
+            sort: "priority:desc,queued_at:asc",
+            ...shift ? { shift: shift.id } : {}
           });
           noteReachable();
+          if (shift && standing?.id === shift.id && standing.endedAt !== null)
+            await shiftReleased(standing);
           if (directives && Object.keys(directives).length > 0 && opts.onDirectives) {
             try {
               await opts.onDirectives(directives);
@@ -22658,8 +22764,9 @@ var require_subscribe = __commonJS({
       }
       async function runOnce(available) {
         let dispatch;
+        const shiftId = shift?.id ?? null;
         try {
-          dispatch = await opts.jobsClient.acceptDispatch(available.id);
+          dispatch = await opts.jobsClient.acceptDispatch(available.id, shiftId ? { shiftId } : void 0);
         } catch (err) {
           if (isFatal(err)) {
             void fail(err);
@@ -22676,8 +22783,13 @@ var require_subscribe = __commonJS({
           });
           return;
         }
+        logger.info(`Claimed job ${dispatch.job.id} (attempt ${dispatch.job.attempt})`, {
+          jobId: dispatch.job.id,
+          pool: dispatch.pool.name,
+          ...shiftId ? { shiftId } : {}
+        });
         const controller = new AbortController();
-        controllers.add(controller);
+        controllers.set(controller, { shiftId });
         if (fatalError !== null)
           controller.abort(fatalError.error);
         let quietTimer = null;
@@ -22717,9 +22829,20 @@ var require_subscribe = __commonJS({
           controllers.delete(controller);
         }
         if (fatalError !== null) {
-          logger.debug?.("not submitting a result after a fatal error", { jobId: dispatch.job.id });
+          logger.error(shiftId ? `Dropping job ${dispatch.job.id}'s result after a fatal error. Its claim goes back to the queue when this Sifter's shift lapses.` : `Dropping job ${dispatch.job.id}'s result after a fatal error. Its claim is held until the job's deadline.`, { jobId: dispatch.job.id, status: result.status });
           return;
         }
+        if (controller.signal.reason === RELEASED) {
+          logger.warn(`Job ${dispatch.job.id} was handed back to the queue while it ran; its result is not submitted.`, {
+            jobId: dispatch.job.id
+          });
+          return;
+        }
+        const handBack = controller.signal.reason instanceof HandBack ? controller.signal.reason.reason : null;
+        if (handBack !== null && result.status !== "completed") {
+          result = { status: "abandoned", reason: handBack };
+          logger.warn(`Handing job ${dispatch.job.id} back to the queue: ${handBack}`, { jobId: dispatch.job.id });
+        }
         try {
           for (const output of result.outputs ?? []) {
             await opts.jobsClient.createOutput(dispatch.job.id, {
@@ -22741,7 +22864,12 @@ var require_subscribe = __commonJS({
             jobId: dispatch.job.id,
             error: err instanceof Error ? err.message : String(err)
           });
+          return;
         }
+        logger.info(`Submitted job ${dispatch.job.id} as ${result.status}`, {
+          jobId: dispatch.job.id,
+          ...result.reason !== void 0 ? { reason: result.reason } : {}
+        });
       }
       return {
         start,
@@ -26382,9 +26510,12 @@ var require_logger = __commonJS({
   "../../packages/shuttle/dist/logging/logger.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
+    exports2.LOG_FILE_MAX_BYTES = void 0;
     exports2.createLogger = createLogger;
     exports2.setGlobalLogger = setGlobalLogger;
     exports2.getLogger = getLogger;
+    var node_fs_1 = require("node:fs");
+    var node_path_1 = require("node:path");
     var LEVEL_ORDER = {
       debug: 0,
       info: 1,
@@ -26412,13 +26543,42 @@ var require_logger = __commonJS({
       };
       return JSON.stringify(entry);
     }
+    exports2.LOG_FILE_MAX_BYTES = 5 * 1024 * 1024;
+    function fileSink(path) {
+      let size = 0;
+      let broken = false;
+      try {
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(path), { recursive: true });
+        size = (0, node_fs_1.statSync)(path).size;
+      } catch {
+        size = 0;
+      }
+      return (line) => {
+        if (broken)
+          return;
+        try {
+          if (size > exports2.LOG_FILE_MAX_BYTES) {
+            (0, node_fs_1.renameSync)(path, `${path}.1`);
+            size = 0;
+          }
+          const entry = `${line}
+`;
+          (0, node_fs_1.appendFileSync)(path, entry, { mode: 384 });
+          size += Buffer.byteLength(entry);
+        } catch {
+          broken = true;
+        }
+      };
+    }
     function createLogger(options) {
       const threshold = LEVEL_ORDER[options.level];
       const format = options.format === "json" ? formatJson : formatPretty;
+      const toFile = options.file ? fileSink(options.file) : null;
       function emit(level, message, context) {
         if (LEVEL_ORDER[level] < threshold)
           return;
         const line = format(level, message, context);
+        toFile?.(line);
         if (level === "error") {
           console.error(line);
         } else if (level === "warn") {
@@ -27714,6 +27874,7 @@ var require_shuttle = __commonJS({
     exports2.mergeExecutorDefaults = mergeExecutorDefaults;
     exports2.resolveAuth = resolveAuth;
     var os_1 = require("os");
+    var path_1 = require("path");
     var jobs_1 = require_dist4();
     var self_update_1 = require_self_update();
     var decommission_1 = require_decommission();
@@ -27854,13 +28015,18 @@ var require_shuttle = __commonJS({
         return this.spendTracker;
       }
       async start() {
+        const paths = (0, apply_1.resolvePaths)(this.brand, this.profile);
+        const logFile = this.config.credentials === void 0 ? (0, path_1.join)(paths.configDir, `${this.brand.product.id}.log`) : void 0;
         (0, logger_1.setGlobalLogger)((0, logger_1.createLogger)({
           level: this.config.logging.level,
-          format: this.config.logging.format
+          format: this.config.logging.format,
+          ...logFile ? { file: logFile } : {}
         }));
         const logger = (0, logger_1.getLogger)();
-        logger.info(`${this.brand.product.title} starting...`);
-        const paths = (0, apply_1.resolvePaths)(this.brand, this.profile);
+        logger.info(`${this.brand.product.title} starting...`, {
+          version: this.brand.product.version,
+          ...logFile ? { log: logFile } : {}
+        });
         this.broker = (0, credentials_1.createCredentialBroker)(this.config.credentials, this.credentialDeps);
         const subscribeLogger = {
           debug: (msg, meta) => logger.debug(msg, meta),
@@ -27998,7 +28164,7 @@ var require_shuttle = __commonJS({
           const runOn = async (executor) => {
             await observerChain.notify({ kind: "executor.dispatch.started", jobId, attendanceId, executor: executor.capability.id, strategy }, onObserverError);
             const dispatchStartedAt = Date.now();
-            const outcome = await (0, run_dispatch_1.runDispatch)(dispatch, executor, {
+            const ran = await (0, run_dispatch_1.runDispatch)(dispatch, executor, {
               credentials: this.broker,
               signal,
               reportProgress: (channel, frames) => {
@@ -28006,6 +28172,8 @@ var require_shuttle = __commonJS({
                 });
               }
             });
+            subscribeLogger.info(`Job ${jobId} ran on ${executor.capability.id} for ${formatDuration(Date.now() - dispatchStartedAt)}: ` + (ran.kind === "completed" ? "finished" : ran.kind === "gate-failed" ? "its output failed a gate" : "the executor failed"), { jobId, ...signal.aborted ? { aborted: true } : {} });
+            const outcome = ran;
             if (outcome.kind === "executor-failed") {
               const err = outcome.error;
               const reason = `executor failed: ${err instanceof Error ? err.message : String(err)}`;
@@ -28165,14 +28333,21 @@ var require_shuttle = __commonJS({
         logger.info(`${this.brand.product.title} stopping...`);
         this.stopping.abort();
         if (this.subscription) {
-          await this.subscription.stop();
+          await this.subscription.stop({
+            handBack: `The ${this.brand.product.title} was stopped before it finished this job`
+          });
         }
         logger.info(`${this.brand.product.title} stopped`);
         this.lifetime.resolve();
       }
       /** Forward the current subscribe session's fatal, if one ends it, to the run's outcome. */
       watchSession() {
-        this.subscription?.done().catch((err) => this.lifetime.reject(err));
+        this.subscription?.done().catch((err) => {
+          (0, logger_1.getLogger)().error(`${this.brand.product.title} stopped on a fatal error`, {
+            error: err instanceof Error ? `${err.name}: ${err.message}` : String(err)
+          });
+          this.lifetime.reject(err);
+        });
       }
       /**
        * Note that an executor cannot run jobs on this machine, and keep checking
@@ -28336,29 +28511,49 @@ var require_shutdown = __commonJS({
   "../../packages/shuttle/dist/lifecycle/shutdown.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
+    exports2.SHUTDOWN_DEADLINE_MS = void 0;
     exports2.setupShutdownHandler = setupShutdownHandler;
+    exports2.createShutdownHandler = createShutdownHandler;
     var logger_1 = require_logger();
+    exports2.SHUTDOWN_DEADLINE_MS = 2e4;
     function setupShutdownHandler(onShutdown) {
+      const handler = createShutdownHandler(onShutdown);
+      process.on("SIGINT", () => handler("SIGINT"));
+      process.on("SIGTERM", () => handler("SIGTERM"));
+      process.on("SIGHUP", () => handler("SIGHUP"));
+    }
+    function createShutdownHandler(onShutdown, options = {}) {
+      const deadlineMs = options.deadlineMs ?? exports2.SHUTDOWN_DEADLINE_MS;
+      const exit = options.exit ?? ((code) => process.exit(code));
       let shuttingDown = false;
       const handler = (signal) => {
         const logger = (0, logger_1.getLogger)();
         if (shuttingDown) {
-          logger.warn("Forced shutdown", { signal });
-          process.exit(1);
+          logger.warn("Forced shutdown: exiting now. Jobs releases any claim still held once this shift lapses.", {
+            signal
+          });
+          exit(1);
+          return;
         }
         shuttingDown = true;
-        logger.info("Shutting down...", { signal });
+        logger.info("Shutting down: handing any job in progress back to the queue...", { signal });
+        const deadline = setTimeout(() => {
+          logger.warn(`Shutdown did not finish within ${Math.round(deadlineMs / 1e3)}s; exiting. Jobs releases any claim still held once this shift lapses.`);
+          exit(1);
+        }, deadlineMs);
+        deadline.unref?.();
         onShutdown().then(() => {
-          process.exit(0);
+          clearTimeout(deadline);
+          exit(0);
         }).catch((err) => {
+          clearTimeout(deadline);
           logger.error("Shutdown error", {
             error: err instanceof Error ? err.message : String(err)
           });
-          process.exit(1);
+          exit(1);
         });
       };
-      process.on("SIGINT", () => handler("SIGINT"));
-      process.on("SIGTERM", () => handler("SIGTERM"));
+      return handler;
     }
   }
 });
@@ -29192,7 +29387,7 @@ var import_path = require("path");
 var import_promises = require("fs/promises");
 var import_yaml = __toESM(require_dist3());
 var import_shuttle = __toESM(require_dist5());
-var buildVersion = true ? "0.36.0" : pkg.version;
+var buildVersion = true ? "0.37.0" : pkg.version;
 var AGENT_EXECUTORS = ["claude-code", "codex", "copilot"];
 var sifterBrand = {
   product: {