@whittlelabs/sifter 0.30.8 → 0.30.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +12 -0
  2. package/bin.js +721 -83
  3. package/bin.js.map +4 -4
  4. package/package.json +1 -1
package/bin.js CHANGED
@@ -9832,6 +9832,29 @@ var require_dist = __commonJS({
9832
9832
  }
9833
9833
  });
9834
9834
 
9835
+ // ../../packages/shuttle/dist/executors/types.js
9836
+ var require_types2 = __commonJS({
9837
+ "../../packages/shuttle/dist/executors/types.js"(exports2) {
9838
+ "use strict";
9839
+ Object.defineProperty(exports2, "__esModule", { value: true });
9840
+ exports2.ExecutorEnvironmentError = void 0;
9841
+ exports2.isExecutorEnvironmentError = isExecutorEnvironmentError;
9842
+ var ExecutorEnvironmentError = class extends Error {
9843
+ /** What the operator should do, as one clause: "run `claude` and sign in". */
9844
+ remedy;
9845
+ constructor(message, remedy) {
9846
+ super(message);
9847
+ this.name = "ExecutorEnvironmentError";
9848
+ this.remedy = remedy;
9849
+ }
9850
+ };
9851
+ exports2.ExecutorEnvironmentError = ExecutorEnvironmentError;
9852
+ function isExecutorEnvironmentError(err) {
9853
+ return err instanceof Error && err.name === "ExecutorEnvironmentError";
9854
+ }
9855
+ }
9856
+ });
9857
+
9835
9858
  // ../../packages/shuttle/dist/executors/validate-output.js
9836
9859
  var require_validate_output = __commonJS({
9837
9860
  "../../packages/shuttle/dist/executors/validate-output.js"(exports2) {
@@ -10005,6 +10028,7 @@ var require_claude_code = __commonJS({
10005
10028
  return mod && mod.__esModule ? mod : { "default": mod };
10006
10029
  };
10007
10030
  Object.defineProperty(exports2, "__esModule", { value: true });
10031
+ exports2.classifyEnvironmentFault = classifyEnvironmentFault;
10008
10032
  exports2.createClaudeCodeExecutor = createClaudeCodeExecutor;
10009
10033
  exports2.parseStreamJson = parseStreamJson;
10010
10034
  exports2.relativeWorkspaceReads = relativeWorkspaceReads;
@@ -10017,6 +10041,7 @@ var require_claude_code = __commonJS({
10017
10041
  var path_1 = __importDefault(require("path"));
10018
10042
  var string_decoder_1 = require("string_decoder");
10019
10043
  var loom_1 = require_dist();
10044
+ var types_1 = require_types2();
10020
10045
  var validate_output_1 = require_validate_output();
10021
10046
  var progress_1 = require_progress();
10022
10047
  var apply_1 = require_apply();
@@ -10030,6 +10055,39 @@ var require_claude_code = __commonJS({
10030
10055
  const subtype = meta.subtype && meta.subtype !== "success" ? `: ${meta.subtype}` : "";
10031
10056
  return `${fallback}${subtype}${status}`;
10032
10057
  }
10058
+ var READINESS_PROMPT = "Reply with the single word: ok";
10059
+ var READINESS_TIMEOUT_MS = 12e4;
10060
+ var STDERR_LIMIT = 4e3;
10061
+ var INSTALL_REMEDY = "install Claude Code (npm install -g @anthropic-ai/claude-code) where this process can find it on PATH";
10062
+ var LOGIN_REMEDY = "run `claude` once and sign in (or set a valid ANTHROPIC_API_KEY)";
10063
+ var LIMIT_REMEDY = "wait for the plan's usage limit to reset, or raise it";
10064
+ var SIGNED_OUT = /invalid api key|please run \/login|not (?:logged|signed) in|oauth token (?:has )?expired|authentication_error|invalid x-api-key/i;
10065
+ var OUT_OF_USAGE = /usage limit|credit balance is too low|out of (?:extra )?usage|hit your (?:\w+ )?(?:usage |spend )?limit|5-hour limit/i;
10066
+ function classifyEnvironmentFault(failure) {
10067
+ if (failure.spawnErrorCode === "ENOENT")
10068
+ return INSTALL_REMEDY;
10069
+ const text = failure.text ?? "";
10070
+ if (failure.apiErrorStatus === 401 || SIGNED_OUT.test(text))
10071
+ return LOGIN_REMEDY;
10072
+ if (failure.apiErrorStatus === 402 || failure.apiErrorStatus === 429 || OUT_OF_USAGE.test(text)) {
10073
+ return LIMIT_REMEDY;
10074
+ }
10075
+ return null;
10076
+ }
10077
+ function firstLine(text) {
10078
+ for (const line of text.split("\n")) {
10079
+ const trimmed = line.trim();
10080
+ if (trimmed.length > 0)
10081
+ return trimmed.length > 300 ? `${trimmed.slice(0, 300)}...` : trimmed;
10082
+ }
10083
+ return null;
10084
+ }
10085
+ function runFailure(local, message, facts) {
10086
+ if (!local)
10087
+ return new Error(message);
10088
+ const remedy = classifyEnvironmentFault(facts);
10089
+ return remedy ? new types_1.ExecutorEnvironmentError(message, remedy) : new Error(message);
10090
+ }
10033
10091
  var ClaudeCodeExecutor = class {
10034
10092
  capability;
10035
10093
  strategies = [loom_1.PROMPT_EXECUTION_STRATEGY];
@@ -10094,7 +10152,21 @@ var require_claude_code = __commonJS({
10094
10152
  promptOnlyScratch = await fs_1.promises.mkdtemp(path_1.default.join(tmpBase, "shuttle-cc-run-"));
10095
10153
  }
10096
10154
  const runCwd = workspace?.cwd ?? promptOnlyScratch;
10097
- const { response, readPaths, toolInputs } = await this.runClaude(prompt, spec, runCwd, childEnv, signal, hintedModel, reporter, void 0, dispatch.job.id);
10155
+ const { response, readPaths, toolInputs } = await this.runClaude(
10156
+ prompt,
10157
+ spec,
10158
+ runCwd,
10159
+ childEnv,
10160
+ signal,
10161
+ hintedModel,
10162
+ reporter,
10163
+ void 0,
10164
+ dispatch.job.id,
10165
+ // Local mode is the operator's own login: a failure of THAT is the
10166
+ // machine's, and the runner pauses on it. With a revealed key it is one
10167
+ // customer's, and only their job fails.
10168
+ modelKey === null
10169
+ );
10098
10170
  if (workspace) {
10099
10171
  const reads = [...readPaths];
10100
10172
  for (const rel of creditOverlayReads(toolInputs, workspace.overlay?.writtenPaths ?? [], runCwd)) {
@@ -10156,14 +10228,84 @@ var require_claude_code = __commonJS({
10156
10228
  outputLabel: handle.outputLabel,
10157
10229
  outputSchema: handle.outputSchema
10158
10230
  };
10159
- const { response } = await this.runClaude(followUp, spec, scratch, handle.childEnv, signal, handle.model, null, handle.sessionId, `${handle.sessionId}-repair`);
10231
+ const { response } = await this.runClaude(followUp, spec, scratch, handle.childEnv, signal, handle.model, null, handle.sessionId, `${handle.sessionId}-repair`, handle.childEnv === void 0);
10160
10232
  return response;
10161
10233
  } finally {
10162
10234
  await fs_1.promises.rm(scratch, { recursive: true, force: true }).catch(() => {
10163
10235
  });
10164
10236
  }
10165
10237
  }
10166
- runClaude(prompt, spec, cwd, childEnv, signal, modelOverride, reporter, resumeSessionId, transcriptName) {
10238
+ /**
10239
+ * Can this machine run a job now? One real one-turn call through the CLI,
10240
+ * the same probe `doctor --deep` makes. The runner asks it only while
10241
+ * claiming is paused on an `ExecutorEnvironmentError`, and only in local
10242
+ * mode. A signed-out or missing CLI fails before any model call, so the
10243
+ * re-check spends nothing until the login works again, and then one turn.
10244
+ */
10245
+ async checkReadiness(signal) {
10246
+ return new Promise((resolve) => {
10247
+ let child;
10248
+ try {
10249
+ child = (0, child_process_1.spawn)("claude", ["--print", "--output-format", "stream-json", "--verbose", "--max-turns", "1", READINESS_PROMPT], { stdio: ["ignore", "pipe", "pipe"] });
10250
+ } catch (err) {
10251
+ resolve({ ready: false, reason: err instanceof Error ? err.message : String(err) });
10252
+ return;
10253
+ }
10254
+ let stdout = "";
10255
+ let stderr = "";
10256
+ let settled = false;
10257
+ const settle = (value) => {
10258
+ if (settled)
10259
+ return;
10260
+ settled = true;
10261
+ clearTimeout(timer);
10262
+ signal.removeEventListener("abort", onAbort);
10263
+ resolve(value);
10264
+ };
10265
+ const timer = setTimeout(() => {
10266
+ child.kill("SIGKILL");
10267
+ settle({ ready: false, reason: `the claude CLI did not answer within ${READINESS_TIMEOUT_MS / 1e3}s` });
10268
+ }, READINESS_TIMEOUT_MS);
10269
+ function onAbort() {
10270
+ child.kill("SIGTERM");
10271
+ settle({ ready: false, reason: "readiness check aborted" });
10272
+ }
10273
+ if (signal.aborted) {
10274
+ onAbort();
10275
+ return;
10276
+ }
10277
+ signal.addEventListener("abort", onAbort, { once: true });
10278
+ child.stdout?.on("data", (chunk) => {
10279
+ stdout += chunk.toString("utf-8");
10280
+ });
10281
+ child.stderr?.on("data", (chunk) => {
10282
+ if (stderr.length < STDERR_LIMIT)
10283
+ stderr += chunk.toString("utf-8");
10284
+ });
10285
+ child.on("error", (err) => {
10286
+ const code = err.code;
10287
+ settle({
10288
+ ready: false,
10289
+ reason: code === "ENOENT" ? `the claude CLI is not on this machine's PATH (${err.message})` : err.message
10290
+ });
10291
+ });
10292
+ child.on("close", (code) => {
10293
+ let meta = null;
10294
+ try {
10295
+ meta = parseStreamJson(stdout).resultMeta;
10296
+ } catch {
10297
+ meta = null;
10298
+ }
10299
+ if (code === 0 && !meta?.isError) {
10300
+ settle({ ready: true });
10301
+ return;
10302
+ }
10303
+ const said = meta?.detail ?? firstLine(stderr) ?? firstLine(stdout);
10304
+ settle({ ready: false, reason: said ?? `the claude CLI exited with code ${code}` });
10305
+ });
10306
+ });
10307
+ }
10308
+ runClaude(prompt, spec, cwd, childEnv, signal, modelOverride, reporter, resumeSessionId, transcriptName, local = false) {
10167
10309
  const outputLabel = spec.outputLabel;
10168
10310
  const model = modelOverride ?? this.config.model;
10169
10311
  return new Promise((resolve, reject) => {
@@ -10194,6 +10336,7 @@ var require_claude_code = __commonJS({
10194
10336
  return;
10195
10337
  }
10196
10338
  const stdoutChunks = [];
10339
+ let stderrText = "";
10197
10340
  let settled = false;
10198
10341
  const settle = (fn) => {
10199
10342
  if (settled)
@@ -10228,6 +10371,10 @@ var require_claude_code = __commonJS({
10228
10371
  if (tap && decoder)
10229
10372
  tap.push(decoder.write(chunk));
10230
10373
  });
10374
+ child.stderr?.on("data", (chunk) => {
10375
+ if (stderrText.length < STDERR_LIMIT)
10376
+ stderrText += chunk.toString("utf-8");
10377
+ });
10231
10378
  function onAbort() {
10232
10379
  child.kill("SIGTERM");
10233
10380
  settle(() => reject(new Error("Dispatch aborted")));
@@ -10242,6 +10389,11 @@ var require_claude_code = __commonJS({
10242
10389
  child.on("error", (err) => {
10243
10390
  clearTimeout(timer);
10244
10391
  signal.removeEventListener("abort", onAbort);
10392
+ const code = err.code;
10393
+ if (local && code === "ENOENT") {
10394
+ settle(() => reject(new types_1.ExecutorEnvironmentError(`the claude CLI is not on this machine's PATH (${err.message})`, INSTALL_REMEDY)));
10395
+ return;
10396
+ }
10245
10397
  settle(() => reject(err));
10246
10398
  });
10247
10399
  child.on("close", (code) => {
@@ -10259,7 +10411,10 @@ var require_claude_code = __commonJS({
10259
10411
  try {
10260
10412
  const stream = parseStreamJson(stdout);
10261
10413
  if (stream.resultMeta?.isError) {
10262
- throw new Error(describeClaudeFailure(stream.resultMeta, "Claude Code reported a failure"));
10414
+ throw runFailure(local, describeClaudeFailure(stream.resultMeta, "Claude Code reported a failure"), {
10415
+ apiErrorStatus: stream.resultMeta.apiErrorStatus,
10416
+ text: stream.resultMeta.detail ?? firstLine(stderrText)
10417
+ });
10263
10418
  }
10264
10419
  const answerText = stream.finalText ?? stdout;
10265
10420
  const parsed = stream.structuredOutput ?? extractJsonObject(answerText);
@@ -10301,7 +10456,17 @@ var require_claude_code = __commonJS({
10301
10456
  meta = parseStreamJson(stdout).resultMeta;
10302
10457
  } catch {
10303
10458
  }
10304
- settle(() => reject(new Error(describeClaudeFailure(meta, `Claude Code exited with code ${code}`))));
10459
+ const said = meta?.detail ?? firstLine(stderrText) ?? firstLine(stdout);
10460
+ const fallback = `Claude Code exited with code ${code}`;
10461
+ const failure = runFailure(local, describeClaudeFailure(meta, fallback), {
10462
+ apiErrorStatus: meta?.apiErrorStatus ?? null,
10463
+ text: said
10464
+ });
10465
+ if (failure instanceof types_1.ExecutorEnvironmentError && !meta && said) {
10466
+ settle(() => reject(new types_1.ExecutorEnvironmentError(`${fallback}: ${said}`, failure.remedy)));
10467
+ return;
10468
+ }
10469
+ settle(() => reject(failure));
10305
10470
  }
10306
10471
  });
10307
10472
  });
@@ -11109,7 +11274,7 @@ var require_agent_token = __commonJS({
11109
11274
  "../../packages/keep/dist/agent-token.js"(exports2) {
11110
11275
  "use strict";
11111
11276
  Object.defineProperty(exports2, "__esModule", { value: true });
11112
- exports2.AgentTokenManager = exports2.AgentCredentialRevokedError = void 0;
11277
+ exports2.AgentTokenManager = exports2.KeepUnavailableError = exports2.AgentCredentialRevokedError = void 0;
11113
11278
  var AgentCredentialRevokedError2 = class extends Error {
11114
11279
  code = "PAIRING_REVOKED";
11115
11280
  constructor(message) {
@@ -11118,6 +11283,20 @@ var require_agent_token = __commonJS({
11118
11283
  }
11119
11284
  };
11120
11285
  exports2.AgentCredentialRevokedError = AgentCredentialRevokedError2;
11286
+ var KeepUnavailableError = class extends Error {
11287
+ code = "SERVICE_UNAVAILABLE";
11288
+ /** Host the exchange was sent to, so a log line names the service that is down. */
11289
+ host;
11290
+ /** The status whatever answered sent, or null when nothing answered at all. */
11291
+ statusCode;
11292
+ constructor(message, detail) {
11293
+ super(message, detail.cause !== void 0 ? { cause: detail.cause } : void 0);
11294
+ this.name = "KeepUnavailableError";
11295
+ this.host = detail.host;
11296
+ this.statusCode = detail.statusCode;
11297
+ }
11298
+ };
11299
+ exports2.KeepUnavailableError = KeepUnavailableError;
11121
11300
  var AgentTokenManager = class {
11122
11301
  token = null;
11123
11302
  expiresAt = 0;
@@ -11152,16 +11331,35 @@ var require_agent_token = __commonJS({
11152
11331
  }
11153
11332
  async exchange() {
11154
11333
  const url = `${this.keepApiUrl}/api/agent-tokens`;
11155
- const response = await fetch(url, {
11156
- method: "POST",
11157
- headers: { "Content-Type": "application/json" },
11158
- body: JSON.stringify({
11159
- clientId: this.clientId,
11160
- clientSecret: this.clientSecret
11161
- })
11162
- });
11334
+ const host = hostOf(url);
11335
+ let response;
11336
+ try {
11337
+ response = await fetch(url, {
11338
+ method: "POST",
11339
+ headers: { "Content-Type": "application/json" },
11340
+ body: JSON.stringify({
11341
+ clientId: this.clientId,
11342
+ clientSecret: this.clientSecret
11343
+ })
11344
+ });
11345
+ } catch (err) {
11346
+ throw new KeepUnavailableError(`[AgentTokenManager] could not reach Keep at ${host}: ${describeTransportError(err)}`, { host, statusCode: null, cause: err });
11347
+ }
11163
11348
  this.onResponse?.(response);
11164
- const json = await response.json();
11349
+ const text = await response.text().catch(() => "");
11350
+ let json;
11351
+ try {
11352
+ json = JSON.parse(text);
11353
+ } catch {
11354
+ json = null;
11355
+ }
11356
+ if (response.status >= 500) {
11357
+ const reason = json?.error?.message ?? `Agent token exchange failed with status ${response.status}`;
11358
+ throw new KeepUnavailableError(`[AgentTokenManager] Keep at ${host} is unavailable (${response.status}): ${reason}`, { host, statusCode: response.status });
11359
+ }
11360
+ if (json === null) {
11361
+ throw new Error(`[AgentTokenManager] Keep at ${host} answered ${response.status} with a body that is not JSON`);
11362
+ }
11165
11363
  if (!response.ok || !json.success) {
11166
11364
  const msg = json.error?.message ?? `Agent token exchange failed with status ${response.status}`;
11167
11365
  if (response.status === 401) {
@@ -11175,6 +11373,23 @@ var require_agent_token = __commonJS({
11175
11373
  }
11176
11374
  };
11177
11375
  exports2.AgentTokenManager = AgentTokenManager;
11376
+ function hostOf(url) {
11377
+ try {
11378
+ return new URL(url).host;
11379
+ } catch {
11380
+ return url;
11381
+ }
11382
+ }
11383
+ function describeTransportError(err) {
11384
+ if (err instanceof Error) {
11385
+ const cause = err.cause;
11386
+ if (cause instanceof Error && cause.message && cause.message !== err.message) {
11387
+ return `${err.message}: ${cause.message}`;
11388
+ }
11389
+ return err.message;
11390
+ }
11391
+ return String(err);
11392
+ }
11178
11393
  }
11179
11394
  });
11180
11395
 
@@ -11183,7 +11398,7 @@ var require_dist2 = __commonJS({
11183
11398
  "../../packages/keep/dist/index.js"(exports2) {
11184
11399
  "use strict";
11185
11400
  Object.defineProperty(exports2, "__esModule", { value: true });
11186
- exports2.AgentCredentialRevokedError = exports2.AgentTokenManager = exports2.ServiceTokenManager = exports2.KeepClient = void 0;
11401
+ exports2.KeepUnavailableError = exports2.AgentCredentialRevokedError = exports2.AgentTokenManager = exports2.ServiceTokenManager = exports2.KeepClient = void 0;
11187
11402
  var client_1 = require_client();
11188
11403
  Object.defineProperty(exports2, "KeepClient", { enumerable: true, get: function() {
11189
11404
  return client_1.KeepClient;
@@ -11199,6 +11414,9 @@ var require_dist2 = __commonJS({
11199
11414
  Object.defineProperty(exports2, "AgentCredentialRevokedError", { enumerable: true, get: function() {
11200
11415
  return agent_token_1.AgentCredentialRevokedError;
11201
11416
  } });
11417
+ Object.defineProperty(exports2, "KeepUnavailableError", { enumerable: true, get: function() {
11418
+ return agent_token_1.KeepUnavailableError;
11419
+ } });
11202
11420
  }
11203
11421
  });
11204
11422
 
@@ -12064,6 +12282,9 @@ var require_checks = __commonJS({
12064
12282
  return skip("pairing-live", "no readable pairing; see pairing-present");
12065
12283
  }
12066
12284
  const minted = await probes.mintAgentToken(stored.value);
12285
+ if (!minted.ok && minted.unavailable) {
12286
+ return warn("pairing-live", `Keep did not answer, so the pairing could not be checked: ${oneLine(minted.error).replace(/^\[AgentTokenManager\]\s*/, "")}`, subject.phase === "run" ? "nothing to do if this is a deployment window; `run` waits for Keep and starts claiming when it answers" : `check ${stored.value.keepApiUrl} is reachable, then run doctor again`);
12287
+ }
12067
12288
  if (!minted.ok) {
12068
12289
  return fail("pairing-live", `Keep did not issue a token: ${oneLine(minted.error)}`, `check ${stored.value.keepApiUrl} is reachable, then run \`${subject.brand.product.cliBinary} init\` if the pairing was revoked`);
12069
12290
  }
@@ -20092,6 +20313,27 @@ var require_subscribe = __commonJS({
20092
20313
  function sleep(ms) {
20093
20314
  return new Promise((resolve) => setTimeout(resolve, ms));
20094
20315
  }
20316
+ function isUnavailable(err) {
20317
+ if (err instanceof response_1.ServiceUnavailableError)
20318
+ return true;
20319
+ if (err?.code === "SERVICE_UNAVAILABLE")
20320
+ return true;
20321
+ return err instanceof TypeError && err.message === "fetch failed";
20322
+ }
20323
+ function describeOutage(err) {
20324
+ const e = err ?? {};
20325
+ if (typeof e.host === "string" && typeof e.statusCode === "number") {
20326
+ const text = typeof e.statusText === "string" && e.statusText ? ` ${e.statusText}` : "";
20327
+ return `${e.host} answered ${e.statusCode}${text}`;
20328
+ }
20329
+ if (typeof e.host === "string")
20330
+ return `${e.host} did not answer`;
20331
+ if (err instanceof Error) {
20332
+ const cause = err.cause;
20333
+ return cause instanceof Error ? `${err.message}: ${cause.message}` : err.message;
20334
+ }
20335
+ return String(err);
20336
+ }
20095
20337
  function formatDuration(ms) {
20096
20338
  const seconds = Math.round(ms / 1e3);
20097
20339
  if (seconds < 60)
@@ -20100,105 +20342,228 @@ var require_subscribe = __commonJS({
20100
20342
  const rest = seconds % 60;
20101
20343
  return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`;
20102
20344
  }
20345
+ function newOutcome() {
20346
+ let resolve;
20347
+ let reject;
20348
+ const promise = new Promise((res, rej) => {
20349
+ resolve = res;
20350
+ reject = rej;
20351
+ });
20352
+ promise.catch(() => {
20353
+ });
20354
+ const outcome = {
20355
+ promise,
20356
+ settled: false,
20357
+ resolve: () => {
20358
+ if (outcome.settled)
20359
+ return;
20360
+ outcome.settled = true;
20361
+ resolve();
20362
+ },
20363
+ reject: (err) => {
20364
+ if (outcome.settled)
20365
+ return;
20366
+ outcome.settled = true;
20367
+ reject(err);
20368
+ }
20369
+ };
20370
+ return outcome;
20371
+ }
20372
+ var STOPPED = Symbol("stopped");
20103
20373
  function subscribe(opts) {
20104
20374
  const logger = opts.logger ?? NULL_LOGGER;
20105
20375
  const pollIntervalMs = opts.pollIntervalMs ?? 2e3;
20106
20376
  const maxConcurrency = opts.maxConcurrency ?? 1;
20377
+ const isFatal = opts.isFatal ?? (() => false);
20107
20378
  let stopped = true;
20379
+ let session = 0;
20108
20380
  let pollTimer = null;
20109
20381
  let inflight = 0;
20110
20382
  let resolvedPoolIds = [];
20383
+ const controllers = /* @__PURE__ */ new Set();
20384
+ const holds = /* @__PURE__ */ new Set();
20385
+ let fatalError = null;
20386
+ let outcome = newOutcome();
20387
+ let wakeStartup = null;
20111
20388
  let unavailableSince = null;
20389
+ function noteReachable() {
20390
+ if (unavailableSince !== null) {
20391
+ logger.info(`Sift is reachable again after ${formatDuration(Date.now() - unavailableSince)}`);
20392
+ unavailableSince = null;
20393
+ }
20394
+ }
20395
+ function noteUnavailable(err) {
20396
+ if (unavailableSince === null) {
20397
+ unavailableSince = Date.now();
20398
+ logger.warn(`Sift is unavailable (${describeOutage(err)}). This is typical during a deployment window; the Sifter will keep trying.`);
20399
+ } else {
20400
+ const e = err ?? {};
20401
+ logger.debug?.("Sift still unavailable", {
20402
+ ...typeof e.host === "string" ? { host: e.host } : {},
20403
+ ...typeof e.statusCode === "number" ? { status: e.statusCode } : {},
20404
+ unavailableFor: formatDuration(Date.now() - unavailableSince)
20405
+ });
20406
+ }
20407
+ }
20408
+ function startupPause(ms) {
20409
+ return new Promise((resolve) => {
20410
+ const timer = setTimeout(() => {
20411
+ wakeStartup = null;
20412
+ resolve();
20413
+ }, ms);
20414
+ wakeStartup = () => {
20415
+ clearTimeout(timer);
20416
+ wakeStartup = null;
20417
+ resolve();
20418
+ };
20419
+ });
20420
+ }
20421
+ async function untilAvailable(call, mySession) {
20422
+ for (; ; ) {
20423
+ try {
20424
+ const value = await call();
20425
+ noteReachable();
20426
+ return value;
20427
+ } catch (err) {
20428
+ if (isFatal(err) || !isUnavailable(err))
20429
+ throw err;
20430
+ noteUnavailable(err);
20431
+ }
20432
+ await startupPause(pollIntervalMs);
20433
+ if (stopped || mySession !== session)
20434
+ return STOPPED;
20435
+ }
20436
+ }
20112
20437
  async function start() {
20113
20438
  if (!stopped)
20114
20439
  return;
20115
20440
  stopped = false;
20441
+ session += 1;
20442
+ const mySession = session;
20443
+ if (outcome.settled)
20444
+ outcome = newOutcome();
20445
+ fatalError = null;
20116
20446
  resolvedPoolIds = [];
20117
- for (const ref of opts.pools) {
20118
- const id = await opts.jobsClient.resolvePoolRef(ref);
20119
- resolvedPoolIds.push(id);
20120
- await opts.jobsClient.registerMember(id, {
20121
- identityType: opts.subscriber.identityType,
20122
- identityId: opts.subscriber.identityId,
20123
- intelligence: opts.subscriber.intelligence,
20124
- ...opts.subscriber.displayName !== void 0 ? { displayName: opts.subscriber.displayName } : {},
20125
- ...opts.subscriber.metadata !== void 0 ? { metadata: opts.subscriber.metadata } : {}
20126
- });
20447
+ try {
20448
+ for (const ref of opts.pools) {
20449
+ const id = await untilAvailable(() => opts.jobsClient.resolvePoolRef(ref), mySession);
20450
+ if (id === STOPPED)
20451
+ return;
20452
+ resolvedPoolIds.push(id);
20453
+ const registered = await untilAvailable(() => opts.jobsClient.registerMember(id, {
20454
+ identityType: opts.subscriber.identityType,
20455
+ identityId: opts.subscriber.identityId,
20456
+ intelligence: opts.subscriber.intelligence,
20457
+ ...opts.subscriber.displayName !== void 0 ? { displayName: opts.subscriber.displayName } : {},
20458
+ ...opts.subscriber.metadata !== void 0 ? { metadata: opts.subscriber.metadata } : {}
20459
+ }), mySession);
20460
+ if (registered === STOPPED)
20461
+ return;
20462
+ }
20463
+ } catch (err) {
20464
+ if (mySession === session) {
20465
+ stopped = true;
20466
+ outcome.reject(err);
20467
+ }
20468
+ throw err;
20127
20469
  }
20470
+ if (stopped || mySession !== session)
20471
+ return;
20128
20472
  logger.info("subscribe loop started", { pools: resolvedPoolIds, maxConcurrency });
20129
20473
  schedulePoll(pollIntervalMs);
20130
20474
  }
20131
- async function stop() {
20132
- if (stopped)
20133
- return;
20475
+ async function finish() {
20476
+ while (inflight > 0) {
20477
+ await sleep(100);
20478
+ }
20479
+ logger.info("subscribe loop stopped");
20480
+ if (fatalError !== null)
20481
+ outcome.reject(fatalError.error);
20482
+ else
20483
+ outcome.resolve();
20484
+ }
20485
+ function halt() {
20134
20486
  stopped = true;
20135
20487
  if (pollTimer !== null) {
20136
20488
  clearTimeout(pollTimer);
20137
20489
  pollTimer = null;
20138
20490
  }
20139
- while (inflight > 0) {
20140
- await sleep(100);
20141
- }
20142
- logger.info("subscribe loop stopped");
20491
+ wakeStartup?.();
20492
+ }
20493
+ async function stop() {
20494
+ if (stopped)
20495
+ return;
20496
+ halt();
20497
+ await finish();
20498
+ }
20499
+ async function fail(err) {
20500
+ if (fatalError === null)
20501
+ fatalError = { error: err };
20502
+ for (const controller of controllers)
20503
+ controller.abort(err);
20504
+ if (stopped)
20505
+ return;
20506
+ halt();
20507
+ await finish();
20143
20508
  }
20144
20509
  function schedulePoll(delayMs) {
20510
+ const mySession = session;
20145
20511
  pollTimer = setTimeout(() => {
20146
- void poll();
20512
+ void poll(mySession);
20147
20513
  }, delayMs);
20148
20514
  }
20149
- async function poll() {
20150
- if (stopped)
20515
+ async function poll(mySession) {
20516
+ if (stopped || mySession !== session)
20151
20517
  return;
20152
20518
  if (inflight >= maxConcurrency) {
20153
20519
  schedulePoll(pollIntervalMs);
20154
20520
  return;
20155
20521
  }
20156
20522
  try {
20523
+ const paused = holds.size > 0;
20157
20524
  const { data, directives } = await opts.jobsClient.listDispatchableJobs({
20158
20525
  pools: resolvedPoolIds,
20159
- limit: maxConcurrency - inflight,
20526
+ limit: paused ? 1 : maxConcurrency - inflight,
20160
20527
  sort: "priority:desc,queued_at:asc"
20161
20528
  });
20162
- if (unavailableSince !== null) {
20163
- logger.info(`Sift is reachable again after ${formatDuration(Date.now() - unavailableSince)}`);
20164
- unavailableSince = null;
20165
- }
20529
+ noteReachable();
20166
20530
  if (directives && Object.keys(directives).length > 0 && opts.onDirectives) {
20167
20531
  try {
20168
20532
  await opts.onDirectives(directives);
20169
20533
  } catch (err) {
20534
+ if (isFatal(err)) {
20535
+ void fail(err);
20536
+ return;
20537
+ }
20170
20538
  logger.error("onDirectives handler failed", {
20171
20539
  error: err instanceof Error ? err.message : String(err)
20172
20540
  });
20173
20541
  }
20174
20542
  }
20175
- if (stopped)
20543
+ if (stopped || mySession !== session)
20176
20544
  return;
20177
- for (const job of data) {
20178
- if (inflight >= maxConcurrency)
20179
- break;
20180
- inflight++;
20181
- void runOnce(job).finally(() => {
20182
- inflight--;
20183
- });
20184
- }
20185
- } catch (err) {
20186
- if (err instanceof response_1.ServiceUnavailableError) {
20187
- if (unavailableSince === null) {
20188
- unavailableSince = Date.now();
20189
- logger.warn(`Sift is unavailable (${err.host} answered ${err.statusCode}${err.statusText ? ` ${err.statusText}` : ""}). This is typical during a deployment window; the Sifter will keep trying.`);
20190
- } else {
20191
- logger.debug?.("Sift still unavailable", {
20192
- host: err.host,
20193
- status: err.statusCode,
20194
- unavailableFor: formatDuration(Date.now() - unavailableSince)
20545
+ if (holds.size === 0) {
20546
+ for (const job of data) {
20547
+ if (inflight >= maxConcurrency)
20548
+ break;
20549
+ inflight++;
20550
+ void runOnce(job).finally(() => {
20551
+ inflight--;
20195
20552
  });
20196
20553
  }
20554
+ }
20555
+ } catch (err) {
20556
+ if (isFatal(err)) {
20557
+ void fail(err);
20558
+ return;
20559
+ }
20560
+ if (isUnavailable(err)) {
20561
+ noteUnavailable(err);
20197
20562
  } else {
20198
20563
  logger.error("poll failed", { error: err instanceof Error ? err.message : String(err) });
20199
20564
  }
20200
20565
  }
20201
- if (!stopped)
20566
+ if (!stopped && mySession === session)
20202
20567
  schedulePoll(pollIntervalMs);
20203
20568
  }
20204
20569
  async function runOnce(available) {
@@ -20206,6 +20571,10 @@ var require_subscribe = __commonJS({
20206
20571
  try {
20207
20572
  dispatch = await opts.jobsClient.acceptDispatch(available.id);
20208
20573
  } catch (err) {
20574
+ if (isFatal(err)) {
20575
+ void fail(err);
20576
+ return;
20577
+ }
20209
20578
  const statusCode = err.statusCode;
20210
20579
  if (statusCode === 409) {
20211
20580
  logger.debug?.("claim conflict; another member accepted", { jobId: available.id });
@@ -20218,6 +20587,9 @@ var require_subscribe = __commonJS({
20218
20587
  return;
20219
20588
  }
20220
20589
  const controller = new AbortController();
20590
+ controllers.add(controller);
20591
+ if (fatalError !== null)
20592
+ controller.abort(fatalError.error);
20221
20593
  let quietTimer = null;
20222
20594
  const quietWindowMs = dispatch.deadline ? new Date(dispatch.deadline).getTime() - Date.now() : null;
20223
20595
  const armQuietTimer = () => {
@@ -20243,6 +20615,8 @@ var require_subscribe = __commonJS({
20243
20615
  try {
20244
20616
  result = await opts.onJob(dispatch, controller.signal, activity);
20245
20617
  } catch (err) {
20618
+ if (isFatal(err))
20619
+ void fail(err);
20246
20620
  result = {
20247
20621
  status: "failed",
20248
20622
  reason: `onJob threw: ${err instanceof Error ? err.message : String(err)}`
@@ -20250,6 +20624,11 @@ var require_subscribe = __commonJS({
20250
20624
  } finally {
20251
20625
  if (quietTimer !== null)
20252
20626
  clearTimeout(quietTimer);
20627
+ controllers.delete(controller);
20628
+ }
20629
+ if (fatalError !== null) {
20630
+ logger.debug?.("not submitting a result after a fatal error", { jobId: dispatch.job.id });
20631
+ return;
20253
20632
  }
20254
20633
  try {
20255
20634
  for (const output of result.outputs ?? []) {
@@ -20263,13 +20642,27 @@ var require_subscribe = __commonJS({
20263
20642
  ...result.reason !== void 0 ? { reason: result.reason } : {}
20264
20643
  });
20265
20644
  } catch (err) {
20645
+ if (isFatal(err)) {
20646
+ void fail(err);
20647
+ return;
20648
+ }
20266
20649
  logger.error("submit/close failed", {
20267
20650
  jobId: dispatch.job.id,
20268
20651
  error: err instanceof Error ? err.message : String(err)
20269
20652
  });
20270
20653
  }
20271
20654
  }
20272
- return { start, stop };
20655
+ return {
20656
+ start,
20657
+ stop,
20658
+ done: () => outcome.promise,
20659
+ pause: (reason) => {
20660
+ holds.add(reason);
20661
+ },
20662
+ resume: (reason) => {
20663
+ holds.delete(reason);
20664
+ }
20665
+ };
20273
20666
  }
20274
20667
  }
20275
20668
  });
@@ -20785,7 +21178,7 @@ var require_probes = __commonJS({
20785
21178
  } catch (err) {
20786
21179
  if (isRevoked(err) || isUpgradeRequired(err))
20787
21180
  throw err;
20788
- return { ok: false, error: describe(err) };
21181
+ return { ok: false, error: describe(err), ...isKeepUnavailable(err) ? { unavailable: true } : {} };
20789
21182
  }
20790
21183
  },
20791
21184
  async reachJobs(config, pairing) {
@@ -20903,6 +21296,9 @@ var require_probes = __commonJS({
20903
21296
  function isRevoked(err) {
20904
21297
  return err instanceof Error && err.name === "AgentCredentialRevokedError";
20905
21298
  }
21299
+ function isKeepUnavailable(err) {
21300
+ return err instanceof Error && err.name === "KeepUnavailableError";
21301
+ }
20906
21302
  function isUpgradeRequired(err) {
20907
21303
  return err instanceof Error && err.name === "UpgradeRequiredError";
20908
21304
  }
@@ -23465,6 +23861,8 @@ var require_self_update = __commonJS({
23465
23861
  Object.defineProperty(exports2, "__esModule", { value: true });
23466
23862
  exports2.readUpgradeDirective = readUpgradeDirective;
23467
23863
  exports2.performSelfUpdate = performSelfUpdate;
23864
+ exports2.floorUpdateTarget = floorUpdateTarget;
23865
+ exports2.updateForFloor = updateForFloor;
23468
23866
  var child_process_1 = require("child_process");
23469
23867
  var version_check_1 = require_version_check();
23470
23868
  function readUpgradeDirective(directives, currentVersion, attemptedVersions) {
@@ -23506,6 +23904,32 @@ var require_self_update = __commonJS({
23506
23904
  respawn();
23507
23905
  return true;
23508
23906
  }
23907
+ function floorUpdateTarget(minVersion, latestVersion) {
23908
+ return latestVersion !== null && (0, version_check_1.compareSemver)(latestVersion, minVersion) >= 0 ? latestVersion : minVersion;
23909
+ }
23910
+ async function updateForFloor(opts) {
23911
+ const { productTitle, packageName, currentVersion, minVersion, hosted, logger, hooks } = opts;
23912
+ const refusal = new version_check_1.UpgradeRequiredError({ currentVersion, minVersion, packageName });
23913
+ const target = floorUpdateTarget(minVersion, opts.latestVersion);
23914
+ logger.warn(`The backend now requires ${packageName} ${minVersion} or newer, and this ${productTitle} is ${currentVersion}. It has stopped claiming and is finishing the work in hand, then ` + (hosted ? `exiting: a hosted ${productTitle} is updated by redeploying its image.` : `updating itself to ${target}.`));
23915
+ await hooks.drain();
23916
+ if (hosted) {
23917
+ hooks.exit(refusal);
23918
+ return;
23919
+ }
23920
+ const spec = `${packageName}@${target}`;
23921
+ try {
23922
+ await (hooks.install ?? defaultInstall)(spec);
23923
+ } catch (err) {
23924
+ logger.error(`could not install ${spec}, so this ${productTitle} cannot continue`, {
23925
+ error: err instanceof Error ? err.message : String(err)
23926
+ });
23927
+ hooks.exit(refusal);
23928
+ return;
23929
+ }
23930
+ logger.info("self-update installed; restarting", { packageSpec: spec });
23931
+ (hooks.respawn ?? defaultRespawn)();
23932
+ }
23509
23933
  function defaultInstall(packageSpec) {
23510
23934
  return new Promise((resolve, reject) => {
23511
23935
  const child = (0, child_process_1.spawn)("npm", ["install", "-g", packageSpec], {
@@ -24954,12 +25378,14 @@ var require_shuttle = __commonJS({
24954
25378
  var decommission_1 = require_decommission();
24955
25379
  var loom_1 = require_dist();
24956
25380
  var chain_1 = require_chain();
25381
+ var types_1 = require_types2();
24957
25382
  var workspace_1 = require_workspace();
24958
25383
  var credentials_1 = require_credentials();
24959
25384
  var output_gates_1 = require_output_gates();
24960
25385
  var keep_1 = require_dist2();
24961
25386
  var apply_1 = require_apply();
24962
25387
  var version_check_1 = require_version_check();
25388
+ var typed_exit_1 = require_typed_exit();
24963
25389
  var logger_1 = require_logger();
24964
25390
  var registry_1 = require_registry();
24965
25391
  var claude_code_1 = require_claude_code();
@@ -24972,6 +25398,50 @@ var require_shuttle = __commonJS({
24972
25398
  var spend_tracker_1 = require_spend_tracker();
24973
25399
  var store_1 = require_store();
24974
25400
  var GATE_REPAIR_BUDGET = 2;
25401
+ var ENVIRONMENT_HOLD = "executor-environment";
25402
+ var RECHECK_INITIAL_MS = 15e3;
25403
+ var RECHECK_MAX_MS = 3e5;
25404
+ var KEEP_RETRY_INITIAL_MS = 2e3;
25405
+ var KEEP_RETRY_MAX_MS = 3e4;
25406
+ function formatDuration(ms) {
25407
+ const seconds = Math.round(ms / 1e3);
25408
+ if (seconds < 60)
25409
+ return `${seconds}s`;
25410
+ const minutes = Math.floor(seconds / 60);
25411
+ const rest = seconds % 60;
25412
+ return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`;
25413
+ }
25414
+ function abortableSleep(ms, signal) {
25415
+ return new Promise((resolve) => {
25416
+ if (signal?.aborted) {
25417
+ resolve();
25418
+ return;
25419
+ }
25420
+ const timer = setTimeout(() => {
25421
+ signal?.removeEventListener("abort", onAbort);
25422
+ resolve();
25423
+ }, ms);
25424
+ function onAbort() {
25425
+ clearTimeout(timer);
25426
+ resolve();
25427
+ }
25428
+ signal?.addEventListener("abort", onAbort, { once: true });
25429
+ });
25430
+ }
25431
+ function deferred() {
25432
+ let resolve;
25433
+ let reject;
25434
+ const promise = new Promise((res, rej) => {
25435
+ resolve = res;
25436
+ reject = rej;
25437
+ });
25438
+ promise.catch(() => {
25439
+ });
25440
+ return { promise, resolve, reject };
25441
+ }
25442
+ function isKeepUnavailable(err) {
25443
+ return err instanceof Error && err.name === "KeepUnavailableError";
25444
+ }
24975
25445
  async function runOutputGates(dispatch, response, executor, signal, ctx) {
24976
25446
  let spec;
24977
25447
  try {
@@ -25007,6 +25477,7 @@ var require_shuttle = __commonJS({
25007
25477
  config;
25008
25478
  profile;
25009
25479
  credentialDeps;
25480
+ environmentRecheck;
25010
25481
  subscription = null;
25011
25482
  spendTracker = null;
25012
25483
  /**
@@ -25015,11 +25486,26 @@ var require_shuttle = __commonJS({
25015
25486
  * (no such config) → consumers reveal nothing and use ambient credentials.
25016
25487
  */
25017
25488
  broker = null;
25489
+ /**
25490
+ * The run's outcome, which `done()` hands out. Resolves on `stop()`.
25491
+ * Rejects with what ended the run otherwise: a refused credential
25492
+ * (`AgentCredentialRevokedError`, exit 3) or a floor this build could not
25493
+ * update past (`UpgradeRequiredError`, exit 2).
25494
+ */
25495
+ lifetime = deferred();
25496
+ /** Aborted by `stop()`: ends the startup waits and the readiness re-checks. */
25497
+ stopping = new AbortController();
25498
+ /** When claiming was paused on an executor environment fault; null while claiming. */
25499
+ environmentPausedAt = null;
25018
25500
  constructor(options) {
25019
25501
  this.brand = options.brand;
25020
25502
  this.config = options.config;
25021
25503
  this.profile = options.profile ?? apply_1.DEFAULT_PROFILE;
25022
25504
  this.credentialDeps = options.credentialDeps ?? {};
25505
+ this.environmentRecheck = options.environmentRecheck ?? {
25506
+ initialMs: RECHECK_INITIAL_MS,
25507
+ maxMs: RECHECK_MAX_MS
25508
+ };
25023
25509
  }
25024
25510
  /** Exposed so the CLI's `caps set` can reload after a write. */
25025
25511
  get spendTrackerHandle() {
@@ -25034,19 +25520,62 @@ var require_shuttle = __commonJS({
25034
25520
  logger.info(`${this.brand.product.title} starting...`);
25035
25521
  const paths = (0, apply_1.resolvePaths)(this.brand, this.profile);
25036
25522
  this.broker = (0, credentials_1.createCredentialBroker)(this.config.credentials, this.credentialDeps);
25037
- const enforceMinVersion = (response) => {
25038
- (0, version_check_1.checkResponseMinVersion)({
25039
- response,
25523
+ const subscribeLogger = {
25524
+ debug: (msg, meta) => logger.debug(msg, meta),
25525
+ info: (msg, meta) => logger.info(msg, meta),
25526
+ warn: (msg, meta) => logger.warn(msg, meta),
25527
+ error: (msg, meta) => logger.error(msg, meta)
25528
+ };
25529
+ let running = false;
25530
+ let latestSeen = null;
25531
+ let floorRising = false;
25532
+ const onBackendResponse = (response) => {
25533
+ latestSeen = (0, version_check_1.readResponseLatestVersion)(response) ?? latestSeen;
25534
+ if (!running) {
25535
+ (0, version_check_1.checkResponseMinVersion)({
25536
+ response,
25537
+ currentVersion: this.brand.product.version,
25538
+ packageName: this.brand.product.packageName
25539
+ });
25540
+ return;
25541
+ }
25542
+ if (floorRising)
25543
+ return;
25544
+ const floor = response.headers.get(version_check_1.MIN_VERSION_HEADER);
25545
+ if (!floor || (0, version_check_1.compareSemver)(this.brand.product.version, floor) >= 0)
25546
+ return;
25547
+ floorRising = true;
25548
+ void (0, self_update_1.updateForFloor)({
25549
+ productTitle: this.brand.product.title,
25550
+ packageName: this.brand.product.packageName,
25040
25551
  currentVersion: this.brand.product.version,
25041
- packageName: this.brand.product.packageName
25042
- });
25552
+ minVersion: floor,
25553
+ latestVersion: latestSeen,
25554
+ // A `credentials` block is what makes a deployment hosted (ADR 0027).
25555
+ hosted: this.config.credentials !== void 0,
25556
+ logger: subscribeLogger,
25557
+ hooks: {
25558
+ drain: () => this.subscription?.stop() ?? Promise.resolve(),
25559
+ exit: (err) => this.lifetime.reject(err)
25560
+ }
25561
+ }).catch((err) => this.lifetime.reject(err));
25043
25562
  };
25044
- const auth = await resolveAuth(this.brand, this.config, paths.configFile, enforceMinVersion);
25563
+ let auth;
25564
+ try {
25565
+ auth = await resolveAuth(this.brand, this.config, paths.configFile, onBackendResponse, {
25566
+ signal: this.stopping.signal,
25567
+ logger: subscribeLogger
25568
+ });
25569
+ } catch (err) {
25570
+ if (this.stopping.signal.aborted)
25571
+ return;
25572
+ throw err;
25573
+ }
25045
25574
  const jobsClient = new jobs_1.JobsClient({
25046
25575
  baseUrl: this.config.auth.jobsApiUrl,
25047
25576
  apiKey: auth.kind === "apiKey" ? auth.apiKey : void 0,
25048
25577
  getAccessToken: auth.kind === "tokenProvider" ? auth.getToken : void 0,
25049
- onResponse: enforceMinVersion
25578
+ onResponse: onBackendResponse
25050
25579
  });
25051
25580
  const registry = new registry_1.ExecutorRegistry(this.brand.executorAllowlist);
25052
25581
  registerBuiltInExecutors(registry, this.brand.executorAllowlist);
@@ -25137,6 +25666,9 @@ var require_shuttle = __commonJS({
25137
25666
  response = await executor.execute(dispatch, signal, executionCtx);
25138
25667
  } catch (err) {
25139
25668
  const reason = `executor failed: ${err instanceof Error ? err.message : String(err)}`;
25669
+ if (this.broker === null && (0, types_1.isExecutorEnvironmentError)(err)) {
25670
+ this.pauseForEnvironment(executor, err, subscribeLogger);
25671
+ }
25140
25672
  await observerChain.notify({ kind: "attendance.closed", jobId, attendanceId, status: "failed", reason, totalDurationMs: Date.now() - startedAt }, onObserverError);
25141
25673
  return { status: "failed", reason };
25142
25674
  } finally {
@@ -25168,12 +25700,6 @@ var require_shuttle = __commonJS({
25168
25700
  return { status: "completed", outputs };
25169
25701
  };
25170
25702
  const pools = this.config.pools.map((p) => p.id ? { id: p.id } : { name: p.name });
25171
- const subscribeLogger = {
25172
- debug: (msg, meta) => logger.debug(msg, meta),
25173
- info: (msg, meta) => logger.info(msg, meta),
25174
- warn: (msg, meta) => logger.warn(msg, meta),
25175
- error: (msg, meta) => logger.error(msg, meta)
25176
- };
25177
25703
  const attemptedUpgrades = /* @__PURE__ */ new Set();
25178
25704
  this.subscription = (0, jobs_1.subscribe)({
25179
25705
  jobsClient,
@@ -25206,6 +25732,8 @@ var require_shuttle = __commonJS({
25206
25732
  });
25207
25733
  return;
25208
25734
  }
25735
+ if (floorRising)
25736
+ return;
25209
25737
  const directive = (0, self_update_1.readUpgradeDirective)(directives, this.brand.product.version, attemptedUpgrades);
25210
25738
  if (!directive)
25211
25739
  return;
@@ -25216,15 +25744,33 @@ var require_shuttle = __commonJS({
25216
25744
  logger: subscribeLogger,
25217
25745
  hooks: {
25218
25746
  stopLoop: () => this.subscription?.stop() ?? Promise.resolve(),
25219
- restartLoop: () => this.subscription?.start() ?? Promise.resolve()
25747
+ // The restarted session's outcome needs the same watch as the
25748
+ // first one's, or a fatal after an upgrade that failed to install
25749
+ // would end nobody's wait. A restart that cannot register ends the
25750
+ // run the same way.
25751
+ restartLoop: () => (this.subscription?.start() ?? Promise.resolve()).then(() => this.watchSession()).catch((err) => this.lifetime.reject(err))
25220
25752
  }
25221
25753
  });
25222
25754
  },
25755
+ // The two failures that end a run with their own exit code and banner:
25756
+ // a refused credential (exit 3) and a below-floor build (exit 2),
25757
+ // matched by name as bin.ts's typed exits are. In-flight jobs are
25758
+ // aborted and their results not submitted (see subscribe's isFatal):
25759
+ // with the credential refused nothing can be submitted anyway, and the
25760
+ // claims lapse back to the pool for another Sifter. A floor rise
25761
+ // normally arrives through onBackendResponse, which drains instead; one
25762
+ // reaching this predicate came from somewhere unexpected and is
25763
+ // treated as the refusal it is.
25764
+ isFatal: typed_exit_1.isTypedExit,
25223
25765
  logger: subscribeLogger,
25224
25766
  pollIntervalMs: this.config.polling.intervalMs,
25225
25767
  maxConcurrency: this.config.concurrency.maxJobs
25226
25768
  });
25227
25769
  await this.subscription.start();
25770
+ if (this.stopping.signal.aborted)
25771
+ return;
25772
+ this.watchSession();
25773
+ running = true;
25228
25774
  logger.info(`${this.brand.product.title} running`, {
25229
25775
  profile: this.profile,
25230
25776
  pools: pools.length,
@@ -25233,13 +25779,72 @@ var require_shuttle = __commonJS({
25233
25779
  observers: observers.length
25234
25780
  });
25235
25781
  }
25782
+ /**
25783
+ * The run's outcome: resolves after `stop()`, rejects with the typed error
25784
+ * that ended it otherwise (a refused credential, or a floor this build could
25785
+ * not update past). `run` awaits this after `start()`, so those errors reach
25786
+ * the whitelabel's exit codes and banners instead of dying in the loop.
25787
+ */
25788
+ done() {
25789
+ return this.lifetime.promise;
25790
+ }
25236
25791
  async stop() {
25237
25792
  const logger = (0, logger_1.getLogger)();
25238
25793
  logger.info(`${this.brand.product.title} stopping...`);
25794
+ this.stopping.abort();
25239
25795
  if (this.subscription) {
25240
25796
  await this.subscription.stop();
25241
25797
  }
25242
25798
  logger.info(`${this.brand.product.title} stopped`);
25799
+ this.lifetime.resolve();
25800
+ }
25801
+ /** Forward the current subscribe session's fatal, if one ends it, to the run's outcome. */
25802
+ watchSession() {
25803
+ this.subscription?.done().catch((err) => this.lifetime.reject(err));
25804
+ }
25805
+ /**
25806
+ * Stop claiming because the executor says this machine cannot run jobs,
25807
+ * and keep checking until it can. One warning says why and what to do; the
25808
+ * re-checks stay at debug; one line says when claiming resumes. Idempotent:
25809
+ * a second failing job while paused (maxJobs > 1) changes nothing.
25810
+ */
25811
+ pauseForEnvironment(executor, err, logger) {
25812
+ if (this.environmentPausedAt !== null || !this.subscription)
25813
+ return;
25814
+ this.environmentPausedAt = Date.now();
25815
+ this.subscription.pause(ENVIRONMENT_HOLD);
25816
+ logger.warn(`Claiming paused: ${executor.capability.id} cannot run jobs on this machine (${err.message}). To fix it, ${err.remedy}. The ${this.brand.product.title} checks again on its own and resumes claiming when it works.`);
25817
+ void this.recheckEnvironment(executor, logger);
25818
+ }
25819
+ async recheckEnvironment(executor, logger) {
25820
+ const signal = this.stopping.signal;
25821
+ let delay = this.environmentRecheck.initialMs;
25822
+ for (; ; ) {
25823
+ await abortableSleep(delay, signal);
25824
+ if (signal.aborted)
25825
+ return;
25826
+ let readiness;
25827
+ try {
25828
+ readiness = executor.checkReadiness ? await executor.checkReadiness(signal) : { ready: true };
25829
+ } catch (err) {
25830
+ readiness = { ready: false, reason: err instanceof Error ? err.message : String(err) };
25831
+ }
25832
+ if (signal.aborted)
25833
+ return;
25834
+ if (readiness.ready) {
25835
+ const pausedFor = formatDuration(Date.now() - (this.environmentPausedAt ?? Date.now()));
25836
+ this.environmentPausedAt = null;
25837
+ this.subscription?.resume(ENVIRONMENT_HOLD);
25838
+ logger.info(`${executor.capability.id} can run jobs again; claiming resumed after ${pausedFor} paused.`);
25839
+ return;
25840
+ }
25841
+ delay = Math.min(delay * 2, this.environmentRecheck.maxMs);
25842
+ logger.debug?.("executor still cannot run jobs", {
25843
+ executor: executor.capability.id,
25844
+ reason: readiness.reason,
25845
+ nextCheckIn: formatDuration(delay)
25846
+ });
25847
+ }
25243
25848
  }
25244
25849
  };
25245
25850
  exports2.Shuttle = Shuttle;
@@ -25267,7 +25872,7 @@ var require_shuttle = __commonJS({
25267
25872
  }
25268
25873
  }
25269
25874
  }
25270
- async function resolveAuth(brand, config, pairingFile, onResponse) {
25875
+ async function resolveAuth(brand, config, pairingFile, onResponse, options = {}) {
25271
25876
  const store = new store_1.PairingConfigStore(pairingFile);
25272
25877
  if (store.exists()) {
25273
25878
  const pairing = store.read();
@@ -25277,7 +25882,7 @@ var require_shuttle = __commonJS({
25277
25882
  clientSecret: pairing.credentials.clientSecret,
25278
25883
  onResponse
25279
25884
  });
25280
- const token = await tokens.getToken();
25885
+ const token = await mintWaitingOutKeep(() => tokens.getToken(), brand, options);
25281
25886
  if (token) {
25282
25887
  return {
25283
25888
  kind: "tokenProvider",
@@ -25290,6 +25895,34 @@ var require_shuttle = __commonJS({
25290
25895
  }
25291
25896
  throw new Error(`No credentials available. Run \`${brand.product.cliBinary} init\` to pair, or set auth.apiKey in your config.`);
25292
25897
  }
25898
+ async function mintWaitingOutKeep(mint, brand, options) {
25899
+ const maxDelay = options.retryMaxMs ?? KEEP_RETRY_MAX_MS;
25900
+ let delay = options.retryInitialMs ?? KEEP_RETRY_INITIAL_MS;
25901
+ let unavailableSince = null;
25902
+ for (; ; ) {
25903
+ try {
25904
+ const token = await mint();
25905
+ if (unavailableSince !== null) {
25906
+ options.logger?.info(`Keep is reachable again after ${formatDuration(Date.now() - unavailableSince)}`);
25907
+ }
25908
+ return token;
25909
+ } catch (err) {
25910
+ if (!isKeepUnavailable(err))
25911
+ throw err;
25912
+ if (unavailableSince === null) {
25913
+ unavailableSince = Date.now();
25914
+ const e = err;
25915
+ const what = typeof e.statusCode === "number" ? `${e.host ?? "Keep"} answered ${e.statusCode}` : `${e.host ?? "Keep"} did not answer`;
25916
+ options.logger?.warn(`Keep is unavailable (${what}). This is typical during a deployment window; the ${brand.product.title} will wait for it and start when it answers.`);
25917
+ }
25918
+ }
25919
+ await abortableSleep(delay, options.signal);
25920
+ if (options.signal?.aborted) {
25921
+ throw options.signal.reason instanceof Error ? options.signal.reason : new Error("stopped while waiting for Keep");
25922
+ }
25923
+ delay = Math.min(delay * 2, maxDelay);
25924
+ }
25925
+ }
25293
25926
  }
25294
25927
  });
25295
25928
 
@@ -25361,8 +25994,9 @@ var require_run = __commonJS({
25361
25994
  process.exit(preflight_1.PREFLIGHT_EXIT_CODE);
25362
25995
  }
25363
25996
  const shuttle = new shuttle_1.Shuttle({ brand, config, profile });
25364
- await shuttle.start();
25365
25997
  (0, shutdown_1.setupShutdownHandler)(() => shuttle.stop());
25998
+ await shuttle.start();
25999
+ await shuttle.done();
25366
26000
  } catch (err) {
25367
26001
  if ((0, typed_exit_1.isTypedExit)(err))
25368
26002
  throw err;
@@ -25714,8 +26348,8 @@ var require_dist5 = __commonJS({
25714
26348
  "../../packages/shuttle/dist/index.js"(exports2) {
25715
26349
  "use strict";
25716
26350
  Object.defineProperty(exports2, "__esModule", { value: true });
25717
- exports2.pair = exports2.scaffoldedConfigPath = exports2.planScaffold = exports2.DEFAULT_REFUSAL_MESSAGE = exports2.DEFAULT_MONTHLY_PERIOD = exports2.DEFAULT_DAILY_PERIOD = exports2.SUBSTRATE_BRAND = exports2.verificationJson = exports2.resolveEnvironment = exports2.renderVerificationLine = exports2.renderEnvironmentReport = exports2.isWorthReporting = exports2.hostOf = exports2.environmentReportJson = exports2.applyEnvironment = exports2.renderTemplate = exports2.expandHome = exports2.resolvePaths = exports2.SpendStore = exports2.AuditLogStore = exports2.createSpendTrackerObserver = exports2.createAuditLogObserver = exports2.createLoggerObserver = exports2.SpendCapExceeded = exports2.ObserverChain = exports2.createCustomScriptExecutor = exports2.createWebhookExecutor = exports2.createHttpApiExecutor = exports2.createClaudeCodeExecutor = exports2.ExecutorRegistry = exports2.AgentCredentialRevokedError = exports2.runPreflight = exports2.renderLines = exports2.renderJson = exports2.printReport = exports2.hasFailure = exports2.createProbes = exports2.checksFor = exports2.PREFLIGHT_FLOORS = exports2.PREFLIGHT_EXIT_CODE = exports2.CHECKS = exports2.readResponseLatestVersion = exports2.checkResponseMinVersion = exports2.compareSemver = exports2.UpgradeRequiredError = exports2.LATEST_VERSION_HEADER = exports2.MIN_VERSION_HEADER = exports2.loadConfig = exports2.Shuttle = exports2.createCli = void 0;
25718
- exports2.revokePairing = exports2.listPairings = exports2.PairingConfigStore = void 0;
26351
+ exports2.scaffoldedConfigPath = exports2.planScaffold = exports2.DEFAULT_REFUSAL_MESSAGE = exports2.DEFAULT_MONTHLY_PERIOD = exports2.DEFAULT_DAILY_PERIOD = exports2.SUBSTRATE_BRAND = exports2.verificationJson = exports2.resolveEnvironment = exports2.renderVerificationLine = exports2.renderEnvironmentReport = exports2.isWorthReporting = exports2.hostOf = exports2.environmentReportJson = exports2.applyEnvironment = exports2.renderTemplate = exports2.expandHome = exports2.resolvePaths = exports2.SpendStore = exports2.AuditLogStore = exports2.createSpendTrackerObserver = exports2.createAuditLogObserver = exports2.createLoggerObserver = exports2.SpendCapExceeded = exports2.ObserverChain = exports2.createCustomScriptExecutor = exports2.createWebhookExecutor = exports2.createHttpApiExecutor = exports2.createClaudeCodeExecutor = exports2.ExecutorEnvironmentError = exports2.ExecutorRegistry = exports2.AgentCredentialRevokedError = exports2.runPreflight = exports2.renderLines = exports2.renderJson = exports2.printReport = exports2.hasFailure = exports2.createProbes = exports2.checksFor = exports2.PREFLIGHT_FLOORS = exports2.PREFLIGHT_EXIT_CODE = exports2.CHECKS = exports2.readResponseLatestVersion = exports2.checkResponseMinVersion = exports2.compareSemver = exports2.UpgradeRequiredError = exports2.LATEST_VERSION_HEADER = exports2.MIN_VERSION_HEADER = exports2.loadConfig = exports2.Shuttle = exports2.createCli = void 0;
26352
+ exports2.revokePairing = exports2.listPairings = exports2.PairingConfigStore = exports2.pair = void 0;
25719
26353
  var cli_1 = require_cli();
25720
26354
  Object.defineProperty(exports2, "createCli", { enumerable: true, get: function() {
25721
26355
  return cli_1.createCli;
@@ -25786,6 +26420,10 @@ var require_dist5 = __commonJS({
25786
26420
  Object.defineProperty(exports2, "ExecutorRegistry", { enumerable: true, get: function() {
25787
26421
  return registry_1.ExecutorRegistry;
25788
26422
  } });
26423
+ var types_1 = require_types2();
26424
+ Object.defineProperty(exports2, "ExecutorEnvironmentError", { enumerable: true, get: function() {
26425
+ return types_1.ExecutorEnvironmentError;
26426
+ } });
25789
26427
  var claude_code_1 = require_claude_code();
25790
26428
  Object.defineProperty(exports2, "createClaudeCodeExecutor", { enumerable: true, get: function() {
25791
26429
  return claude_code_1.createClaudeCodeExecutor;
@@ -25910,7 +26548,7 @@ var import_path = require("path");
25910
26548
  var import_promises = require("fs/promises");
25911
26549
  var import_yaml = __toESM(require_dist3());
25912
26550
  var import_shuttle = __toESM(require_dist5());
25913
- var buildVersion = true ? "0.30.8" : pkg.version;
26551
+ var buildVersion = true ? "0.30.9" : pkg.version;
25914
26552
  var sifterBrand = {
25915
26553
  product: {
25916
26554
  id: "sifter",