@indigoai-us/hq-cli 5.130.0 → 5.132.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +155 -1
  2. package/contracts/preflight/v1/fixtures.json +87 -0
  3. package/contracts/preflight/v1/schema.json +58 -0
  4. package/dist/command-catalog.generated.d.ts +39 -0
  5. package/dist/command-catalog.generated.js +50 -0
  6. package/dist/commands/dm.d.ts +2 -0
  7. package/dist/commands/dm.js +1 -1
  8. package/dist/commands/files.js +137 -9
  9. package/dist/commands/lanes.d.ts +13 -2
  10. package/dist/commands/lanes.js +394 -42
  11. package/dist/commands/mesh.js +66 -13
  12. package/dist/commands/update.js +1 -1
  13. package/dist/lib/bot/paths.d.ts +7 -5
  14. package/dist/lib/bot/paths.js +8 -1
  15. package/dist/lib/bot/prompt.d.ts +0 -2
  16. package/dist/lib/bot/prompt.js +5 -11
  17. package/dist/lib/bot/room-policy.d.ts +1 -24
  18. package/dist/lib/bot/room-policy.js +2 -35
  19. package/dist/lib/bot/run.d.ts +7 -8
  20. package/dist/lib/bot/run.js +21 -33
  21. package/dist/lib/bot/session.d.ts +1 -1
  22. package/dist/lib/bot/session.js +1 -1
  23. package/dist/lib/lanes/accounts.d.ts +6 -0
  24. package/dist/lib/lanes/accounts.js +2 -1
  25. package/dist/lib/lanes/config.d.ts +4 -0
  26. package/dist/lib/lanes/config.js +9 -0
  27. package/dist/lib/lanes/index.d.ts +10 -3
  28. package/dist/lib/lanes/index.js +5 -1
  29. package/dist/lib/lanes/mesh-events.js +4 -0
  30. package/dist/lib/lanes/models.d.ts +78 -0
  31. package/dist/lib/lanes/models.js +205 -0
  32. package/dist/lib/lanes/move-account.d.ts +56 -0
  33. package/dist/lib/lanes/move-account.js +193 -0
  34. package/dist/lib/lanes/provider-rejection.d.ts +86 -0
  35. package/dist/lib/lanes/provider-rejection.js +258 -0
  36. package/dist/lib/lanes/quota.d.ts +17 -0
  37. package/dist/lib/lanes/quota.js +24 -0
  38. package/dist/lib/lanes/reconcile-install.d.ts +24 -3
  39. package/dist/lib/lanes/reconcile-install.js +48 -12
  40. package/dist/lib/lanes/reconcile.js +41 -11
  41. package/dist/lib/lanes/spawn.js +26 -5
  42. package/dist/lib/lanes/store.js +10 -0
  43. package/dist/lib/lanes/types.d.ts +49 -2
  44. package/dist/lib/lanes/types.js +4 -3
  45. package/dist/lib/lanes/watcher.d.ts +27 -0
  46. package/dist/lib/lanes/watcher.js +68 -5
  47. package/dist/lib/mesh/api.d.ts +2 -0
  48. package/dist/lib/mesh/api.js +15 -2
  49. package/dist/lib/mesh/live/backfill-held.d.ts +4 -3
  50. package/dist/lib/mesh/live/backfill-held.js +37 -14
  51. package/dist/lib/mesh/live/daemon/doctor.d.ts +4 -0
  52. package/dist/lib/mesh/live/daemon/doctor.js +15 -4
  53. package/dist/lib/mesh/live/daemon/install.d.ts +5 -0
  54. package/dist/lib/mesh/live/daemon/install.js +84 -1
  55. package/dist/lib/mesh/live/daemon/run.d.ts +5 -0
  56. package/dist/lib/mesh/live/daemon/run.js +17 -4
  57. package/dist/lib/mesh/live/emit-client.d.ts +2 -0
  58. package/dist/lib/mesh/live/emit-client.js +15 -8
  59. package/dist/lib/mesh/live/emit-evidence.d.ts +19 -0
  60. package/dist/lib/mesh/live/emit-evidence.js +20 -2
  61. package/dist/lib/mesh/live/emit.d.ts +9 -0
  62. package/dist/lib/mesh/live/emit.js +28 -2
  63. package/dist/lib/mesh/live/flush.js +9 -3
  64. package/dist/lib/search-index/index.js +22 -3
  65. package/dist/lib/work-context/company.d.ts +16 -0
  66. package/dist/lib/work-context/company.js +46 -14
  67. package/dist/lib/work-context/preflight-contract.d.ts +28 -0
  68. package/dist/lib/work-context/preflight-contract.js +85 -0
  69. package/dist/main.js +11 -1
  70. package/dist/utils/cognito-session.d.ts +39 -0
  71. package/dist/utils/cognito-session.js +119 -3
  72. package/dist/utils/environmental-error.d.ts +18 -0
  73. package/dist/utils/environmental-error.js +240 -7
  74. package/dist/utils/expected-cli-error.d.ts +2 -0
  75. package/dist/utils/expected-cli-error.js +4 -0
  76. package/dist/utils/self-update.js +4 -1
  77. package/dist/utils/vault-api.d.ts +1 -0
  78. package/dist/utils/vault-api.js +11 -3
  79. package/package.json +3 -2
@@ -979,13 +979,19 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
979
979
  }
980
980
  return existing;
981
981
  }
982
+ // A queued lane that a move parked at the target's cap carries the turn
983
+ // it was resumed with: the admit spawn continues that session instead of
984
+ // starting fresh from the brief.
985
+ const pendingResume = launch.trigger === undefined && existing.state === "queued" ? existing.pending_resume : undefined;
982
986
  const trigger = launch.trigger ?? inferTrigger(existing);
983
987
  // A message into a finished lane must not turn that lane `failed` when the
984
988
  // account cannot be bound: nothing has started yet, so the record keeps its
985
989
  // state and the caller sees the error. Admission and question resume keep
986
990
  // today's behaviour of naming the terminal reason on the record.
987
991
  const markPreSpawnFailure = trigger.kind !== "message" && trigger.kind !== "resume";
988
- const plan = planLaneLaunch(existing, trigger);
992
+ const plan = pendingResume !== undefined && providerSupportsSessionResume(existing.provider)
993
+ ? { mode: "resume", session_id: pendingResume.session_id }
994
+ : planLaneLaunch(existing, trigger);
989
995
  if (existing.host.kind === "outpost") {
990
996
  refuseOutpostSelfTarget(hqRoot, existing, deps);
991
997
  }
@@ -1065,8 +1071,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1065
1071
  ? trigger.text
1066
1072
  : trigger.kind === "resume"
1067
1073
  ? trigger.text
1068
- : undefined,
1069
- resumedFrom: trigger.kind === "resume" ? trigger.from : undefined,
1074
+ : pendingResume?.text,
1075
+ resumedFrom: trigger.kind === "resume" ? trigger.from : pendingResume?.resumed_from,
1070
1076
  answersPath: trigger.kind === "answered_questions" && existsSync(answersPath)
1071
1077
  ? answersPath
1072
1078
  : undefined,
@@ -1119,7 +1125,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1119
1125
  // Resume-time compaction (compaction.ts). Runs before the claim so the
1120
1126
  // lane is never `running` without a worker while `/compact` works.
1121
1127
  let compaction;
1122
- if (resume && launch.compact !== false) {
1128
+ const compactWanted = pendingResume?.compact ?? launch.compact;
1129
+ if (resume && compactWanted !== false) {
1123
1130
  const nowFn = deps.now ?? (() => new Date());
1124
1131
  const decision = compactionDue(existing, nowFn().getTime(), providerCompactAfterMs(config.providers[lane.provider]));
1125
1132
  if (decision.due) {
@@ -1169,6 +1176,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1169
1176
  ...current,
1170
1177
  state: "running",
1171
1178
  queued_reason: undefined,
1179
+ // An account move's reservation ends with the claim.
1180
+ pending_resume: undefined,
1172
1181
  account: resolved.id,
1173
1182
  ...(model !== undefined ? { model } : {}),
1174
1183
  effort_source: effort.source,
@@ -1187,10 +1196,22 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1187
1196
  }
1188
1197
  if (compaction)
1189
1198
  next.last_compaction = { ...compaction, resume_at: nowIso };
1190
- if (resume && trigger.kind !== "admit") {
1199
+ if (resume && pendingResume !== undefined) {
1191
1200
  next.last_resume = {
1192
1201
  session_id: resume.session_id,
1193
1202
  at: nowIso,
1203
+ account: resolved.id,
1204
+ trigger: pendingResume.trigger,
1205
+ ...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
1206
+ ...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
1207
+ ...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
1208
+ };
1209
+ }
1210
+ else if (resume && trigger.kind !== "admit") {
1211
+ next.last_resume = {
1212
+ session_id: resume.session_id,
1213
+ at: nowIso,
1214
+ account: resolved.id,
1194
1215
  trigger: trigger.kind,
1195
1216
  ...(trigger.kind === "message" ? { message_id: trigger.message_id } : {}),
1196
1217
  ...(trigger.kind === "resume"
@@ -325,6 +325,11 @@ export function parseLaneRecord(raw, filePath) {
325
325
  path: filePath,
326
326
  });
327
327
  }
328
+ if (raw.account_history !== undefined && !Array.isArray(raw.account_history)) {
329
+ throw new LaneSchemaError("account_history must be an array when present", {
330
+ path: filePath,
331
+ });
332
+ }
328
333
  if (!Array.isArray(raw.attribution_history)) {
329
334
  throw new LaneSchemaError("attribution_history must be an array", {
330
335
  path: filePath,
@@ -466,6 +471,11 @@ export function mergeLaneRecords(live, sidecars) {
466
471
  ...base,
467
472
  senior_history: unionHistory(all.map((record) => record.senior_history), entryIdentity, base.senior_history),
468
473
  attribution_history: unionHistory(all.map((record) => record.attribution_history), attributionIdentity, base.attribution_history),
474
+ ...(all.some((record) => record.account_history !== undefined)
475
+ ? {
476
+ account_history: unionHistory(all.map((record) => record.account_history ?? []), entryIdentity, base.account_history ?? []),
477
+ }
478
+ : {}),
469
479
  };
470
480
  }
471
481
  export function mergeQuestionRecords(live, sidecars) {
@@ -250,6 +250,13 @@ export interface LaneRecord {
250
250
  senior_history: SeniorHistoryEntry[];
251
251
  state: LaneState;
252
252
  queued_reason?: QueuedReason;
253
+ /**
254
+ * Set by `hq lanes pickup` when it stores an outpost lane and cleared by
255
+ * the account bind once the model probe has finished. While set, the
256
+ * reconcile pass does not admit the lane: a spawn during the probe would
257
+ * leave a running worker behind a record the probe result then rewrites.
258
+ */
259
+ pickup_validation_pending?: true;
253
260
  priority: number;
254
261
  worker?: string;
255
262
  /** Registry-relative directory of the resolved worker, when --worker was set. */
@@ -258,6 +265,20 @@ export interface LaneRecord {
258
265
  worker_pack?: string;
259
266
  provider: LaneProvider;
260
267
  account?: string;
268
+ /**
269
+ * Every operator move of the lane's session to another account of the
270
+ * same provider (`hq lanes resume|message --account`), newest last, capped
271
+ * like the other histories.
272
+ */
273
+ account_history?: AccountHistoryEntry[];
274
+ /**
275
+ * Set by an account move (`--account`) until the spawn claims the lane:
276
+ * the turn the move was asked to run. While set it reserves a slot on the
277
+ * account for the capacity check of later moves. When the target was at
278
+ * its cap the lane is queued there and the reconcile pass resumes the
279
+ * recorded session with this turn instead of starting fresh from the brief.
280
+ */
281
+ pending_resume?: PendingResume;
261
282
  host: LaneHost;
262
283
  /**
263
284
  * Creating machine's identity when `host.kind` is `outpost` (spec §5.2).
@@ -308,8 +329,10 @@ export interface LaneRecord {
308
329
  terminal_reason?: string;
309
330
  /**
310
331
  * Structured detail behind a named `terminal_reason` (dispatch, pickup and
311
- * spawn failures), or the missing-binary search path behind
312
- * `provider_binary_missing`. Written by the command and spawn writers.
332
+ * spawn failures), the missing-binary search path behind
333
+ * `provider_binary_missing`, or the provider's own message, status, model
334
+ * and account behind `provider_rejected`. Written by the command, spawn
335
+ * and watcher writers.
313
336
  */
314
337
  terminal_details?: Record<string, unknown>;
315
338
  /** SSM command id of the `hq outposts exec` pickup that started this lane. */
@@ -368,9 +391,27 @@ export interface LaneEnvelopeRepair {
368
391
  envelope_ref?: string;
369
392
  at: string;
370
393
  }
394
+ export interface AccountHistoryEntry {
395
+ from: string | null;
396
+ to: string;
397
+ at: string;
398
+ reason: "operator_move";
399
+ }
400
+ export interface PendingResume {
401
+ session_id: string;
402
+ trigger: "message" | "resume";
403
+ resumed_from?: "interrupted" | "done" | "blocked";
404
+ text?: string;
405
+ message_id?: string;
406
+ /** False when the command passed --no-compact; the deferred spawn honours it. */
407
+ compact?: boolean;
408
+ at: string;
409
+ }
371
410
  export interface LaneResumeRecord {
372
411
  session_id: string;
373
412
  at: string;
413
+ /** Account the resumed run was bound to. */
414
+ account?: string;
374
415
  trigger: "answered_questions" | "message" | "resume";
375
416
  message_id?: string;
376
417
  /**
@@ -508,6 +549,12 @@ export interface ProviderConfig {
508
549
  * means the five-minute default.
509
550
  */
510
551
  progress_event_min_interval_ms?: number;
552
+ /**
553
+ * Hold applied to an account after a 429 whose message names no reset
554
+ * time: the account is skipped for this long. Absent means the
555
+ * fifteen-minute default.
556
+ */
557
+ quota_default_hold_ms?: number;
511
558
  accounts: Record<string, AccountConfig>;
512
559
  }
513
560
  export interface ThrottleConfig {
@@ -198,13 +198,14 @@ export const LANE_FIELD_WRITERS = {
198
198
  project_id: ["command"], story_id: ["command"], thread_id: ["creator", "command"],
199
199
  depth: ["command"], senior: ["command"], senior_history: ["command"],
200
200
  state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "reconcile"],
201
+ pickup_validation_pending: ["creator", "command", "spawn"],
201
202
  priority: ["command"], worker: ["creator"], worker_path: ["creator"], worker_pack: ["creator"],
202
- provider: ["creator"], account: ["creator", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
203
+ provider: ["creator"], account: ["creator", "spawn", "command"], account_history: ["command"], pending_resume: ["command", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
203
204
  version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
204
- last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn"], effort: ["creator", "spawn"], effort_source: ["creator", "spawn"],
205
+ last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], effort: ["creator", "spawn"], effort_source: ["creator", "spawn"],
205
206
  last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
206
207
  overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
207
- admitted_reason: ["reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn"],
208
+ admitted_reason: ["reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
208
209
  quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
209
210
  provider_session_id: ["watcher"], provider_session_id_absent: ["watcher"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], last_progress_event_at: ["watcher"],
210
211
  };
@@ -33,6 +33,7 @@
33
33
  */
34
34
  import { type LaneEnvelope, type LaneRecord, type LaneState } from "./types.js";
35
35
  import { type QuotaSignal } from "./quota.js";
36
+ import { type ProviderRejection } from "./provider-rejection.js";
36
37
  import { type LaneMesh } from "./mesh-events.js";
37
38
  import { type EnvelopeRepairSpawnInput } from "./spawn.js";
38
39
  /** Spec §5.7: no signal for 20 minutes while the process is still alive. */
@@ -73,6 +74,7 @@ export interface WatcherTickInput {
73
74
  invalidEnvelope?: {
74
75
  errors: string[];
75
76
  } | null;
77
+ providerRejection?: ProviderRejection | null;
76
78
  }
77
79
  export interface WatcherTickResult {
78
80
  emit?: WatcherSignal;
@@ -97,7 +99,30 @@ export declare function classifyLaneExit(input: {
97
99
  invalidEnvelope?: {
98
100
  errors: string[];
99
101
  } | null;
102
+ /**
103
+ * Set when the worker's output stream carries a provider-level turn
104
+ * failure (codex `turn.failed`, a Claude error result with an HTTP status,
105
+ * a grok error object). Failed, never interrupted: the provider refused
106
+ * the turn, so there is nothing to resume and no envelope to repair.
107
+ */
108
+ providerRejection?: ProviderRejection | null;
100
109
  }): TerminalClassification;
110
+ /**
111
+ * One decision for an exited worker's output. An explicit provider status
112
+ * wins over the quota text heuristics: a 429 from any provider (including
113
+ * codex `turn.failed` and grok error objects, which the claude-shaped quota
114
+ * reader never sees) becomes a quota signal; any other explicit status is a
115
+ * rejection even when its text sounds like a quota message (a 402 "requires
116
+ * usage credits" is a rejection, not a hold). A failure with no status at
117
+ * all is matched against the quota text heuristics first: a status-less
118
+ * "usage limit" / "rate limit" turn.failed still creates the account hold;
119
+ * any other status-less message is a rejection. Only when the stream
120
+ * carries no failure object does the quota reader's own read apply.
121
+ */
122
+ export declare function routeProviderFailure(failure: ProviderRejection | null, readQuota: () => QuotaSignal | null, model: string | undefined, now: Date): {
123
+ quota: QuotaSignal | null;
124
+ rejection: ProviderRejection | null;
125
+ };
101
126
  export declare function tickWatcher(input: WatcherTickInput): WatcherTickResult;
102
127
  export declare function watcherEventsPath(hqRoot: string, laneId: string): string;
103
128
  export declare function watcherOutLogPath(hqRoot: string, laneId: string): string;
@@ -119,6 +144,8 @@ export interface ApplyTerminalOpts {
119
144
  envelopeRef?: string;
120
145
  at?: string;
121
146
  quota?: QuotaSignal | null;
147
+ /** Written to `terminal_details` with the model and account the run used. */
148
+ providerRejection?: ProviderRejection | null;
122
149
  }
123
150
  export declare function stalledReason(silenceMs: number): string;
124
151
  /**
@@ -38,9 +38,10 @@ import { raiseEnvelopeQuestions } from "./questions.js";
38
38
  import { isTerminalLaneState, parseLaneEnvelope, } from "./types.js";
39
39
  import { readLane, retryUpdateLane } from "./store.js";
40
40
  import { readWorkerRunResult } from "./delivery.js";
41
- import { readQuotaSignal, recordAccountQuota } from "./quota.js";
41
+ import { applyDefaultQuotaHold, quotaSignalFromFields, readQuotaSignal, recordAccountQuota, } from "./quota.js";
42
+ import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
42
43
  import { readToolActivity, toolActivityPath } from "./dropbox.js";
43
- import { loadLanesConfig, providerProgressEventMinIntervalMs } from "./config.js";
44
+ import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
44
45
  import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, progressEventDue, } from "./mesh-events.js";
45
46
  import { envelopeQuestionId } from "./questions.js";
46
47
  import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, } from "./spawn.js";
@@ -86,6 +87,9 @@ export function classifyLaneExit(input) {
86
87
  return { state: "blocked" };
87
88
  throw new LanesError("unknown_envelope_decision", `Unknown envelope decision ${JSON.stringify(input.envelope.decision)}. Accepted: done, ask, blocked`, { accepted: ["done", "ask", "blocked"], value: input.envelope.decision });
88
89
  }
90
+ if (input.providerRejection) {
91
+ return { state: "failed", reason: PROVIDER_REJECTED_REASON };
92
+ }
89
93
  if (input.invalidEnvelope) {
90
94
  return { state: "failed", reason: ENVELOPE_INVALID_REASON };
91
95
  }
@@ -100,6 +104,34 @@ export function classifyLaneExit(input) {
100
104
  }
101
105
  return { state: "failed", reason: "exited_without_envelope" };
102
106
  }
107
+ /**
108
+ * One decision for an exited worker's output. An explicit provider status
109
+ * wins over the quota text heuristics: a 429 from any provider (including
110
+ * codex `turn.failed` and grok error objects, which the claude-shaped quota
111
+ * reader never sees) becomes a quota signal; any other explicit status is a
112
+ * rejection even when its text sounds like a quota message (a 402 "requires
113
+ * usage credits" is a rejection, not a hold). A failure with no status at
114
+ * all is matched against the quota text heuristics first: a status-less
115
+ * "usage limit" / "rate limit" turn.failed still creates the account hold;
116
+ * any other status-less message is a rejection. Only when the stream
117
+ * carries no failure object does the quota reader's own read apply.
118
+ */
119
+ export function routeProviderFailure(failure, readQuota, model, now) {
120
+ if (failure && failure.status === QUOTA_STATUS) {
121
+ return {
122
+ quota: quotaSignalFromFields({ api_error_status: QUOTA_STATUS, is_error: true, result: failure.message }, model, now),
123
+ rejection: null,
124
+ };
125
+ }
126
+ if (failure && failure.status === null) {
127
+ const quota = quotaSignalFromFields({ api_error_status: undefined, is_error: true, result: failure.message }, model, now);
128
+ if (quota)
129
+ return { quota, rejection: null };
130
+ }
131
+ if (failure)
132
+ return { quota: null, rejection: failure };
133
+ return { quota: readQuota(), rejection: null };
134
+ }
103
135
  export function tickWatcher(input) {
104
136
  const silenceMs = input.silenceMs ?? SILENCE_MS;
105
137
  const at = new Date(input.nowMs).toISOString();
@@ -109,6 +141,7 @@ export function tickWatcher(input) {
109
141
  hasPartialOutput: input.hasPartialOutput,
110
142
  quota: input.quota,
111
143
  invalidEnvelope: input.invalidEnvelope,
144
+ providerRejection: input.providerRejection,
112
145
  });
113
146
  return {
114
147
  emit: { kind: "terminal", at, classification },
@@ -301,6 +334,9 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
301
334
  if (opts.quota?.marker) {
302
335
  next.model_quota_marker = opts.quota.marker;
303
336
  }
337
+ if (opts.providerRejection) {
338
+ next.terminal_details = providerRejectionDetails(opts.providerRejection, current);
339
+ }
304
340
  if (opts.envelope && opts.envelopeRef) {
305
341
  next.last_envelope = {
306
342
  ref: opts.envelopeRef,
@@ -589,19 +625,37 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
589
625
  const hasPartialOutput = logFiles.some((file) => fileNonEmpty(file));
590
626
  let quota = null;
591
627
  let quotaLane = null;
628
+ let providerRejection = null;
592
629
  if (!processAlive && !envelope) {
593
630
  try {
594
631
  quotaLane = readLane(hqRoot, laneId);
595
- quota = readQuotaSignal(runDir, quotaLane.model, new Date(now()), readSpawnManifest(runDir).worker_out_offset ?? 0);
632
+ const workerOutOffset = readSpawnManifest(runDir).worker_out_offset ?? 0;
633
+ const routed = routeProviderFailure(readProviderRejection(runDir, quotaLane.provider, workerOutOffset), () => readQuotaSignal(runDir, quotaLane.model, new Date(now()), workerOutOffset), quotaLane.model, new Date(now()));
634
+ // A reset-less 429 still holds the account: the configured default
635
+ // hold becomes its reset, recorded as such.
636
+ quota = routed.quota
637
+ ? applyDefaultQuotaHold(routed.quota, providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[quotaLane.provider]), new Date(now()))
638
+ : null;
639
+ providerRejection = routed.rejection;
596
640
  }
597
641
  catch (err) {
598
642
  process.stderr.write(`hq lanes watcher: quota read failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
599
643
  }
600
644
  }
601
645
  // Spec §4: the process exited and its envelope file does not parse.
602
- // Exactly one repair turn, then failed with both errors.
646
+ // Exactly one repair turn, then failed with both errors. A provider
647
+ // rejection skips the repair: the provider refused the turn, so a
648
+ // second turn on the same model and account would be refused the same
649
+ // way. A quota signal skips it too: a repair turn on an exhausted account
650
+ // fails the same way, and would end the lane envelope_invalid without
651
+ // ever recording the hold. The quota result applies immediately instead.
603
652
  let invalidEnvelope = null;
604
- if (!processAlive && !envelope && latestFile && latestFile.envelope === null) {
653
+ if (!processAlive &&
654
+ !envelope &&
655
+ !providerRejection &&
656
+ !quota &&
657
+ latestFile &&
658
+ latestFile.envelope === null) {
605
659
  const repair = decideEnvelopeRepair(hqRoot, laneId, latestFile, now(), spawnRepair, logDecision);
606
660
  if (repair.action === "spawned") {
607
661
  lastSignalMs = now();
@@ -626,6 +680,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
626
680
  silenceMs,
627
681
  quota,
628
682
  invalidEnvelope,
683
+ providerRejection,
629
684
  });
630
685
  if (processAlive) {
631
686
  maybeHeartbeatLane(hqRoot, laneId, now());
@@ -702,6 +757,13 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
702
757
  state: classification.state,
703
758
  reason: classification.reason,
704
759
  ...(invalidEnvelope ? { envelope_errors: invalidEnvelope.errors } : {}),
760
+ ...(providerRejection
761
+ ? {
762
+ provider_status: providerRejection.status,
763
+ provider_message: providerRejection.message,
764
+ retryable: providerRejection.retryable,
765
+ }
766
+ : {}),
705
767
  });
706
768
  if (quota &&
707
769
  quotaLane?.account &&
@@ -719,6 +781,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
719
781
  envelopeRef: latest?.ref,
720
782
  at: tick.emit.at,
721
783
  quota,
784
+ providerRejection,
722
785
  });
723
786
  const updated = terminalWrite.record;
724
787
  logDecision({
@@ -56,6 +56,7 @@ export declare function meshJson(token: string, path: string, init?: {
56
56
  method?: string;
57
57
  body?: Record<string, unknown>;
58
58
  query?: Record<string, string>;
59
+ headers?: Record<string, string>;
59
60
  signal?: AbortSignal;
60
61
  }): Promise<unknown>;
61
62
  export declare function resolveMeshCompany(token: string, company?: string): Promise<MeshCompany>;
@@ -133,6 +134,7 @@ export declare function putProjectView(token: string, companyUid: string, projec
133
134
  stories?: MeshProjectStoryInput[];
134
135
  repos?: MeshProjectRepo[];
135
136
  version?: number;
137
+ expectedVersion?: number;
136
138
  }): Promise<MeshProjectView>;
137
139
  /** Register the project genesis thread and deterministic project channel. */
138
140
  export declare function registerMeshProject(token: string, companyUid: string, projectId: string): Promise<MeshProjectRegistration>;
@@ -30,6 +30,7 @@ export async function meshJson(token, path, init = {}) {
30
30
  method: init.method,
31
31
  body: init.body,
32
32
  query: init.query,
33
+ headers: init.headers,
33
34
  signal: init.signal,
34
35
  });
35
36
  const text = await res.text();
@@ -53,7 +54,11 @@ export async function meshJson(token, path, init = {}) {
53
54
  typeof err.stage === "string" && err.stage ? `stage=${err.stage}` : "",
54
55
  typeof err.recovery === "string" && err.recovery ? `recovery=${err.recovery}` : "",
55
56
  ].filter(Boolean);
56
- throw new Error(`${res.status} ${code}${detail}${context.length ? ` (${context.join("; ")})` : ""}`);
57
+ const failure = new Error(`${res.status} ${code}${detail}${context.length ? ` (${context.join("; ")})` : ""}`);
58
+ failure.status = res.status;
59
+ if (typeof err.code === "string" && err.code)
60
+ failure.code = err.code;
61
+ throw failure;
57
62
  }
58
63
  return data;
59
64
  }
@@ -415,9 +420,17 @@ export async function getProjectView(token, companyUid, projectId) {
415
420
  }
416
421
  /** Replace a project's durable Board view and keep the local project cache current. */
417
422
  export async function putProjectView(token, companyUid, projectId, input) {
423
+ const { expectedVersion, ...document } = input;
418
424
  const view = asRecord(await meshJson(token, `/v1/work-mesh/projects/${encodeURIComponent(projectId)}`, {
419
425
  method: "PUT",
420
- body: { companyUid, ...input },
426
+ body: {
427
+ companyUid,
428
+ ...document,
429
+ ...(expectedVersion !== undefined ? { expectedVersion } : {}),
430
+ },
431
+ ...(expectedVersion !== undefined
432
+ ? { headers: { "If-Match": String(expectedVersion) } }
433
+ : {}),
421
434
  }));
422
435
  if (!view)
423
436
  throw new Error("Work mesh returned an invalid project view");
@@ -49,15 +49,16 @@ export interface HeldSessionRef {
49
49
  /** Task id carried on the event, if any. */
50
50
  task?: string;
51
51
  /**
52
- * Per-event local evidence retained while coalescing a held session. Event
53
- * timestamps are instance boundaries: a session id may be reused, so a root
54
- * observed before the current state.startedAt must not attribute it.
52
+ * Per-event local evidence retained while coalescing a held session. A
53
+ * session id may be reused. The current instance is state.bindingEpisodeId,
54
+ * never the event timestamp: a later `at` on another episode is still stale.
55
55
  */
56
56
  eventEvidence?: Array<{
57
57
  hqRoot?: string;
58
58
  cwd?: string;
59
59
  harness?: string;
60
60
  at?: string;
61
+ bindingEpisodeId?: string;
61
62
  }>;
62
63
  }
63
64
  export interface BackfillHeldResult {
@@ -163,9 +163,11 @@ function localCompanyCandidates(ref, state, canonicalHqRoot, canonicalHqRoots =
163
163
  return candidates;
164
164
  }
165
165
  /**
166
- * Keep only event evidence that belongs to the current session instance. A
167
- * current state is a durable instance marker; metadata from a held event that
168
- * predates it could belong to a prior use of the same session id.
166
+ * Keep only event evidence for the current session instance. When the state
167
+ * file has a binding episode, that id is the instance. A timestamp, including
168
+ * one later than startedAt, does not make another episode current.
169
+ * Sessions that predate episodes still use startedAt so an older event is
170
+ * not treated as this instance.
169
171
  */
170
172
  function currentInstanceEventEvidence(ref, state) {
171
173
  const evidence = ref.eventEvidence ?? [
@@ -176,6 +178,9 @@ function currentInstanceEventEvidence(ref, state) {
176
178
  },
177
179
  ...(ref.hqRoots ?? []).filter((hqRoot) => hqRoot !== ref.hqRoot).map((hqRoot) => ({ hqRoot })),
178
180
  ];
181
+ if (state?.bindingEpisodeId) {
182
+ return evidence.filter((entry) => Boolean(entry.hqRoot) && entry.bindingEpisodeId === state.bindingEpisodeId);
183
+ }
179
184
  if (!state?.startedAt)
180
185
  return evidence;
181
186
  const startedAt = Date.parse(state.startedAt);
@@ -188,8 +193,19 @@ function currentInstanceEventEvidence(ref, state) {
188
193
  return Number.isFinite(eventAt) && eventAt >= startedAt;
189
194
  });
190
195
  }
196
+ /** Plain reason when every rooted held event belongs to another episode. */
197
+ function episodeMismatchReason(ref, state) {
198
+ if (!state?.bindingEpisodeId)
199
+ return null;
200
+ const rooted = (ref.eventEvidence ?? []).filter((entry) => entry.hqRoot);
201
+ if (rooted.length === 0)
202
+ return null;
203
+ if (rooted.some((entry) => entry.bindingEpisodeId === state.bindingEpisodeId))
204
+ return null;
205
+ return "episode does not match the current instance";
206
+ }
191
207
  function hasOnlyStaleEventEvidence(ref, state) {
192
- return Boolean(state?.startedAt &&
208
+ return Boolean((state?.bindingEpisodeId || state?.startedAt) &&
193
209
  ref.eventEvidence?.some((entry) => entry.hqRoot) &&
194
210
  !currentInstanceEventEvidence(ref, state).some((entry) => entry.hqRoot));
195
211
  }
@@ -310,12 +326,18 @@ function latestAcknowledgedRegistration(sessionId, operations) {
310
326
  function acknowledgedRegistrationMatches(operation, ref, evidence, state) {
311
327
  if (!operation.companyUid || !operation.receiptId)
312
328
  return false;
313
- if (!state?.startedAt)
314
- return false;
315
- const startedAt = Date.parse(state.startedAt);
316
- const registeredAt = Date.parse(operation.createdAt);
317
- if (!Number.isFinite(startedAt) || !Number.isFinite(registeredAt) || registeredAt < startedAt) {
318
- return false;
329
+ if (state?.bindingEpisodeId) {
330
+ if (state.bindingEpisodeId !== operation.receiptId)
331
+ return false;
332
+ }
333
+ else {
334
+ if (!state?.startedAt)
335
+ return false;
336
+ const startedAt = Date.parse(state.startedAt);
337
+ const registeredAt = Date.parse(operation.createdAt);
338
+ if (!Number.isFinite(startedAt) || !Number.isFinite(registeredAt) || registeredAt < startedAt) {
339
+ return false;
340
+ }
319
341
  }
320
342
  if (evidence &&
321
343
  !candidatesAgree(evidence, {
@@ -325,8 +347,6 @@ function acknowledgedRegistrationMatches(operation, ref, evidence, state) {
325
347
  return false;
326
348
  if (ref.harness && operation.harness && ref.harness !== operation.harness)
327
349
  return false;
328
- if (state?.bindingEpisodeId && state.bindingEpisodeId !== operation.receiptId)
329
- return false;
330
350
  return true;
331
351
  }
332
352
  /** Shared doctor/backfill gate for a receipt that is safe to treat as current evidence. */
@@ -407,8 +427,9 @@ export function readHeldSessions(workMeshRoot) {
407
427
  if (task && !ref.task)
408
428
  ref.task = task;
409
429
  const at = str(event.at) ?? (envelope ? str(envelope.heldAt) : undefined);
430
+ const bindingEpisodeId = str(event.bindingEpisodeId);
410
431
  if (hqRoot || cwd) {
411
- ref.eventEvidence = [...(ref.eventEvidence ?? []), { hqRoot, cwd, harness, at }];
432
+ ref.eventEvidence = [...(ref.eventEvidence ?? []), { hqRoot, cwd, harness, at, bindingEpisodeId }];
412
433
  }
413
434
  seen.set(sessionId, ref);
414
435
  }
@@ -481,7 +502,9 @@ export async function backfillHeldSessions(deps) {
481
502
  if (!evidence.candidate?.slug && deps.useDeviceDefault !== true) {
482
503
  result.unresolved += 1;
483
504
  result.noEvidence += 1;
484
- deps.log?.(`would stay unresolved ${sessionId} (${hasOnlyStaleEventEvidence(ref, state) ? "STALE_EVIDENCE" : "no local company evidence"})`);
505
+ const staleReason = episodeMismatchReason(ref, state)
506
+ ?? (hasOnlyStaleEventEvidence(ref, state) ? "STALE_EVIDENCE" : "no local company evidence");
507
+ deps.log?.(`would stay unresolved ${sessionId} (${staleReason})`);
485
508
  continue;
486
509
  }
487
510
  const evidenceRef = {
@@ -33,6 +33,10 @@ export interface DaemonDoctorReport {
33
33
  tokenSource: CognitoTokenSource;
34
34
  /** human | agent | unknown — from token claims / machine signal. */
35
35
  actorKind: CognitoActorKind;
36
+ /** Subject in the person login file (`hq whoami`), or empty claims. */
37
+ whoamiSubject: string;
38
+ /** Subject the daemon will authenticate as (machine cache, else person file). */
39
+ daemonSubject: string;
36
40
  /** Credential vend refused (disabled / unsupported / 403). */
37
41
  presenceRefusal?: DaemonStateFile["presenceRefusal"];
38
42
  /** Companies the current presence credential did not include. */
@@ -5,7 +5,7 @@
5
5
  import * as fs from "node:fs";
6
6
  import * as os from "node:os";
7
7
  import * as path from "node:path";
8
- import { describeCognitoTokenSource, } from "../../../../utils/cognito-session.js";
8
+ import { describeCognitoTokenSource, formatCognitoSubject, loadMachineCachedTokens, readPersonTokenFile, } from "../../../../utils/cognito-session.js";
9
9
  import { listOutboxOperations, outboxStats, } from "../../../work-context/outbox.js";
10
10
  import { readSessionState } from "../../../work-context/state.js";
11
11
  import { workContextRoot } from "../../../work-context/paths.js";
@@ -126,7 +126,7 @@ export function collectDaemonDoctor(deps = {}) {
126
126
  if (deadLetterCount > 0) {
127
127
  unhealthyReasons.push(`dead-letter count=${deadLetterCount}`);
128
128
  }
129
- if (emitMode === "legacy" && heldCount > 0) {
129
+ if (heldCount > 0) {
130
130
  unhealthyReasons.push(`held events awaiting company attribution=${heldCount}`);
131
131
  }
132
132
  if (heldWithNoLocalEvidence > 0) {
@@ -145,6 +145,12 @@ export function collectDaemonDoctor(deps = {}) {
145
145
  unhealthyReasons.push(`presence credential refused (${presenceRefusal.code}); next retry ${presenceRefusal.nextRetryAt}`);
146
146
  }
147
147
  const auth = describeCognitoTokenSource({ home, env });
148
+ const whoamiTokens = readPersonTokenFile(home, env);
149
+ const daemonTokens = auth.tokenSource === "machine"
150
+ ? loadMachineCachedTokens(home, env)
151
+ : whoamiTokens;
152
+ const whoamiSubject = formatCognitoSubject(whoamiTokens);
153
+ const daemonSubject = formatCognitoSubject(daemonTokens);
148
154
  const emitState = readEmitState(meshRoot);
149
155
  const emitRetry = emitRetryDepth(meshRoot);
150
156
  if (emitRetry > 0) {
@@ -169,6 +175,8 @@ export function collectDaemonDoctor(deps = {}) {
169
175
  oldestSpoolAgeMs,
170
176
  tokenSource: auth.tokenSource,
171
177
  actorKind: auth.actorKind,
178
+ whoamiSubject,
179
+ daemonSubject,
172
180
  presenceRefusal,
173
181
  presenceDroppedCompanies,
174
182
  emitMode,
@@ -197,10 +205,13 @@ export function formatDaemonDoctor(report) {
197
205
  if (report.emitMode === "legacy") {
198
206
  // Legacy emit path only: the spool/held/outbox queues do not exist in
199
207
  // receive-only (direct) mode.
200
- lines.push(`spool depth: ${report.spoolDepth}`, `held: ${report.heldCount}`, `held with no local evidence: ${report.heldWithNoLocalEvidence}`, `outbox depth: ${report.outboxDepth}`);
208
+ lines.push(`spool depth: ${report.spoolDepth}`, `held: ${report.heldCount}${report.heldCount > 0 ? " (needs_company; not sent as NONE)" : ""}`, `held with no local evidence: ${report.heldWithNoLocalEvidence}`, `outbox depth: ${report.outboxDepth}`);
209
+ }
210
+ if (report.emitMode !== "legacy" && report.heldCount > 0) {
211
+ lines.push(`held: ${report.heldCount} (needs_company; not sent as NONE)`);
201
212
  }
202
213
  lines.push(`attributed by device default: ${report.attributedByDeviceDefault}`, `device-default authorization denials: ${report.deviceDefaultAuthorizationDenials}`, `dead-letter: ${report.deadLetterCount}`, `unattributed events: ${report.unattributedEvents.total} ` +
203
- `(spool=${report.unattributedEvents.spool} held=${report.unattributedEvents.held} dead-letter=${report.unattributedEvents.deadLetter})`, `token source: ${report.tokenSource}`, `actor kind: ${report.actorKind}`);
214
+ `(spool=${report.unattributedEvents.spool} held=${report.unattributedEvents.held} dead-letter=${report.unattributedEvents.deadLetter})`, `token source: ${report.tokenSource}`, `actor kind: ${report.actorKind}`, `identity whoami: ${report.whoamiSubject}`, `identity daemon: ${report.daemonSubject}`);
204
215
  if (report.emitMode === "legacy") {
205
216
  lines.push(`last flush: ${report.lastFlushAt ?? "(never)"}${report.lastFlushResult
206
217
  ? ` ok=${report.lastFlushResult.ok} posted=${report.lastFlushResult.posted}`