@indigoai-us/hq-cli 5.131.0 → 5.132.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/dist/command-catalog.generated.d.ts +12 -0
- package/dist/command-catalog.generated.js +16 -0
- package/dist/commands/lanes.d.ts +13 -2
- package/dist/commands/lanes.js +394 -42
- package/dist/commands/mesh.js +20 -1
- package/dist/commands/update.js +1 -1
- package/dist/lib/bot/paths.d.ts +7 -5
- package/dist/lib/bot/paths.js +8 -1
- package/dist/lib/bot/prompt.d.ts +0 -2
- package/dist/lib/bot/prompt.js +5 -11
- package/dist/lib/bot/room-policy.d.ts +1 -24
- package/dist/lib/bot/room-policy.js +2 -35
- package/dist/lib/bot/run.d.ts +7 -8
- package/dist/lib/bot/run.js +21 -33
- package/dist/lib/bot/session.d.ts +1 -1
- package/dist/lib/bot/session.js +1 -1
- package/dist/lib/lanes/accounts.d.ts +6 -0
- package/dist/lib/lanes/accounts.js +2 -1
- package/dist/lib/lanes/config.d.ts +4 -0
- package/dist/lib/lanes/config.js +9 -0
- package/dist/lib/lanes/index.d.ts +10 -3
- package/dist/lib/lanes/index.js +5 -1
- package/dist/lib/lanes/mesh-events.js +4 -0
- package/dist/lib/lanes/models.d.ts +78 -0
- package/dist/lib/lanes/models.js +205 -0
- package/dist/lib/lanes/move-account.d.ts +56 -0
- package/dist/lib/lanes/move-account.js +193 -0
- package/dist/lib/lanes/provider-rejection.d.ts +86 -0
- package/dist/lib/lanes/provider-rejection.js +258 -0
- package/dist/lib/lanes/quota.d.ts +17 -0
- package/dist/lib/lanes/quota.js +24 -0
- package/dist/lib/lanes/reconcile-install.d.ts +24 -3
- package/dist/lib/lanes/reconcile-install.js +48 -12
- package/dist/lib/lanes/reconcile.js +41 -11
- package/dist/lib/lanes/spawn.js +26 -5
- package/dist/lib/lanes/store.js +10 -0
- package/dist/lib/lanes/types.d.ts +49 -2
- package/dist/lib/lanes/types.js +4 -3
- package/dist/lib/lanes/watcher.d.ts +27 -0
- package/dist/lib/lanes/watcher.js +68 -5
- package/dist/lib/mesh/live/backfill-held.d.ts +4 -3
- package/dist/lib/mesh/live/backfill-held.js +37 -14
- package/dist/lib/mesh/live/daemon/doctor.js +5 -2
- package/dist/lib/mesh/live/daemon/install.d.ts +5 -0
- package/dist/lib/mesh/live/daemon/install.js +84 -1
- package/dist/lib/mesh/live/daemon/run.d.ts +5 -0
- package/dist/lib/mesh/live/daemon/run.js +14 -4
- package/dist/lib/mesh/live/emit-evidence.d.ts +19 -0
- package/dist/lib/mesh/live/emit-evidence.js +20 -2
- package/dist/lib/work-context/company.d.ts +16 -0
- package/dist/lib/work-context/company.js +46 -14
- package/dist/utils/self-update.js +4 -1
- package/package.json +1 -1
|
@@ -250,6 +250,13 @@ export interface LaneRecord {
|
|
|
250
250
|
senior_history: SeniorHistoryEntry[];
|
|
251
251
|
state: LaneState;
|
|
252
252
|
queued_reason?: QueuedReason;
|
|
253
|
+
/**
|
|
254
|
+
* Set by `hq lanes pickup` when it stores an outpost lane and cleared by
|
|
255
|
+
* the account bind once the model probe has finished. While set, the
|
|
256
|
+
* reconcile pass does not admit the lane: a spawn during the probe would
|
|
257
|
+
* leave a running worker behind a record the probe result then rewrites.
|
|
258
|
+
*/
|
|
259
|
+
pickup_validation_pending?: true;
|
|
253
260
|
priority: number;
|
|
254
261
|
worker?: string;
|
|
255
262
|
/** Registry-relative directory of the resolved worker, when --worker was set. */
|
|
@@ -258,6 +265,20 @@ export interface LaneRecord {
|
|
|
258
265
|
worker_pack?: string;
|
|
259
266
|
provider: LaneProvider;
|
|
260
267
|
account?: string;
|
|
268
|
+
/**
|
|
269
|
+
* Every operator move of the lane's session to another account of the
|
|
270
|
+
* same provider (`hq lanes resume|message --account`), newest last, capped
|
|
271
|
+
* like the other histories.
|
|
272
|
+
*/
|
|
273
|
+
account_history?: AccountHistoryEntry[];
|
|
274
|
+
/**
|
|
275
|
+
* Set by an account move (`--account`) until the spawn claims the lane:
|
|
276
|
+
* the turn the move was asked to run. While set it reserves a slot on the
|
|
277
|
+
* account for the capacity check of later moves. When the target was at
|
|
278
|
+
* its cap the lane is queued there and the reconcile pass resumes the
|
|
279
|
+
* recorded session with this turn instead of starting fresh from the brief.
|
|
280
|
+
*/
|
|
281
|
+
pending_resume?: PendingResume;
|
|
261
282
|
host: LaneHost;
|
|
262
283
|
/**
|
|
263
284
|
* Creating machine's identity when `host.kind` is `outpost` (spec §5.2).
|
|
@@ -308,8 +329,10 @@ export interface LaneRecord {
|
|
|
308
329
|
terminal_reason?: string;
|
|
309
330
|
/**
|
|
310
331
|
* Structured detail behind a named `terminal_reason` (dispatch, pickup and
|
|
311
|
-
* spawn failures),
|
|
312
|
-
* `provider_binary_missing
|
|
332
|
+
* spawn failures), the missing-binary search path behind
|
|
333
|
+
* `provider_binary_missing`, or the provider's own message, status, model
|
|
334
|
+
* and account behind `provider_rejected`. Written by the command, spawn
|
|
335
|
+
* and watcher writers.
|
|
313
336
|
*/
|
|
314
337
|
terminal_details?: Record<string, unknown>;
|
|
315
338
|
/** SSM command id of the `hq outposts exec` pickup that started this lane. */
|
|
@@ -368,9 +391,27 @@ export interface LaneEnvelopeRepair {
|
|
|
368
391
|
envelope_ref?: string;
|
|
369
392
|
at: string;
|
|
370
393
|
}
|
|
394
|
+
export interface AccountHistoryEntry {
|
|
395
|
+
from: string | null;
|
|
396
|
+
to: string;
|
|
397
|
+
at: string;
|
|
398
|
+
reason: "operator_move";
|
|
399
|
+
}
|
|
400
|
+
export interface PendingResume {
|
|
401
|
+
session_id: string;
|
|
402
|
+
trigger: "message" | "resume";
|
|
403
|
+
resumed_from?: "interrupted" | "done" | "blocked";
|
|
404
|
+
text?: string;
|
|
405
|
+
message_id?: string;
|
|
406
|
+
/** False when the command passed --no-compact; the deferred spawn honours it. */
|
|
407
|
+
compact?: boolean;
|
|
408
|
+
at: string;
|
|
409
|
+
}
|
|
371
410
|
export interface LaneResumeRecord {
|
|
372
411
|
session_id: string;
|
|
373
412
|
at: string;
|
|
413
|
+
/** Account the resumed run was bound to. */
|
|
414
|
+
account?: string;
|
|
374
415
|
trigger: "answered_questions" | "message" | "resume";
|
|
375
416
|
message_id?: string;
|
|
376
417
|
/**
|
|
@@ -508,6 +549,12 @@ export interface ProviderConfig {
|
|
|
508
549
|
* means the five-minute default.
|
|
509
550
|
*/
|
|
510
551
|
progress_event_min_interval_ms?: number;
|
|
552
|
+
/**
|
|
553
|
+
* Hold applied to an account after a 429 whose message names no reset
|
|
554
|
+
* time: the account is skipped for this long. Absent means the
|
|
555
|
+
* fifteen-minute default.
|
|
556
|
+
*/
|
|
557
|
+
quota_default_hold_ms?: number;
|
|
511
558
|
accounts: Record<string, AccountConfig>;
|
|
512
559
|
}
|
|
513
560
|
export interface ThrottleConfig {
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -198,13 +198,14 @@ export const LANE_FIELD_WRITERS = {
|
|
|
198
198
|
project_id: ["command"], story_id: ["command"], thread_id: ["creator", "command"],
|
|
199
199
|
depth: ["command"], senior: ["command"], senior_history: ["command"],
|
|
200
200
|
state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "reconcile"],
|
|
201
|
+
pickup_validation_pending: ["creator", "command", "spawn"],
|
|
201
202
|
priority: ["command"], worker: ["creator"], worker_path: ["creator"], worker_pack: ["creator"],
|
|
202
|
-
provider: ["creator"], account: ["creator", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
203
|
+
provider: ["creator"], account: ["creator", "spawn", "command"], account_history: ["command"], pending_resume: ["command", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
203
204
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
204
|
-
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn"], effort: ["creator", "spawn"], effort_source: ["creator", "spawn"],
|
|
205
|
+
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], effort: ["creator", "spawn"], effort_source: ["creator", "spawn"],
|
|
205
206
|
last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
206
207
|
overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
|
|
207
|
-
admitted_reason: ["reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn"],
|
|
208
|
+
admitted_reason: ["reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
208
209
|
quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
|
|
209
210
|
provider_session_id: ["watcher"], provider_session_id_absent: ["watcher"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], last_progress_event_at: ["watcher"],
|
|
210
211
|
};
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
*/
|
|
34
34
|
import { type LaneEnvelope, type LaneRecord, type LaneState } from "./types.js";
|
|
35
35
|
import { type QuotaSignal } from "./quota.js";
|
|
36
|
+
import { type ProviderRejection } from "./provider-rejection.js";
|
|
36
37
|
import { type LaneMesh } from "./mesh-events.js";
|
|
37
38
|
import { type EnvelopeRepairSpawnInput } from "./spawn.js";
|
|
38
39
|
/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
|
|
@@ -73,6 +74,7 @@ export interface WatcherTickInput {
|
|
|
73
74
|
invalidEnvelope?: {
|
|
74
75
|
errors: string[];
|
|
75
76
|
} | null;
|
|
77
|
+
providerRejection?: ProviderRejection | null;
|
|
76
78
|
}
|
|
77
79
|
export interface WatcherTickResult {
|
|
78
80
|
emit?: WatcherSignal;
|
|
@@ -97,7 +99,30 @@ export declare function classifyLaneExit(input: {
|
|
|
97
99
|
invalidEnvelope?: {
|
|
98
100
|
errors: string[];
|
|
99
101
|
} | null;
|
|
102
|
+
/**
|
|
103
|
+
* Set when the worker's output stream carries a provider-level turn
|
|
104
|
+
* failure (codex `turn.failed`, a Claude error result with an HTTP status,
|
|
105
|
+
* a grok error object). Failed, never interrupted: the provider refused
|
|
106
|
+
* the turn, so there is nothing to resume and no envelope to repair.
|
|
107
|
+
*/
|
|
108
|
+
providerRejection?: ProviderRejection | null;
|
|
100
109
|
}): TerminalClassification;
|
|
110
|
+
/**
|
|
111
|
+
* One decision for an exited worker's output. An explicit provider status
|
|
112
|
+
* wins over the quota text heuristics: a 429 from any provider (including
|
|
113
|
+
* codex `turn.failed` and grok error objects, which the claude-shaped quota
|
|
114
|
+
* reader never sees) becomes a quota signal; any other explicit status is a
|
|
115
|
+
* rejection even when its text sounds like a quota message (a 402 "requires
|
|
116
|
+
* usage credits" is a rejection, not a hold). A failure with no status at
|
|
117
|
+
* all is matched against the quota text heuristics first: a status-less
|
|
118
|
+
* "usage limit" / "rate limit" turn.failed still creates the account hold;
|
|
119
|
+
* any other status-less message is a rejection. Only when the stream
|
|
120
|
+
* carries no failure object does the quota reader's own read apply.
|
|
121
|
+
*/
|
|
122
|
+
export declare function routeProviderFailure(failure: ProviderRejection | null, readQuota: () => QuotaSignal | null, model: string | undefined, now: Date): {
|
|
123
|
+
quota: QuotaSignal | null;
|
|
124
|
+
rejection: ProviderRejection | null;
|
|
125
|
+
};
|
|
101
126
|
export declare function tickWatcher(input: WatcherTickInput): WatcherTickResult;
|
|
102
127
|
export declare function watcherEventsPath(hqRoot: string, laneId: string): string;
|
|
103
128
|
export declare function watcherOutLogPath(hqRoot: string, laneId: string): string;
|
|
@@ -119,6 +144,8 @@ export interface ApplyTerminalOpts {
|
|
|
119
144
|
envelopeRef?: string;
|
|
120
145
|
at?: string;
|
|
121
146
|
quota?: QuotaSignal | null;
|
|
147
|
+
/** Written to `terminal_details` with the model and account the run used. */
|
|
148
|
+
providerRejection?: ProviderRejection | null;
|
|
122
149
|
}
|
|
123
150
|
export declare function stalledReason(silenceMs: number): string;
|
|
124
151
|
/**
|
|
@@ -38,9 +38,10 @@ import { raiseEnvelopeQuestions } from "./questions.js";
|
|
|
38
38
|
import { isTerminalLaneState, parseLaneEnvelope, } from "./types.js";
|
|
39
39
|
import { readLane, retryUpdateLane } from "./store.js";
|
|
40
40
|
import { readWorkerRunResult } from "./delivery.js";
|
|
41
|
-
import { readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
41
|
+
import { applyDefaultQuotaHold, quotaSignalFromFields, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
42
|
+
import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
42
43
|
import { readToolActivity, toolActivityPath } from "./dropbox.js";
|
|
43
|
-
import { loadLanesConfig, providerProgressEventMinIntervalMs } from "./config.js";
|
|
44
|
+
import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
|
|
44
45
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, progressEventDue, } from "./mesh-events.js";
|
|
45
46
|
import { envelopeQuestionId } from "./questions.js";
|
|
46
47
|
import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, } from "./spawn.js";
|
|
@@ -86,6 +87,9 @@ export function classifyLaneExit(input) {
|
|
|
86
87
|
return { state: "blocked" };
|
|
87
88
|
throw new LanesError("unknown_envelope_decision", `Unknown envelope decision ${JSON.stringify(input.envelope.decision)}. Accepted: done, ask, blocked`, { accepted: ["done", "ask", "blocked"], value: input.envelope.decision });
|
|
88
89
|
}
|
|
90
|
+
if (input.providerRejection) {
|
|
91
|
+
return { state: "failed", reason: PROVIDER_REJECTED_REASON };
|
|
92
|
+
}
|
|
89
93
|
if (input.invalidEnvelope) {
|
|
90
94
|
return { state: "failed", reason: ENVELOPE_INVALID_REASON };
|
|
91
95
|
}
|
|
@@ -100,6 +104,34 @@ export function classifyLaneExit(input) {
|
|
|
100
104
|
}
|
|
101
105
|
return { state: "failed", reason: "exited_without_envelope" };
|
|
102
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* One decision for an exited worker's output. An explicit provider status
|
|
109
|
+
* wins over the quota text heuristics: a 429 from any provider (including
|
|
110
|
+
* codex `turn.failed` and grok error objects, which the claude-shaped quota
|
|
111
|
+
* reader never sees) becomes a quota signal; any other explicit status is a
|
|
112
|
+
* rejection even when its text sounds like a quota message (a 402 "requires
|
|
113
|
+
* usage credits" is a rejection, not a hold). A failure with no status at
|
|
114
|
+
* all is matched against the quota text heuristics first: a status-less
|
|
115
|
+
* "usage limit" / "rate limit" turn.failed still creates the account hold;
|
|
116
|
+
* any other status-less message is a rejection. Only when the stream
|
|
117
|
+
* carries no failure object does the quota reader's own read apply.
|
|
118
|
+
*/
|
|
119
|
+
export function routeProviderFailure(failure, readQuota, model, now) {
|
|
120
|
+
if (failure && failure.status === QUOTA_STATUS) {
|
|
121
|
+
return {
|
|
122
|
+
quota: quotaSignalFromFields({ api_error_status: QUOTA_STATUS, is_error: true, result: failure.message }, model, now),
|
|
123
|
+
rejection: null,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
if (failure && failure.status === null) {
|
|
127
|
+
const quota = quotaSignalFromFields({ api_error_status: undefined, is_error: true, result: failure.message }, model, now);
|
|
128
|
+
if (quota)
|
|
129
|
+
return { quota, rejection: null };
|
|
130
|
+
}
|
|
131
|
+
if (failure)
|
|
132
|
+
return { quota: null, rejection: failure };
|
|
133
|
+
return { quota: readQuota(), rejection: null };
|
|
134
|
+
}
|
|
103
135
|
export function tickWatcher(input) {
|
|
104
136
|
const silenceMs = input.silenceMs ?? SILENCE_MS;
|
|
105
137
|
const at = new Date(input.nowMs).toISOString();
|
|
@@ -109,6 +141,7 @@ export function tickWatcher(input) {
|
|
|
109
141
|
hasPartialOutput: input.hasPartialOutput,
|
|
110
142
|
quota: input.quota,
|
|
111
143
|
invalidEnvelope: input.invalidEnvelope,
|
|
144
|
+
providerRejection: input.providerRejection,
|
|
112
145
|
});
|
|
113
146
|
return {
|
|
114
147
|
emit: { kind: "terminal", at, classification },
|
|
@@ -301,6 +334,9 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
|
|
|
301
334
|
if (opts.quota?.marker) {
|
|
302
335
|
next.model_quota_marker = opts.quota.marker;
|
|
303
336
|
}
|
|
337
|
+
if (opts.providerRejection) {
|
|
338
|
+
next.terminal_details = providerRejectionDetails(opts.providerRejection, current);
|
|
339
|
+
}
|
|
304
340
|
if (opts.envelope && opts.envelopeRef) {
|
|
305
341
|
next.last_envelope = {
|
|
306
342
|
ref: opts.envelopeRef,
|
|
@@ -589,19 +625,37 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
|
589
625
|
const hasPartialOutput = logFiles.some((file) => fileNonEmpty(file));
|
|
590
626
|
let quota = null;
|
|
591
627
|
let quotaLane = null;
|
|
628
|
+
let providerRejection = null;
|
|
592
629
|
if (!processAlive && !envelope) {
|
|
593
630
|
try {
|
|
594
631
|
quotaLane = readLane(hqRoot, laneId);
|
|
595
|
-
|
|
632
|
+
const workerOutOffset = readSpawnManifest(runDir).worker_out_offset ?? 0;
|
|
633
|
+
const routed = routeProviderFailure(readProviderRejection(runDir, quotaLane.provider, workerOutOffset), () => readQuotaSignal(runDir, quotaLane.model, new Date(now()), workerOutOffset), quotaLane.model, new Date(now()));
|
|
634
|
+
// A reset-less 429 still holds the account: the configured default
|
|
635
|
+
// hold becomes its reset, recorded as such.
|
|
636
|
+
quota = routed.quota
|
|
637
|
+
? applyDefaultQuotaHold(routed.quota, providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[quotaLane.provider]), new Date(now()))
|
|
638
|
+
: null;
|
|
639
|
+
providerRejection = routed.rejection;
|
|
596
640
|
}
|
|
597
641
|
catch (err) {
|
|
598
642
|
process.stderr.write(`hq lanes watcher: quota read failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
599
643
|
}
|
|
600
644
|
}
|
|
601
645
|
// Spec §4: the process exited and its envelope file does not parse.
|
|
602
|
-
// Exactly one repair turn, then failed with both errors.
|
|
646
|
+
// Exactly one repair turn, then failed with both errors. A provider
|
|
647
|
+
// rejection skips the repair: the provider refused the turn, so a
|
|
648
|
+
// second turn on the same model and account would be refused the same
|
|
649
|
+
// way. A quota signal skips it too: a repair turn on an exhausted account
|
|
650
|
+
// fails the same way, and would end the lane envelope_invalid without
|
|
651
|
+
// ever recording the hold. The quota result applies immediately instead.
|
|
603
652
|
let invalidEnvelope = null;
|
|
604
|
-
if (!processAlive &&
|
|
653
|
+
if (!processAlive &&
|
|
654
|
+
!envelope &&
|
|
655
|
+
!providerRejection &&
|
|
656
|
+
!quota &&
|
|
657
|
+
latestFile &&
|
|
658
|
+
latestFile.envelope === null) {
|
|
605
659
|
const repair = decideEnvelopeRepair(hqRoot, laneId, latestFile, now(), spawnRepair, logDecision);
|
|
606
660
|
if (repair.action === "spawned") {
|
|
607
661
|
lastSignalMs = now();
|
|
@@ -626,6 +680,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
|
626
680
|
silenceMs,
|
|
627
681
|
quota,
|
|
628
682
|
invalidEnvelope,
|
|
683
|
+
providerRejection,
|
|
629
684
|
});
|
|
630
685
|
if (processAlive) {
|
|
631
686
|
maybeHeartbeatLane(hqRoot, laneId, now());
|
|
@@ -702,6 +757,13 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
|
702
757
|
state: classification.state,
|
|
703
758
|
reason: classification.reason,
|
|
704
759
|
...(invalidEnvelope ? { envelope_errors: invalidEnvelope.errors } : {}),
|
|
760
|
+
...(providerRejection
|
|
761
|
+
? {
|
|
762
|
+
provider_status: providerRejection.status,
|
|
763
|
+
provider_message: providerRejection.message,
|
|
764
|
+
retryable: providerRejection.retryable,
|
|
765
|
+
}
|
|
766
|
+
: {}),
|
|
705
767
|
});
|
|
706
768
|
if (quota &&
|
|
707
769
|
quotaLane?.account &&
|
|
@@ -719,6 +781,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
|
719
781
|
envelopeRef: latest?.ref,
|
|
720
782
|
at: tick.emit.at,
|
|
721
783
|
quota,
|
|
784
|
+
providerRejection,
|
|
722
785
|
});
|
|
723
786
|
const updated = terminalWrite.record;
|
|
724
787
|
logDecision({
|
|
@@ -49,15 +49,16 @@ export interface HeldSessionRef {
|
|
|
49
49
|
/** Task id carried on the event, if any. */
|
|
50
50
|
task?: string;
|
|
51
51
|
/**
|
|
52
|
-
* Per-event local evidence retained while coalescing a held session.
|
|
53
|
-
*
|
|
54
|
-
*
|
|
52
|
+
* Per-event local evidence retained while coalescing a held session. A
|
|
53
|
+
* session id may be reused. The current instance is state.bindingEpisodeId,
|
|
54
|
+
* never the event timestamp: a later `at` on another episode is still stale.
|
|
55
55
|
*/
|
|
56
56
|
eventEvidence?: Array<{
|
|
57
57
|
hqRoot?: string;
|
|
58
58
|
cwd?: string;
|
|
59
59
|
harness?: string;
|
|
60
60
|
at?: string;
|
|
61
|
+
bindingEpisodeId?: string;
|
|
61
62
|
}>;
|
|
62
63
|
}
|
|
63
64
|
export interface BackfillHeldResult {
|
|
@@ -163,9 +163,11 @@ function localCompanyCandidates(ref, state, canonicalHqRoot, canonicalHqRoots =
|
|
|
163
163
|
return candidates;
|
|
164
164
|
}
|
|
165
165
|
/**
|
|
166
|
-
* Keep only event evidence
|
|
167
|
-
*
|
|
168
|
-
*
|
|
166
|
+
* Keep only event evidence for the current session instance. When the state
|
|
167
|
+
* file has a binding episode, that id is the instance. A timestamp, including
|
|
168
|
+
* one later than startedAt, does not make another episode current.
|
|
169
|
+
* Sessions that predate episodes still use startedAt so an older event is
|
|
170
|
+
* not treated as this instance.
|
|
169
171
|
*/
|
|
170
172
|
function currentInstanceEventEvidence(ref, state) {
|
|
171
173
|
const evidence = ref.eventEvidence ?? [
|
|
@@ -176,6 +178,9 @@ function currentInstanceEventEvidence(ref, state) {
|
|
|
176
178
|
},
|
|
177
179
|
...(ref.hqRoots ?? []).filter((hqRoot) => hqRoot !== ref.hqRoot).map((hqRoot) => ({ hqRoot })),
|
|
178
180
|
];
|
|
181
|
+
if (state?.bindingEpisodeId) {
|
|
182
|
+
return evidence.filter((entry) => Boolean(entry.hqRoot) && entry.bindingEpisodeId === state.bindingEpisodeId);
|
|
183
|
+
}
|
|
179
184
|
if (!state?.startedAt)
|
|
180
185
|
return evidence;
|
|
181
186
|
const startedAt = Date.parse(state.startedAt);
|
|
@@ -188,8 +193,19 @@ function currentInstanceEventEvidence(ref, state) {
|
|
|
188
193
|
return Number.isFinite(eventAt) && eventAt >= startedAt;
|
|
189
194
|
});
|
|
190
195
|
}
|
|
196
|
+
/** Plain reason when every rooted held event belongs to another episode. */
|
|
197
|
+
function episodeMismatchReason(ref, state) {
|
|
198
|
+
if (!state?.bindingEpisodeId)
|
|
199
|
+
return null;
|
|
200
|
+
const rooted = (ref.eventEvidence ?? []).filter((entry) => entry.hqRoot);
|
|
201
|
+
if (rooted.length === 0)
|
|
202
|
+
return null;
|
|
203
|
+
if (rooted.some((entry) => entry.bindingEpisodeId === state.bindingEpisodeId))
|
|
204
|
+
return null;
|
|
205
|
+
return "episode does not match the current instance";
|
|
206
|
+
}
|
|
191
207
|
function hasOnlyStaleEventEvidence(ref, state) {
|
|
192
|
-
return Boolean(state?.startedAt &&
|
|
208
|
+
return Boolean((state?.bindingEpisodeId || state?.startedAt) &&
|
|
193
209
|
ref.eventEvidence?.some((entry) => entry.hqRoot) &&
|
|
194
210
|
!currentInstanceEventEvidence(ref, state).some((entry) => entry.hqRoot));
|
|
195
211
|
}
|
|
@@ -310,12 +326,18 @@ function latestAcknowledgedRegistration(sessionId, operations) {
|
|
|
310
326
|
function acknowledgedRegistrationMatches(operation, ref, evidence, state) {
|
|
311
327
|
if (!operation.companyUid || !operation.receiptId)
|
|
312
328
|
return false;
|
|
313
|
-
if (
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
329
|
+
if (state?.bindingEpisodeId) {
|
|
330
|
+
if (state.bindingEpisodeId !== operation.receiptId)
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
else {
|
|
334
|
+
if (!state?.startedAt)
|
|
335
|
+
return false;
|
|
336
|
+
const startedAt = Date.parse(state.startedAt);
|
|
337
|
+
const registeredAt = Date.parse(operation.createdAt);
|
|
338
|
+
if (!Number.isFinite(startedAt) || !Number.isFinite(registeredAt) || registeredAt < startedAt) {
|
|
339
|
+
return false;
|
|
340
|
+
}
|
|
319
341
|
}
|
|
320
342
|
if (evidence &&
|
|
321
343
|
!candidatesAgree(evidence, {
|
|
@@ -325,8 +347,6 @@ function acknowledgedRegistrationMatches(operation, ref, evidence, state) {
|
|
|
325
347
|
return false;
|
|
326
348
|
if (ref.harness && operation.harness && ref.harness !== operation.harness)
|
|
327
349
|
return false;
|
|
328
|
-
if (state?.bindingEpisodeId && state.bindingEpisodeId !== operation.receiptId)
|
|
329
|
-
return false;
|
|
330
350
|
return true;
|
|
331
351
|
}
|
|
332
352
|
/** Shared doctor/backfill gate for a receipt that is safe to treat as current evidence. */
|
|
@@ -407,8 +427,9 @@ export function readHeldSessions(workMeshRoot) {
|
|
|
407
427
|
if (task && !ref.task)
|
|
408
428
|
ref.task = task;
|
|
409
429
|
const at = str(event.at) ?? (envelope ? str(envelope.heldAt) : undefined);
|
|
430
|
+
const bindingEpisodeId = str(event.bindingEpisodeId);
|
|
410
431
|
if (hqRoot || cwd) {
|
|
411
|
-
ref.eventEvidence = [...(ref.eventEvidence ?? []), { hqRoot, cwd, harness, at }];
|
|
432
|
+
ref.eventEvidence = [...(ref.eventEvidence ?? []), { hqRoot, cwd, harness, at, bindingEpisodeId }];
|
|
412
433
|
}
|
|
413
434
|
seen.set(sessionId, ref);
|
|
414
435
|
}
|
|
@@ -481,7 +502,9 @@ export async function backfillHeldSessions(deps) {
|
|
|
481
502
|
if (!evidence.candidate?.slug && deps.useDeviceDefault !== true) {
|
|
482
503
|
result.unresolved += 1;
|
|
483
504
|
result.noEvidence += 1;
|
|
484
|
-
|
|
505
|
+
const staleReason = episodeMismatchReason(ref, state)
|
|
506
|
+
?? (hasOnlyStaleEventEvidence(ref, state) ? "STALE_EVIDENCE" : "no local company evidence");
|
|
507
|
+
deps.log?.(`would stay unresolved ${sessionId} (${staleReason})`);
|
|
485
508
|
continue;
|
|
486
509
|
}
|
|
487
510
|
const evidenceRef = {
|
|
@@ -126,7 +126,7 @@ export function collectDaemonDoctor(deps = {}) {
|
|
|
126
126
|
if (deadLetterCount > 0) {
|
|
127
127
|
unhealthyReasons.push(`dead-letter count=${deadLetterCount}`);
|
|
128
128
|
}
|
|
129
|
-
if (
|
|
129
|
+
if (heldCount > 0) {
|
|
130
130
|
unhealthyReasons.push(`held events awaiting company attribution=${heldCount}`);
|
|
131
131
|
}
|
|
132
132
|
if (heldWithNoLocalEvidence > 0) {
|
|
@@ -205,7 +205,10 @@ export function formatDaemonDoctor(report) {
|
|
|
205
205
|
if (report.emitMode === "legacy") {
|
|
206
206
|
// Legacy emit path only: the spool/held/outbox queues do not exist in
|
|
207
207
|
// receive-only (direct) mode.
|
|
208
|
-
lines.push(`spool depth: ${report.spoolDepth}`, `held: ${report.heldCount}`, `held with no local evidence: ${report.heldWithNoLocalEvidence}`, `outbox depth: ${report.outboxDepth}`);
|
|
208
|
+
lines.push(`spool depth: ${report.spoolDepth}`, `held: ${report.heldCount}${report.heldCount > 0 ? " (needs_company; not sent as NONE)" : ""}`, `held with no local evidence: ${report.heldWithNoLocalEvidence}`, `outbox depth: ${report.outboxDepth}`);
|
|
209
|
+
}
|
|
210
|
+
if (report.emitMode !== "legacy" && report.heldCount > 0) {
|
|
211
|
+
lines.push(`held: ${report.heldCount} (needs_company; not sent as NONE)`);
|
|
209
212
|
}
|
|
210
213
|
lines.push(`attributed by device default: ${report.attributedByDeviceDefault}`, `device-default authorization denials: ${report.deviceDefaultAuthorizationDenials}`, `dead-letter: ${report.deadLetterCount}`, `unattributed events: ${report.unattributedEvents.total} ` +
|
|
211
214
|
`(spool=${report.unattributedEvents.spool} held=${report.unattributedEvents.held} dead-letter=${report.unattributedEvents.deadLetter})`, `token source: ${report.tokenSource}`, `actor kind: ${report.actorKind}`, `identity whoami: ${report.whoamiSubject}`, `identity daemon: ${report.daemonSubject}`);
|
|
@@ -101,7 +101,12 @@ export declare function restartDaemonServiceIfInstalled(opts?: {
|
|
|
101
101
|
platform?: DaemonInstallPlatform;
|
|
102
102
|
home?: string;
|
|
103
103
|
uid?: number;
|
|
104
|
+
/** HQ root the daemon was serving, or the root this update was run from. */
|
|
105
|
+
hqRoot?: string;
|
|
106
|
+
env?: NodeJS.ProcessEnv;
|
|
104
107
|
existsSync?: typeof fs.existsSync;
|
|
108
|
+
readFileSync?: typeof fs.readFileSync;
|
|
109
|
+
writeFileSync?: typeof fs.writeFileSync;
|
|
105
110
|
run?: (command: string, args: string[]) => {
|
|
106
111
|
status: number | null;
|
|
107
112
|
};
|
|
@@ -17,7 +17,7 @@ import * as path from "node:path";
|
|
|
17
17
|
import { spawnSync } from "node:child_process";
|
|
18
18
|
import { resolveDaemonAssetDir } from "./log.js";
|
|
19
19
|
import { LAUNCHD_LABEL, SYSTEMD_UNIT_NAME, WINDOWS_TASK_NAME, daemonDir, daemonLogPath, daemonPidPath, } from "./paths.js";
|
|
20
|
-
import { isInstallableHqRoot, readDaemonConfig, registerTrustedHqRoot } from "./config.js";
|
|
20
|
+
import { isInstallableHqRoot, readDaemonConfig, registerTrustedHqRoot, verifyHqRoot } from "./config.js";
|
|
21
21
|
import { defaultPidLockDeps, pidLockStatus, readPidLock } from "./pid-lock.js";
|
|
22
22
|
/**
|
|
23
23
|
* `schtasks /End` terminates the task's root process only. The daemon is
|
|
@@ -49,6 +49,86 @@ export const defaultSchtasks = (args) => {
|
|
|
49
49
|
stderr: r.error ? r.error.message : (r.stderr ?? ""),
|
|
50
50
|
};
|
|
51
51
|
};
|
|
52
|
+
function readServiceText(file, existsSync, readFileSync) {
|
|
53
|
+
if (!existsSync(file))
|
|
54
|
+
return "";
|
|
55
|
+
const raw = readFileSync(file);
|
|
56
|
+
const buf = Buffer.isBuffer(raw) ? raw : Buffer.from(String(raw));
|
|
57
|
+
if ((buf.length >= 2 && buf[0] === 0xff && buf[1] === 0xfe) || buf.includes(0)) {
|
|
58
|
+
return buf.toString("utf16le").replace(/^\uFEFF/, "");
|
|
59
|
+
}
|
|
60
|
+
return buf.toString("utf8");
|
|
61
|
+
}
|
|
62
|
+
function hqRootFromUnitText(platform, text) {
|
|
63
|
+
if (platform === "win32")
|
|
64
|
+
return parseWindowsTaskHqRoot(text);
|
|
65
|
+
if (platform === "darwin") {
|
|
66
|
+
const match = /<key>HQ_ROOT<\/key>\s*<string>([^<]*)<\/string>/.exec(text);
|
|
67
|
+
return match?.[1] ?? null;
|
|
68
|
+
}
|
|
69
|
+
const match = /^Environment=HQ_ROOT=(.*)$/m.exec(text);
|
|
70
|
+
if (!match)
|
|
71
|
+
return null;
|
|
72
|
+
const value = match[1].trim();
|
|
73
|
+
return value.startsWith("'") && value.endsWith("'") ? value.slice(1, -1) : value;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Record the HQ root the daemon was serving and replace a pre-allowlist unit
|
|
77
|
+
* before the process is restarted. A unit with no HQ_ROOT (or Restart=on-failure)
|
|
78
|
+
* comes back with an empty root allowlist unless this runs first.
|
|
79
|
+
*/
|
|
80
|
+
function migrateLegacyDaemonUnit(opts) {
|
|
81
|
+
const { platform, home } = opts;
|
|
82
|
+
if (platform !== "darwin" && platform !== "linux" && platform !== "win32")
|
|
83
|
+
return;
|
|
84
|
+
const unitPath = platform === "darwin"
|
|
85
|
+
? path.join(home, "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`)
|
|
86
|
+
: platform === "linux"
|
|
87
|
+
? path.join(home, ".config", "systemd", "user", SYSTEMD_UNIT_NAME)
|
|
88
|
+
: path.join(daemonDir(undefined, home, opts.env), `${WINDOWS_TASK_NAME}.xml`);
|
|
89
|
+
if (!opts.existsSync(unitPath))
|
|
90
|
+
return;
|
|
91
|
+
const text = readServiceText(unitPath, opts.existsSync, opts.readFileSync);
|
|
92
|
+
const fromUnit = hqRootFromUnitText(platform, text);
|
|
93
|
+
const verified = verifyHqRoot(fromUnit ?? undefined)
|
|
94
|
+
?? verifyHqRoot(opts.hqRoot)
|
|
95
|
+
?? verifyHqRoot(opts.env?.HQ_ROOT);
|
|
96
|
+
if (!verified)
|
|
97
|
+
return;
|
|
98
|
+
registerTrustedHqRoot(daemonDir(undefined, home, opts.env), verified);
|
|
99
|
+
const legacy = platform === "linux"
|
|
100
|
+
? !text.includes("Environment=HQ_ROOT=") || /Restart=on-failure/.test(text)
|
|
101
|
+
: platform === "darwin"
|
|
102
|
+
? !/<key>HQ_ROOT<\/key>/.test(text)
|
|
103
|
+
: !fromUnit;
|
|
104
|
+
if (!legacy)
|
|
105
|
+
return;
|
|
106
|
+
const paths = buildInstallPaths({
|
|
107
|
+
home,
|
|
108
|
+
env: opts.env,
|
|
109
|
+
hqRoot: verified,
|
|
110
|
+
hqBinary: "/usr/bin/hq",
|
|
111
|
+
nodeBinary: process.execPath || "/usr/bin/node",
|
|
112
|
+
});
|
|
113
|
+
try {
|
|
114
|
+
paths.hqBinary = resolveHqBinary(process.argv, opts.env ?? process.env);
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
// The rendered command only has to name a binary; install already resolved one.
|
|
118
|
+
}
|
|
119
|
+
if (platform === "darwin") {
|
|
120
|
+
opts.writeFileSync(unitPath, renderLaunchdPlist(paths), { mode: 0o600 });
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
if (platform === "linux") {
|
|
124
|
+
opts.writeFileSync(unitPath, renderSystemdUserUnit(paths), { mode: 0o600 });
|
|
125
|
+
opts.run("systemctl", ["--user", "daemon-reload"]);
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
const rendered = renderWindowsTaskXml(paths);
|
|
129
|
+
opts.writeFileSync(unitPath, encodeWindowsTaskXml(rendered));
|
|
130
|
+
opts.run("schtasks.exe", ["/Create", "/TN", WINDOWS_TASK_NAME, "/XML", unitPath, "/F"]);
|
|
131
|
+
}
|
|
52
132
|
/**
|
|
53
133
|
* Restart the installed user daemon after a CLI upgrade. This is deliberately
|
|
54
134
|
* best-effort: an update must remain successful if a user has no service,
|
|
@@ -58,8 +138,11 @@ export function restartDaemonServiceIfInstalled(opts = {}) {
|
|
|
58
138
|
const platform = opts.platform ?? detectPlatform();
|
|
59
139
|
const home = opts.home ?? os.homedir();
|
|
60
140
|
const existsSync = opts.existsSync ?? fs.existsSync;
|
|
141
|
+
const readFileSync = opts.readFileSync ?? fs.readFileSync;
|
|
142
|
+
const writeFileSync = opts.writeFileSync ?? fs.writeFileSync;
|
|
61
143
|
const run = opts.run ?? ((command, args) => spawnSync(command, args, { stdio: "ignore" }));
|
|
62
144
|
try {
|
|
145
|
+
migrateLegacyDaemonUnit({ platform, home, hqRoot: opts.hqRoot, env: opts.env, existsSync, readFileSync, writeFileSync, run });
|
|
63
146
|
if (platform === "darwin") {
|
|
64
147
|
const plist = path.join(home, "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`);
|
|
65
148
|
if (!existsSync(plist))
|
|
@@ -90,6 +90,11 @@ export interface DaemonRunDeps {
|
|
|
90
90
|
resolveMembership?: (token: string, slug: string, signal?: AbortSignal) => Promise<{
|
|
91
91
|
companyUid: string;
|
|
92
92
|
} | null>;
|
|
93
|
+
/**
|
|
94
|
+
* Machine credential (fleet box or local bot). A refused or empty membership
|
|
95
|
+
* probe defers; it does not clear the company. Defaults to isMachineIdentity().
|
|
96
|
+
*/
|
|
97
|
+
machineIdentity?: boolean;
|
|
93
98
|
/** Test seam immediately before a device-default revocation's guarded commit. */
|
|
94
99
|
beforeDeviceDefaultRevocationCommit?: () => void;
|
|
95
100
|
log?: (dir: string, line: string) => void;
|
|
@@ -14,7 +14,7 @@ import * as path from "node:path";
|
|
|
14
14
|
import { CLI_VERSION } from "../../../../cli-version.js";
|
|
15
15
|
import { createWorkSessionDeliverer } from "../../client.js";
|
|
16
16
|
import { listActiveMembershipCompanies, meshJson, resolveActiveMembershipCompany } from "../../api.js";
|
|
17
|
-
import { ensureCognitoToken } from "../../../../utils/cognito-session.js";
|
|
17
|
+
import { ensureCognitoToken, isMachineIdentity } from "../../../../utils/cognito-session.js";
|
|
18
18
|
import { replayOutbox } from "../../../work-context/outbox.js";
|
|
19
19
|
import { workContextRoot } from "../../../work-context/paths.js";
|
|
20
20
|
import { ensureDefaultCompany } from "../../../work-context/default-company.js";
|
|
@@ -143,6 +143,7 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
143
143
|
const defaultGetToken = async () => ensureCognitoToken({ interactive: false, tokenSource: "machine" });
|
|
144
144
|
const getToken = deps.getToken ?? defaultGetToken;
|
|
145
145
|
const resolveMembership = deps.resolveMembership ?? resolveActiveMembershipCompany;
|
|
146
|
+
const machineIdentity = deps.machineIdentity ?? isMachineIdentity();
|
|
146
147
|
/** The sole authorization seam for daemon tenant I/O derived from a device default. */
|
|
147
148
|
const tenantGate = {
|
|
148
149
|
authorize: async (input) => {
|
|
@@ -161,9 +162,18 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
161
162
|
authorization = await Promise.race([
|
|
162
163
|
(async () => {
|
|
163
164
|
const membership = await resolveMembership(await getToken(), slug, controller.signal);
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
165
|
+
if (membership)
|
|
166
|
+
return { status: "authorized", uid: membership.companyUid };
|
|
167
|
+
// A machine token often cannot read /membership/me (401/403, or an
|
|
168
|
+
// empty person-membership list). That is "cannot tell", not "not
|
|
169
|
+
// a member". Leave the company in place and retry.
|
|
170
|
+
if (machineIdentity) {
|
|
171
|
+
return {
|
|
172
|
+
status: "unavailable",
|
|
173
|
+
error: "membership probe on a machine token did not confirm a company; company left unchanged",
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
return { status: "denied" };
|
|
167
177
|
})(),
|
|
168
178
|
new Promise((resolve) => {
|
|
169
179
|
controller.signal.addEventListener("abort", () => resolve({
|