@indigoai-us/hq-cli 5.345.155 → 5.345.159
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +289 -0
- package/assets/scaffold/core/scripts/rebuild-company-knowledge-index.sh +110 -2
- package/assets/scaffold/core/scripts/rebuild-projects-index.sh +101 -5
- package/assets/scaffold/core/scripts/worktree.sh +4 -0
- package/dist/command-catalog.generated.d.ts +536 -7
- package/dist/command-catalog.generated.js +676 -24
- package/dist/command-registration-plan.d.ts +18 -0
- package/dist/command-registration-plan.js +4 -0
- package/dist/commands/_patterns.d.ts +17 -0
- package/dist/commands/_patterns.js +22 -0
- package/dist/commands/agent-inbox.js +2 -2
- package/dist/commands/agent-kit.js +8 -6
- package/dist/commands/bot.js +7 -4
- package/dist/commands/channel-suggest.d.ts +56 -0
- package/dist/commands/channel-suggest.js +170 -0
- package/dist/commands/channels.js +7 -1
- package/dist/commands/cloud.d.ts +45 -0
- package/dist/commands/cloud.js +150 -10
- package/dist/commands/company.js +4 -3
- package/dist/commands/core-checkpoint-native.d.ts +18 -0
- package/dist/commands/core-checkpoint-native.js +67 -2
- package/dist/commands/core-pipeline-driver-native.d.ts +12 -1
- package/dist/commands/core-pipeline-driver-native.js +131 -5
- package/dist/commands/core.js +13 -1
- package/dist/commands/crm.js +83 -6
- package/dist/commands/deploy-access.d.ts +33 -0
- package/dist/commands/deploy-access.js +322 -0
- package/dist/commands/dm-attach.d.ts +106 -0
- package/dist/commands/dm-attach.js +184 -0
- package/dist/commands/dm.d.ts +37 -0
- package/dist/commands/dm.js +172 -10
- package/dist/commands/feedback-show.d.ts +40 -0
- package/dist/commands/feedback-show.js +261 -0
- package/dist/commands/feedback.d.ts +21 -0
- package/dist/commands/feedback.js +152 -2
- package/dist/commands/files-browse.d.ts +16 -0
- package/dist/commands/files-browse.js +52 -1
- package/dist/commands/files.d.ts +24 -0
- package/dist/commands/files.js +52 -1
- package/dist/commands/integrations-connect.js +7 -3
- package/dist/commands/integrations-core.d.ts +13 -1
- package/dist/commands/integrations-core.js +49 -4
- package/dist/commands/integrations.js +4 -2
- package/dist/commands/lanes-accounts.js +16 -1
- package/dist/commands/lanes-loop.js +14 -4
- package/dist/commands/lanes-message.js +6 -4
- package/dist/commands/lanes-resume.js +36 -13
- package/dist/commands/lanes-shared.d.ts +5 -0
- package/dist/commands/lanes-shared.js +13 -3
- package/dist/commands/lanes-watch.js +1 -0
- package/dist/commands/lanes.d.ts +20 -1
- package/dist/commands/lanes.js +201 -51
- package/dist/commands/mesh.js +149 -1
- package/dist/commands/monitor.d.ts +1 -0
- package/dist/commands/monitor.js +4 -1
- package/dist/commands/repo.d.ts +41 -0
- package/dist/commands/repo.js +331 -0
- package/dist/commands/scan.d.ts +3 -1
- package/dist/commands/scan.js +7 -3
- package/dist/commands/secrets.d.ts +37 -0
- package/dist/commands/secrets.js +185 -16
- package/dist/commands/signals.js +17 -5
- package/dist/commands/staff/listings.d.ts +41 -0
- package/dist/commands/staff/listings.js +160 -0
- package/dist/commands/staff/reset-test-account.d.ts +3 -2
- package/dist/commands/staff/reset-test-account.js +4 -3
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +2629 -1036
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/agent-kit/inbox-state.d.ts +7 -0
- package/dist/lib/agent-kit/inbox-state.js +5 -1
- package/dist/lib/agent-kit/mcp/tools.js +2 -1
- package/dist/lib/agent-kit/run/inbox.js +2 -1
- package/dist/lib/agent-kit/wake-secrets.d.ts +7 -0
- package/dist/lib/agent-kit/wake-secrets.js +32 -0
- package/dist/lib/agent-kit/wake.d.ts +4 -0
- package/dist/lib/agent-kit/wake.js +49 -3
- package/dist/lib/bot/inflight.d.ts +7 -1
- package/dist/lib/bot/inflight.js +10 -2
- package/dist/lib/bot/prompt.js +3 -0
- package/dist/lib/bot/run.js +72 -20
- package/dist/lib/bot/session.d.ts +14 -0
- package/dist/lib/bot/session.js +70 -0
- package/dist/lib/capacity/collectors/claude-oauth.d.ts +6 -0
- package/dist/lib/capacity/collectors/claude-oauth.js +12 -0
- package/dist/lib/capacity/lane-login.d.ts +6 -0
- package/dist/lib/capacity/lane-login.js +5 -0
- package/dist/lib/core-utils/worktree.js +2 -0
- package/dist/lib/daemon/exit-cause.d.ts +43 -0
- package/dist/lib/daemon/exit-cause.js +88 -0
- package/dist/lib/daemon/handoff.d.ts +2 -0
- package/dist/lib/daemon/handoff.js +6 -1
- package/dist/lib/daemon/hq-child-bootstrap.mjs +20 -3
- package/dist/lib/daemon/paths.d.ts +2 -0
- package/dist/lib/daemon/paths.js +1 -0
- package/dist/lib/daemon/run.js +13 -0
- package/dist/lib/daemon/supervisor.d.ts +54 -1
- package/dist/lib/daemon/supervisor.js +297 -21
- package/dist/lib/daemon/unit-holds.d.ts +22 -0
- package/dist/lib/daemon/unit-holds.js +86 -0
- package/dist/lib/doctor/checks/sync-health.d.ts +9 -0
- package/dist/lib/doctor/checks/sync-health.js +66 -1
- package/dist/lib/doctor/company-resolution-state.d.ts +30 -0
- package/dist/lib/doctor/company-resolution-state.js +89 -0
- package/dist/lib/doctor/fix/apply.d.ts +36 -6
- package/dist/lib/doctor/fix/apply.js +128 -12
- package/dist/lib/doctor/types.d.ts +1 -1
- package/dist/lib/doctor/types.js +1 -0
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/hook-engine/resolve.js +6 -2
- package/dist/lib/index-render/company-knowledge.d.ts +1 -1
- package/dist/lib/index-render/company-knowledge.js +4 -5
- package/dist/lib/index-render/index.js +44 -3
- package/dist/lib/index-render/projects.d.ts +1 -1
- package/dist/lib/index-render/projects.js +5 -5
- package/dist/lib/lanes/account-company.d.ts +28 -0
- package/dist/lib/lanes/account-company.js +53 -0
- package/dist/lib/lanes/accounts.d.ts +22 -3
- package/dist/lib/lanes/accounts.js +38 -2
- package/dist/lib/lanes/claude-balance.d.ts +6 -1
- package/dist/lib/lanes/claude-balance.js +6 -1
- package/dist/lib/lanes/claude-credential.d.ts +78 -0
- package/dist/lib/lanes/claude-credential.js +163 -0
- package/dist/lib/lanes/compaction.d.ts +2 -0
- package/dist/lib/lanes/compaction.js +9 -0
- package/dist/lib/lanes/config.js +6 -0
- package/dist/lib/lanes/envelope-repair-dispatch.js +2 -1
- package/dist/lib/lanes/hooks-install.js +2 -1
- package/dist/lib/lanes/host-capacity.d.ts +2 -0
- package/dist/lib/lanes/host-capacity.js +1 -0
- package/dist/lib/lanes/keep-alive.d.ts +1 -1
- package/dist/lib/lanes/keep-alive.js +3 -1
- package/dist/lib/lanes/links.js +10 -4
- package/dist/lib/lanes/loop.js +16 -1
- package/dist/lib/lanes/node-binary.d.ts +17 -0
- package/dist/lib/lanes/node-binary.js +30 -0
- package/dist/lib/lanes/outpost-dispatch.d.ts +2 -1
- package/dist/lib/lanes/outpost-dispatch.js +1 -0
- package/dist/lib/lanes/reconcile.js +4 -0
- package/dist/lib/lanes/remote-control-lane.js +2 -0
- package/dist/lib/lanes/spawn.d.ts +3 -0
- package/dist/lib/lanes/spawn.js +119 -9
- package/dist/lib/lanes/store.d.ts +6 -0
- package/dist/lib/lanes/store.js +107 -18
- package/dist/lib/lanes/types.d.ts +2 -0
- package/dist/lib/lanes/watch.js +28 -5
- package/dist/lib/lanes/watcher-main.js +4 -3
- package/dist/lib/lanes/watcher.d.ts +4 -0
- package/dist/lib/lanes/watcher.js +110 -21
- package/dist/lib/mesh/api.d.ts +32 -0
- package/dist/lib/mesh/api.js +75 -0
- package/dist/lib/secret-scan/detect.js +9 -0
- package/dist/lib/secret-scan/repo-config.d.ts +41 -0
- package/dist/lib/secret-scan/repo-config.js +237 -0
- package/dist/lib/secret-scan/rules.d.ts +7 -1
- package/dist/lib/secret-scan/rules.js +3 -1
- package/dist/lib/secret-scan/staged.d.ts +5 -0
- package/dist/lib/secret-scan/staged.js +19 -2
- package/dist/lib/secret-scan/types.d.ts +2 -0
- package/dist/lib/sync/server-owned-local-only.d.ts +29 -0
- package/dist/lib/sync/server-owned-local-only.js +75 -0
- package/dist/sentry-before-send.js +1 -0
- package/dist/utils/access-denied-hint.js +6 -0
- package/dist/utils/client-health.d.ts +34 -0
- package/dist/utils/client-health.js +96 -3
- package/dist/utils/package-use-lease.js +27 -9
- package/dist/utils/version-gate.d.ts +7 -0
- package/dist/utils/version-gate.js +66 -0
- package/package.json +8 -8
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -7,19 +7,22 @@
|
|
|
7
7
|
* probed for existence and never read. Identity in the child is `HQ_LANE_ID`;
|
|
8
8
|
* mutable fields are re-read from the lane record (spec §5.4).
|
|
9
9
|
*/
|
|
10
|
+
import { accountMatchesCompany, accountMayRetainLane, companyMismatchError } from "./account-company.js";
|
|
10
11
|
import { spawn, spawnSync } from "node:child_process";
|
|
11
12
|
import { randomUUID } from "node:crypto";
|
|
12
13
|
import * as fs from "node:fs";
|
|
13
14
|
import * as path from "node:path";
|
|
14
15
|
import { fileURLToPath } from "node:url";
|
|
15
16
|
import { configureRemoteControlChildEnv, prepareRemoteControlLane, remoteControlSelectionConfig } from "./remote-control-lane.js";
|
|
16
|
-
import { PROVIDER_HOME_ENV, accountStatus, isDefaultProviderHome, isLoggedOut, resolveAccount, selectBalancedAccount, } from "./accounts.js";
|
|
17
|
+
import { PROVIDER_HOME_ENV, accountStatus, isDefaultProviderHome, isLoggedOut, isSignInExpired, resolveAccount, selectBalancedAccount, } from "./accounts.js";
|
|
18
|
+
import { readLaneClaudeCredential } from "./claude-credential.js";
|
|
17
19
|
import { effectiveAccountHome } from "./account-home.js";
|
|
18
20
|
import { buildHostCapacitySnapshot } from "./host-capacity.js";
|
|
19
21
|
import { selectClaudeAccount } from "./claude-balance.js";
|
|
20
22
|
import { loadQuotaState } from "./quota.js";
|
|
21
23
|
import { COMPACTION_PROMPT, compactionAfterUsage, compactionDue, compactLaneSession, } from "./compaction.js";
|
|
22
24
|
import { installDrainHook } from "./hooks-install.js";
|
|
25
|
+
import { liveNodeBinary } from "./node-binary.js";
|
|
23
26
|
import { loadLanesConfig, providerAutoCompactTokens, providerCompactAfterMs, } from "./config.js";
|
|
24
27
|
import { appendEffortFlag, resolveLaneEffort, } from "./effort.js";
|
|
25
28
|
import { defaultProbe, validateLaneModel, validateProfileLaneModel, } from "./models.js";
|
|
@@ -252,9 +255,19 @@ function selectSpawnAccount(input) {
|
|
|
252
255
|
};
|
|
253
256
|
if (lane.account === undefined) {
|
|
254
257
|
const modelSpecified = pinnedModel !== undefined || profileModel !== undefined;
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
258
|
+
const company = resolveLaneCompany(hqRoot, lane).company;
|
|
259
|
+
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
260
|
+
.filter(([id, account]) => accountMatchesCompany(account, company, config.providers[lane.provider].accounts) &&
|
|
261
|
+
(!modelSpecified || supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps, hostname)))
|
|
262
|
+
.map(([id]) => id);
|
|
263
|
+
const mismatch = companyMismatchError({
|
|
264
|
+
provider: lane.provider,
|
|
265
|
+
providerAccounts: config.providers[lane.provider].accounts,
|
|
266
|
+
candidates: Object.keys(config.providers[lane.provider].accounts),
|
|
267
|
+
company,
|
|
268
|
+
});
|
|
269
|
+
if (mismatch !== undefined)
|
|
270
|
+
throw mismatch;
|
|
258
271
|
return {
|
|
259
272
|
account: selectBalancedAccount(lane.provider, config, {
|
|
260
273
|
...context,
|
|
@@ -269,7 +282,13 @@ function selectSpawnAccount(input) {
|
|
|
269
282
|
let refusal;
|
|
270
283
|
const operatorMovePending = lane.pending_resume !== undefined &&
|
|
271
284
|
lane.account_history?.at(-1)?.reason === "operator_move";
|
|
272
|
-
if (current
|
|
285
|
+
if (current !== undefined && !accountMayRetainLane(current, resolveLaneCompany(hqRoot, lane).company, config.providers[lane.provider].accounts)) {
|
|
286
|
+
reason = "account_unavailable";
|
|
287
|
+
refusal = new LanesError("account_company_mismatch", `Account ${lane.account} belongs to another company`, {
|
|
288
|
+
provider: lane.provider, account: lane.account,
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
else if (current === undefined || !current.enabled) {
|
|
273
292
|
reason = "account_unavailable";
|
|
274
293
|
refusal = new AccountResolutionError(current === undefined ? "unknown_account" : "account_disabled", `Account ${JSON.stringify(lane.account)} is unavailable for ${lane.provider}`, { provider: lane.provider, account: lane.account });
|
|
275
294
|
}
|
|
@@ -278,11 +297,18 @@ function selectSpawnAccount(input) {
|
|
|
278
297
|
existsSync,
|
|
279
298
|
hostname,
|
|
280
299
|
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
300
|
+
...(accountDeps?.readClaudeCredential ? { readClaudeCredential: accountDeps.readClaudeCredential } : {}),
|
|
281
301
|
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
282
302
|
});
|
|
283
303
|
if (isLoggedOut(status)) {
|
|
284
304
|
reason = repairRecordedAccount ? "recorded_account_logged_out" : "account_unavailable";
|
|
285
|
-
refusal = new LanesError("account_not_logged_in",
|
|
305
|
+
refusal = new LanesError("account_not_logged_in", isSignInExpired(status)
|
|
306
|
+
? expiredSignInMessage(lane.provider, lane.account, status)
|
|
307
|
+
: `Account ${lane.account} for ${lane.provider} is not logged in`, {
|
|
308
|
+
provider: lane.provider,
|
|
309
|
+
account: lane.account,
|
|
310
|
+
...(isSignInExpired(status) ? { sign_in_state: "expired", sign_in_expires_at: status.sign_in_expires_at ?? null } : {}),
|
|
311
|
+
});
|
|
286
312
|
}
|
|
287
313
|
else {
|
|
288
314
|
try {
|
|
@@ -339,6 +365,7 @@ function selectSpawnAccount(input) {
|
|
|
339
365
|
config,
|
|
340
366
|
existsSync,
|
|
341
367
|
probe: accountDeps?.probe,
|
|
368
|
+
readClaudeCredential: accountDeps?.readClaudeCredential,
|
|
342
369
|
env: accountDeps?.env,
|
|
343
370
|
hostname,
|
|
344
371
|
now,
|
|
@@ -348,6 +375,7 @@ function selectSpawnAccount(input) {
|
|
|
348
375
|
targetHost: lane.host,
|
|
349
376
|
actualTargetHost: lane.host,
|
|
350
377
|
requestedPlacement: null,
|
|
378
|
+
company: resolveLaneCompany(hqRoot, lane).company,
|
|
351
379
|
config,
|
|
352
380
|
quotaState: loadQuotaState(hqRoot),
|
|
353
381
|
...((recordedModel ?? pinnedModel ?? profileModel) !== undefined
|
|
@@ -366,11 +394,13 @@ function selectSpawnAccount(input) {
|
|
|
366
394
|
};
|
|
367
395
|
}
|
|
368
396
|
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
397
|
+
.filter(([, account]) => accountMatchesCompany(account, resolveLaneCompany(hqRoot, lane).company, config.providers[lane.provider].accounts))
|
|
369
398
|
.filter(([id, account]) => id !== lane.account && account.enabled)
|
|
370
399
|
.filter(([id]) => accountStatus(lane.provider, id, config, {
|
|
371
400
|
existsSync,
|
|
372
401
|
hostname,
|
|
373
402
|
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
403
|
+
...(accountDeps?.readClaudeCredential ? { readClaudeCredential: accountDeps.readClaudeCredential } : {}),
|
|
374
404
|
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
375
405
|
}).logged_in === true)
|
|
376
406
|
.filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps, hostname))
|
|
@@ -406,6 +436,11 @@ function selectSpawnAccount(input) {
|
|
|
406
436
|
account_reasons: accountReasons,
|
|
407
437
|
});
|
|
408
438
|
}
|
|
439
|
+
if (lane.company !== null && lane.state === "queued" && lane.pending_resume !== undefined && err instanceof AccountResolutionError) {
|
|
440
|
+
throw new LanesError("same_company_account_unavailable", "No eligible account assigned to the lane's company is available", {
|
|
441
|
+
provider: lane.provider, account: lane.account, company: lane.company,
|
|
442
|
+
});
|
|
443
|
+
}
|
|
409
444
|
if (reason === "account_unavailable" || reason === "recorded_account_logged_out") {
|
|
410
445
|
if (refusal instanceof Error)
|
|
411
446
|
throw refusal;
|
|
@@ -545,6 +580,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
545
580
|
const accountDeps = {
|
|
546
581
|
probe: modelDeps.probe ?? defaultProbe,
|
|
547
582
|
env,
|
|
583
|
+
...spawnCredentialReader(deps, modelDeps),
|
|
548
584
|
};
|
|
549
585
|
const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
|
|
550
586
|
let quotaFailoverEnabled = false;
|
|
@@ -572,6 +608,9 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
572
608
|
: undefined;
|
|
573
609
|
const unavailableAccounts = previouslyUnavailableAccounts(lane);
|
|
574
610
|
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
611
|
+
.filter(([id, account]) => id === lane.account
|
|
612
|
+
? accountMayRetainLane(account, resolveLaneCompany(hqRoot, lane).company, config.providers[lane.provider].accounts)
|
|
613
|
+
: accountMatchesCompany(account, resolveLaneCompany(hqRoot, lane).company, config.providers[lane.provider].accounts))
|
|
575
614
|
.filter(([id, account]) => (id !== lane.account || (quotaFailoverEnabled && reason === "usage_limit" && retryingExpiredQuota)) &&
|
|
576
615
|
!unavailableAccounts.has(id) && account.enabled)
|
|
577
616
|
.filter(([id]) => !quotaFailoverEnabled ||
|
|
@@ -583,6 +622,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
583
622
|
existsSync,
|
|
584
623
|
hostname,
|
|
585
624
|
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
625
|
+
...(accountDeps?.readClaudeCredential ? { readClaudeCredential: accountDeps.readClaudeCredential } : {}),
|
|
586
626
|
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
587
627
|
}).logged_in === true)
|
|
588
628
|
.filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps, hostname))
|
|
@@ -635,6 +675,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
635
675
|
existsSync,
|
|
636
676
|
hostname,
|
|
637
677
|
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
678
|
+
...(accountDeps?.readClaudeCredential ? { readClaudeCredential: accountDeps.readClaudeCredential } : {}),
|
|
638
679
|
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
639
680
|
}).logged_in !== true)
|
|
640
681
|
return { account: id, reason: "login_required" };
|
|
@@ -666,6 +707,22 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
666
707
|
return retryUpdateLane(hqRoot, lane.lane_id, writer, holdLane);
|
|
667
708
|
}
|
|
668
709
|
}
|
|
710
|
+
if (resolveLaneCompany(hqRoot, lane).company !== null && err instanceof AccountResolutionError) {
|
|
711
|
+
const waitForAccount = (current) => {
|
|
712
|
+
if (current.state !== retryFromState || current.account !== lane.account || current.provider_session_id !== lane.provider_session_id) {
|
|
713
|
+
throw new LanesError("account_retry_conflict", "Lane changed while company account recovery was prepared", { lane_id: lane.lane_id });
|
|
714
|
+
}
|
|
715
|
+
return {
|
|
716
|
+
...current, state: "queued", queued_reason: "account_retry", queued_details: undefined,
|
|
717
|
+
queued_at: at, admission_reserved_at: undefined, terminal_reason: undefined,
|
|
718
|
+
terminal_details: undefined, interrupted_at: undefined,
|
|
719
|
+
pending_resume: current.provider_session_id
|
|
720
|
+
? pendingResumeForRetry(hqRoot, current, current.provider_session_id, at)
|
|
721
|
+
: undefined,
|
|
722
|
+
};
|
|
723
|
+
};
|
|
724
|
+
return deps.dryRun ? waitForAccount(lane) : retryUpdateLane(hqRoot, lane.lane_id, writer, waitForAccount);
|
|
725
|
+
}
|
|
669
726
|
if (reason === "account_unavailable" || unavailableAccounts.size > 0) {
|
|
670
727
|
throw new LanesError("account_not_logged_in", `Account ${lane.account ?? "(none)"} for ${lane.provider} is unavailable and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account ?? null });
|
|
671
728
|
}
|
|
@@ -845,6 +902,21 @@ function appendCodexWorkerPolicyDirFlag(provider, args, workerPolicyDir) {
|
|
|
845
902
|
return [...args, "-c", setting];
|
|
846
903
|
return [...args.slice(0, separator), "-c", setting, ...args.slice(separator)];
|
|
847
904
|
}
|
|
905
|
+
/**
|
|
906
|
+
* The Claude credential expiry check rides with the real provider probe: a
|
|
907
|
+
* caller on the production probe also gets the real Keychain/file read, and a
|
|
908
|
+
* test that injects a fake probe gets no credential read unless it passes one.
|
|
909
|
+
*/
|
|
910
|
+
function spawnCredentialReader(deps, modelDeps) {
|
|
911
|
+
if (deps.readClaudeCredential !== undefined)
|
|
912
|
+
return { readClaudeCredential: deps.readClaudeCredential };
|
|
913
|
+
return modelDeps.probe === defaultProbe ? { readClaudeCredential: readLaneClaudeCredential } : {};
|
|
914
|
+
}
|
|
915
|
+
function expiredSignInMessage(provider, account, status) {
|
|
916
|
+
const at = status.sign_in_expires_at ? ` at ${status.sign_in_expires_at}` : "";
|
|
917
|
+
return `Account ${account} for ${provider} has an expired Claude sign-in (access token expired${at}). ` +
|
|
918
|
+
`Open Claude Code on this account and run /login, then retry.`;
|
|
919
|
+
}
|
|
848
920
|
function accountHomeHostname(deps) {
|
|
849
921
|
return (deps.hostname?.trim() ||
|
|
850
922
|
deps.admissionHost?.hostname.trim() ||
|
|
@@ -1958,7 +2030,7 @@ function spawnTrackedWorker(spawnProcess, command, args, env, cwd, stdoutPath, s
|
|
|
1958
2030
|
throw error;
|
|
1959
2031
|
}
|
|
1960
2032
|
const runnerPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../dist/lib/lanes/worker-process.js");
|
|
1961
|
-
return spawnDetached(spawnProcess,
|
|
2033
|
+
return spawnDetached(spawnProcess, liveNodeBinary(), [runnerPath, command, ...args], { ...env, [LANE_WORKER_EXIT_PATH_ENV]: exitPath }, cwd, stdoutPath, stderrPath);
|
|
1962
2034
|
}
|
|
1963
2035
|
function requireDetached(proveDetached, pid, kind) {
|
|
1964
2036
|
const proof = proveDetached(pid);
|
|
@@ -2093,7 +2165,7 @@ export function armLaneWatcher(hqRoot, laneId, deps = {}) {
|
|
|
2093
2165
|
if (!cliEntry) {
|
|
2094
2166
|
throw new LanesError("watcher_cli_entry_unavailable", "Cannot arm the lane watcher without the current hq CLI entrypoint", { lane_id: laneId });
|
|
2095
2167
|
}
|
|
2096
|
-
const watcherPid = spawnDetached(spawnProcess,
|
|
2168
|
+
const watcherPid = spawnDetached(spawnProcess, liveNodeBinary(), [watcherMainPath(), laneId], {
|
|
2097
2169
|
...env,
|
|
2098
2170
|
HQ_ROOT: hqRoot,
|
|
2099
2171
|
HQ_LANE_ID: laneId,
|
|
@@ -2332,6 +2404,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2332
2404
|
const accountDeps = {
|
|
2333
2405
|
probe: modelDeps.probe ?? defaultProbe,
|
|
2334
2406
|
env,
|
|
2407
|
+
...spawnCredentialReader(deps, modelDeps),
|
|
2335
2408
|
};
|
|
2336
2409
|
const diskAccount = laneDiskAccount(config, existing.provider, existing.account);
|
|
2337
2410
|
if (diskAccount !== undefined) {
|
|
@@ -2342,6 +2415,22 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2342
2415
|
...current,
|
|
2343
2416
|
state: "queued",
|
|
2344
2417
|
queued_reason: "low_disk",
|
|
2418
|
+
// Commit an explicit resume's instruction together with the hold. A
|
|
2419
|
+
// later admission must continue that turn, not start the old brief.
|
|
2420
|
+
pending_resume: trigger.kind === "resume" ? {
|
|
2421
|
+
...(existing.provider_session_id !== undefined ? { session_id: existing.provider_session_id } : {}),
|
|
2422
|
+
trigger: "resume",
|
|
2423
|
+
caller: typeof env.HQ_LANE_ID === "string" ? env.HQ_LANE_ID.trim() || "user" : "user",
|
|
2424
|
+
resumed_from: trigger.from,
|
|
2425
|
+
...(trigger.fromFailed === true ? { from_failed: true } : {}),
|
|
2426
|
+
...(trigger.expectedSenior !== undefined ? { expected_senior: trigger.expectedSenior } : {}),
|
|
2427
|
+
...(trigger.text !== undefined ? { text: trigger.text } : {}),
|
|
2428
|
+
...(trigger.message_id !== undefined ? { message_id: trigger.message_id } : {}),
|
|
2429
|
+
...(trigger.fresh === true ? { fresh_thread: true } : {}),
|
|
2430
|
+
...(deps.workingDirectory !== undefined ? { working_directory: deps.workingDirectory } : {}),
|
|
2431
|
+
compact: launch.compact !== false,
|
|
2432
|
+
at: measuredAt,
|
|
2433
|
+
} : current.pending_resume,
|
|
2345
2434
|
queued_details: {
|
|
2346
2435
|
reason: "low_disk",
|
|
2347
2436
|
hq_free_bytes: free.hq.free_bytes,
|
|
@@ -2422,7 +2511,28 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2422
2511
|
if (preservedAccountModel !== undefined)
|
|
2423
2512
|
pinnedModel = preservedAccountModel;
|
|
2424
2513
|
const resolved = selection.account;
|
|
2425
|
-
const status = accountStatus(lane.provider, resolved.id, config, {
|
|
2514
|
+
const status = accountStatus(lane.provider, resolved.id, config, {
|
|
2515
|
+
existsSync,
|
|
2516
|
+
hostname,
|
|
2517
|
+
...(accountDeps?.readClaudeCredential ? { readClaudeCredential: accountDeps.readClaudeCredential } : {}),
|
|
2518
|
+
});
|
|
2519
|
+
if (isSignInExpired(status)) {
|
|
2520
|
+
const failed = markPreSpawnFailure
|
|
2521
|
+
? markSpawnFailure(hqRoot, lane.lane_id, "account_not_logged_in", {
|
|
2522
|
+
provider: lane.provider,
|
|
2523
|
+
account: resolved.id,
|
|
2524
|
+
sign_in_state: "expired",
|
|
2525
|
+
})
|
|
2526
|
+
: existing;
|
|
2527
|
+
throw new LanesError("account_not_logged_in", expiredSignInMessage(lane.provider, resolved.id, status), {
|
|
2528
|
+
provider: lane.provider,
|
|
2529
|
+
account: resolved.id,
|
|
2530
|
+
lane_id: lane.lane_id,
|
|
2531
|
+
sign_in_state: "expired",
|
|
2532
|
+
sign_in_expires_at: status.sign_in_expires_at ?? null,
|
|
2533
|
+
lane: failed,
|
|
2534
|
+
});
|
|
2535
|
+
}
|
|
2426
2536
|
if (isLoggedOut(status)) {
|
|
2427
2537
|
const failed = markPreSpawnFailure
|
|
2428
2538
|
? markSpawnFailure(hqRoot, lane.lane_id, "account_not_logged_in", {
|
|
@@ -24,6 +24,10 @@ export interface StoreDeps {
|
|
|
24
24
|
closeSync: typeof fs.closeSync;
|
|
25
25
|
fsyncSync: typeof fs.fsyncSync;
|
|
26
26
|
chmodSync: typeof fs.chmodSync;
|
|
27
|
+
/** Optional so existing fakes keep working; defaults to node:fs. */
|
|
28
|
+
statSync?: (p: string) => fs.Stats;
|
|
29
|
+
fstatSync?: (fd: number) => fs.Stats;
|
|
30
|
+
linkSync?: typeof fs.linkSync;
|
|
27
31
|
}
|
|
28
32
|
export declare function resolveStoreHqRoot(hqRoot?: string, env?: NodeJS.ProcessEnv, cwd?: string): string;
|
|
29
33
|
export declare function lanesStoreRoot(hqRoot?: string): string;
|
|
@@ -35,6 +39,8 @@ export declare function sanitizeHostComponent(hostname: string): string;
|
|
|
35
39
|
export declare function mintLaneId(deps?: Pick<StoreDeps, "now" | "hostname"> & Partial<Pick<StoreDeps, "generateLaneUlid">>): string;
|
|
36
40
|
export declare function mintQuestionId(deps?: Pick<StoreDeps, "now" | "hostname">): string;
|
|
37
41
|
export declare function appendCappedHistory<T>(history: readonly T[], entry: T, cap?: number): T[];
|
|
42
|
+
/** Test seam: run withRecordLock with overridden deps and a shorter timeout. */
|
|
43
|
+
export declare function withRecordLockForTests<T>(recordPath: string, overrides: Partial<StoreDeps>, fn: () => T, timeoutMs?: number): T;
|
|
38
44
|
/** Run a lane-owned file read/modify/write transaction under its sidecar lock. */
|
|
39
45
|
export declare function withLaneRecordFileLock<T>(recordPath: string, fn: () => T): T;
|
|
40
46
|
export declare function parseLaneRecord(raw: unknown, filePath?: string): LaneRecord;
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -24,7 +24,13 @@ export const LANES_STORE_REL = path.join("workspace", "lanes");
|
|
|
24
24
|
export const LANE_RECORDS_DIR = "lanes";
|
|
25
25
|
export const QUESTION_RECORDS_DIR = "questions";
|
|
26
26
|
export const PENDING_INDEX_DIR = path.join("questions", "index", "pending");
|
|
27
|
-
const
|
|
27
|
+
const DEFAULT_LOCK_STALE_MS = 30_000;
|
|
28
|
+
/** Lock age after which any holder is presumed gone. HQ_LANES_LOCK_STALE_MS overrides it. */
|
|
29
|
+
function lockStaleMs(env = process.env) {
|
|
30
|
+
const raw = env.HQ_LANES_LOCK_STALE_MS;
|
|
31
|
+
const parsed = raw === undefined ? NaN : Number(raw);
|
|
32
|
+
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : DEFAULT_LOCK_STALE_MS;
|
|
33
|
+
}
|
|
28
34
|
const LOCK_RETRY_MS = 10;
|
|
29
35
|
const LOCK_TIMEOUT_MS = 5_000;
|
|
30
36
|
function defaultStoreDeps() {
|
|
@@ -153,16 +159,104 @@ function wait(ms) {
|
|
|
153
159
|
function lockPathFor(recordPath) {
|
|
154
160
|
return `${recordPath}.lock`;
|
|
155
161
|
}
|
|
156
|
-
function
|
|
162
|
+
function lockOwnerAlive(pid) {
|
|
163
|
+
try {
|
|
164
|
+
process.kill(pid, 0);
|
|
165
|
+
return true;
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
return err.code !== "ESRCH";
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
function sameFile(a, b) {
|
|
172
|
+
return a.ino === b.ino && a.dev === b.dev;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Remove a record lock that no live process holds, and report whether it was
|
|
176
|
+
* removed. A lock is stale when it was written on this host and its owner pid
|
|
177
|
+
* no longer exists, or when it is older than the stale window. Age is the
|
|
178
|
+
* larger of the timestamp age and the file mtime age, so a future or
|
|
179
|
+
* unparseable timestamp still ages out through mtime.
|
|
180
|
+
*
|
|
181
|
+
* Removal is atomic: the lock is renamed to a unique name, which only one
|
|
182
|
+
* reclaimer can win. If the renamed file is not the lock that was inspected
|
|
183
|
+
* (another process reclaimed it and took a fresh lock in between), the fresh
|
|
184
|
+
* lock is linked back into place and nothing is reclaimed.
|
|
185
|
+
*/
|
|
186
|
+
function recordLockIsStale(lockPath, deps) {
|
|
187
|
+
const statSync = deps.statSync ?? ((p) => fs.statSync(p));
|
|
188
|
+
const linkSync = deps.linkSync ?? fs.linkSync.bind(fs);
|
|
189
|
+
const before = statSync(lockPath);
|
|
190
|
+
const raw = deps.readFileSync(lockPath, "utf8");
|
|
191
|
+
const [pidLine, tsLine, hostLine] = raw.split("\n");
|
|
192
|
+
const pid = pidLine ? Number(pidLine) : NaN;
|
|
193
|
+
const ts = tsLine ? Number(tsLine) : NaN;
|
|
194
|
+
const pidKnown = Number.isSafeInteger(pid) && pid > 0;
|
|
195
|
+
// A pid is only meaningful on the host that wrote it. A lock with a foreign
|
|
196
|
+
// or missing host falls back to the age rule.
|
|
197
|
+
const sameHost = hostLine !== undefined && hostLine.trim() !== "" && hostLine.trim() === deps.hostname();
|
|
198
|
+
const now = Date.now();
|
|
199
|
+
const mtimeAge = now - before.mtimeMs;
|
|
200
|
+
const age = Number.isFinite(ts) ? Math.max(now - ts, mtimeAge) : mtimeAge;
|
|
201
|
+
const stale = (pidKnown && sameHost && pid !== process.pid && !lockOwnerAlive(pid)) ||
|
|
202
|
+
age > lockStaleMs();
|
|
203
|
+
if (!stale)
|
|
204
|
+
return false;
|
|
205
|
+
const parked = `${lockPath}.stale.${process.pid}.${randomBytes(6).toString("hex")}`;
|
|
206
|
+
try {
|
|
207
|
+
deps.renameSync(lockPath, parked);
|
|
208
|
+
}
|
|
209
|
+
catch (renameErr) {
|
|
210
|
+
// Another reclaimer won the rename, or the holder released it.
|
|
211
|
+
if (renameErr.code === "ENOENT")
|
|
212
|
+
return false;
|
|
213
|
+
throw renameErr;
|
|
214
|
+
}
|
|
215
|
+
const removeParked = () => {
|
|
216
|
+
try {
|
|
217
|
+
deps.unlinkSync(parked);
|
|
218
|
+
}
|
|
219
|
+
catch (unlinkErr) {
|
|
220
|
+
if (unlinkErr.code !== "ENOENT")
|
|
221
|
+
throw unlinkErr;
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
let reclaimed;
|
|
225
|
+
try {
|
|
226
|
+
reclaimed = sameFile(statSync(parked), before) && deps.readFileSync(parked, "utf8") === raw;
|
|
227
|
+
if (!reclaimed) {
|
|
228
|
+
// The parked file is a live holder's fresh lock. Put it back.
|
|
229
|
+
try {
|
|
230
|
+
linkSync(parked, lockPath);
|
|
231
|
+
}
|
|
232
|
+
catch (linkErr) {
|
|
233
|
+
if (linkErr.code !== "EEXIST")
|
|
234
|
+
throw linkErr;
|
|
235
|
+
process.stderr.write(`lanes store: could not restore displaced lock ${lockPath}; a newer lock already exists\n`);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
catch (err) {
|
|
240
|
+
removeParked();
|
|
241
|
+
throw err;
|
|
242
|
+
}
|
|
243
|
+
removeParked();
|
|
244
|
+
return reclaimed;
|
|
245
|
+
}
|
|
246
|
+
function withRecordLock(recordPath, deps, fn, timeoutMs = LOCK_TIMEOUT_MS) {
|
|
157
247
|
const lockPath = lockPathFor(recordPath);
|
|
248
|
+
const statSync = deps.statSync ?? ((p) => fs.statSync(p));
|
|
249
|
+
const fstatSync = deps.fstatSync ?? ((fd) => fs.fstatSync(fd));
|
|
158
250
|
ensureDir(path.dirname(recordPath), deps);
|
|
159
251
|
const started = Date.now();
|
|
160
252
|
while (true) {
|
|
161
253
|
let fd;
|
|
162
254
|
try {
|
|
163
255
|
fd = deps.openSync(lockPath, "wx");
|
|
256
|
+
let held;
|
|
164
257
|
try {
|
|
165
|
-
fs.writeSync(fd, Buffer.from(`${process.pid}\n${Date.now()}\n`, "utf8"));
|
|
258
|
+
fs.writeSync(fd, Buffer.from(`${process.pid}\n${Date.now()}\n${deps.hostname()}\n`, "utf8"));
|
|
259
|
+
held = fstatSync(fd);
|
|
166
260
|
}
|
|
167
261
|
catch (err) {
|
|
168
262
|
deps.closeSync(fd);
|
|
@@ -180,7 +274,10 @@ function withRecordLock(recordPath, deps, fn) {
|
|
|
180
274
|
process.stderr.write(`lanes store: lock close failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
181
275
|
}
|
|
182
276
|
try {
|
|
183
|
-
|
|
277
|
+
// Unlink only our own lock. If it was reclaimed and replaced while
|
|
278
|
+
// fn ran, the file at lockPath belongs to the successor.
|
|
279
|
+
if (sameFile(statSync(lockPath), held))
|
|
280
|
+
deps.unlinkSync(lockPath);
|
|
184
281
|
}
|
|
185
282
|
catch (err) {
|
|
186
283
|
const code = err.code;
|
|
@@ -194,24 +291,12 @@ function withRecordLock(recordPath, deps, fn) {
|
|
|
194
291
|
const code = err.code;
|
|
195
292
|
if (code !== "EEXIST")
|
|
196
293
|
throw err;
|
|
197
|
-
if (Date.now() - started >
|
|
294
|
+
if (Date.now() - started > timeoutMs) {
|
|
198
295
|
throw new LanesError("store_lock_timeout", `Timed out waiting for lock ${lockPath}`, { path: lockPath });
|
|
199
296
|
}
|
|
200
297
|
try {
|
|
201
|
-
|
|
202
|
-
const tsLine = raw.split("\n")[1];
|
|
203
|
-
const ts = tsLine ? Number(tsLine) : NaN;
|
|
204
|
-
if (Number.isFinite(ts) && Date.now() - ts > LOCK_STALE_MS) {
|
|
205
|
-
try {
|
|
206
|
-
deps.unlinkSync(lockPath);
|
|
207
|
-
}
|
|
208
|
-
catch (unlinkErr) {
|
|
209
|
-
const unlinkCode = unlinkErr.code;
|
|
210
|
-
if (unlinkCode !== "ENOENT")
|
|
211
|
-
throw unlinkErr;
|
|
212
|
-
}
|
|
298
|
+
if (recordLockIsStale(lockPath, deps))
|
|
213
299
|
continue;
|
|
214
|
-
}
|
|
215
300
|
}
|
|
216
301
|
catch (statErr) {
|
|
217
302
|
const statCode = statErr.code;
|
|
@@ -223,6 +308,10 @@ function withRecordLock(recordPath, deps, fn) {
|
|
|
223
308
|
}
|
|
224
309
|
}
|
|
225
310
|
}
|
|
311
|
+
/** Test seam: run withRecordLock with overridden deps and a shorter timeout. */
|
|
312
|
+
export function withRecordLockForTests(recordPath, overrides, fn, timeoutMs) {
|
|
313
|
+
return withRecordLock(recordPath, { ...defaultStoreDeps(), ...overrides }, fn, timeoutMs);
|
|
314
|
+
}
|
|
226
315
|
/** Run a lane-owned file read/modify/write transaction under its sidecar lock. */
|
|
227
316
|
export function withLaneRecordFileLock(recordPath, fn) {
|
|
228
317
|
return withRecordLock(recordPath, defaultStoreDeps(), fn);
|
|
@@ -1029,6 +1029,8 @@ export interface LaneEnvelope {
|
|
|
1029
1029
|
fingerprint?: string | null;
|
|
1030
1030
|
}
|
|
1031
1031
|
export interface AccountConfig {
|
|
1032
|
+
/** Company slug or UID owning this account. Untagged accounts are personal. */
|
|
1033
|
+
company?: string;
|
|
1032
1034
|
home: string;
|
|
1033
1035
|
/** Optional account home overrides keyed by the owning host's hostname. */
|
|
1034
1036
|
hosts?: Record<string, AccountHostHome>;
|
package/dist/lib/lanes/watch.js
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import * as path from "node:path";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { loopControllerIsAlive, loopStatus } from "./loop.js";
|
|
4
|
+
import { laneRunDir } from "./spawn.js";
|
|
1
5
|
import { expandLocalLaneIdPrefix, listQuestionIds, readLane, readQuestion, } from "./store.js";
|
|
2
6
|
import { LaneNotFoundError, LanesError } from "./errors.js";
|
|
3
7
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
@@ -118,6 +122,7 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
|
118
122
|
while (true) {
|
|
119
123
|
polls += 1;
|
|
120
124
|
const records = [];
|
|
125
|
+
const activeLoops = new Set();
|
|
121
126
|
for (const laneId of watchedIds) {
|
|
122
127
|
let record = null;
|
|
123
128
|
try {
|
|
@@ -131,9 +136,27 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
|
131
136
|
continue;
|
|
132
137
|
}
|
|
133
138
|
records.push(record);
|
|
134
|
-
if (
|
|
135
|
-
|
|
136
|
-
|
|
139
|
+
if (record.loop_mode === true) {
|
|
140
|
+
try {
|
|
141
|
+
const runDir = laneRunDir(hqRoot, record.lane_id);
|
|
142
|
+
// loopStatus defaults missing metadata to waiting; that is not
|
|
143
|
+
// evidence of a controller after a failed admission or launch.
|
|
144
|
+
if (fs.existsSync(path.join(runDir, "loop", "loop.json"))) {
|
|
145
|
+
const status = loopStatus(runDir, record.lane_id);
|
|
146
|
+
if (status.state !== "stopped" && status.state !== "failed" && loopControllerIsAlive(runDir))
|
|
147
|
+
activeLoops.add(record.lane_id);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
unreadable("loop", laneId, err);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
// A phase can be done while its loop controller awaits more work.
|
|
155
|
+
const terminal = isTerminalLaneState(record.state) && !activeLoops.has(record.lane_id);
|
|
156
|
+
const stateKey = `${record.state}:${terminal}`;
|
|
157
|
+
if (previousStates.get(laneId) !== stateKey) {
|
|
158
|
+
emit(`lane ${record.lane_id}: state=${record.state}${terminal ? " (terminal)" : ""}`);
|
|
159
|
+
previousStates.set(laneId, stateKey);
|
|
137
160
|
}
|
|
138
161
|
}
|
|
139
162
|
const keepAliveWaiting = new Set();
|
|
@@ -175,7 +198,7 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
|
175
198
|
}
|
|
176
199
|
if (deps.readAccountStatuses !== undefined) {
|
|
177
200
|
const accountLanes = records.filter((record) => record.account !== undefined &&
|
|
178
|
-
(!isTerminalLaneState(record.state) || keepAliveWaiting.has(record.lane_id)));
|
|
201
|
+
(!isTerminalLaneState(record.state) || activeLoops.has(record.lane_id) || keepAliveWaiting.has(record.lane_id)));
|
|
179
202
|
let statuses;
|
|
180
203
|
try {
|
|
181
204
|
statuses = await deps.readAccountStatuses(accountLanes);
|
|
@@ -242,7 +265,7 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
|
242
265
|
previousAccountLogin.set(key, status.logged_in);
|
|
243
266
|
}
|
|
244
267
|
}
|
|
245
|
-
if (records.every((record) => isTerminalLaneState(record.state) && !keepAliveWaiting.has(record.lane_id))) {
|
|
268
|
+
if (records.every((record) => isTerminalLaneState(record.state) && !activeLoops.has(record.lane_id) && !keepAliveWaiting.has(record.lane_id))) {
|
|
246
269
|
return { polls, lines, suppressed };
|
|
247
270
|
}
|
|
248
271
|
await sleep(intervalMs);
|
|
@@ -12,6 +12,7 @@ import { isMainModule } from "../../utils/is-main-module.js";
|
|
|
12
12
|
import { runWatcherLoop } from "./watcher.js";
|
|
13
13
|
import { spawnEnvelopeRepairViaCurrentCli, } from "./envelope-repair-dispatch.js";
|
|
14
14
|
import { LanesError } from "./errors.js";
|
|
15
|
+
import { liveNodeBinary } from "./node-binary.js";
|
|
15
16
|
import { openMeshTransport } from "../mesh/client.js";
|
|
16
17
|
import { CompanySelectionError } from "../../utils/company-selection-error.js";
|
|
17
18
|
export function watcherMainDeps(hqRoot, laneId, env = process.env, repairDispatchDeps = {}) {
|
|
@@ -21,7 +22,7 @@ export function watcherMainDeps(hqRoot, laneId, env = process.env, repairDispatc
|
|
|
21
22
|
spawnRepair: (root, lane, input) => spawnEnvelopeRepairViaCurrentCli(root, lane, input, { ...repairDispatchDeps, env }),
|
|
22
23
|
deliverLaneSeniorMessage: async (seniorLaneId, text) => deliverLaneSeniorMessage(cliEntry, seniorLaneId, text),
|
|
23
24
|
recoverQueuedMessages: async () => {
|
|
24
|
-
const result = spawnSync(
|
|
25
|
+
const result = spawnSync(liveNodeBinary(), [cliEntry, "lanes", "_recover-queued-messages", laneId], {
|
|
25
26
|
env,
|
|
26
27
|
encoding: "utf8",
|
|
27
28
|
timeout: 120_000,
|
|
@@ -34,7 +35,7 @@ export function watcherMainDeps(hqRoot, laneId, env = process.env, repairDispatc
|
|
|
34
35
|
},
|
|
35
36
|
scheduleTerminalKeepAlive: async ({ laneId: scheduledLaneId, version, dueAt }) => {
|
|
36
37
|
await new Promise((resolve, reject) => {
|
|
37
|
-
const child = spawn(
|
|
38
|
+
const child = spawn(liveNodeBinary(), [
|
|
38
39
|
cliEntry,
|
|
39
40
|
"lanes",
|
|
40
41
|
"_keep-alive-terminal-trigger",
|
|
@@ -54,7 +55,7 @@ export function watcherMainDeps(hqRoot, laneId, env = process.env, repairDispatc
|
|
|
54
55
|
};
|
|
55
56
|
}
|
|
56
57
|
function deliverLaneSeniorMessage(cliEntry, laneId, text) {
|
|
57
|
-
const result = spawnSync(
|
|
58
|
+
const result = spawnSync(liveNodeBinary(), [cliEntry, "lanes", "message", laneId, "--text", text], {
|
|
58
59
|
env: process.env,
|
|
59
60
|
encoding: "utf8",
|
|
60
61
|
timeout: 120_000,
|
|
@@ -201,6 +201,8 @@ export interface WatcherLoopDeps {
|
|
|
201
201
|
progressEventMinIntervalMs?: number;
|
|
202
202
|
/** Bound a hung mesh request so watcher liveness polling continues. */
|
|
203
203
|
meshRequestTimeoutMs?: number;
|
|
204
|
+
/** Bound on waiting for an older run's watcher to release the claim. */
|
|
205
|
+
claimWaitMs?: number;
|
|
204
206
|
/** Spec §4 repair spawn. Default: `spawnEnvelopeRepair`. */
|
|
205
207
|
spawnRepair?: (hqRoot: string, lane: LaneRecord, input: EnvelopeRepairSpawnInput) => number;
|
|
206
208
|
/** Rebind and respawn after a provider quota or authentication refusal. */
|
|
@@ -247,6 +249,8 @@ export interface WatcherLoopDeps {
|
|
|
247
249
|
export declare function emitMeshSoft(event: string, logDecision: (record: Record<string, unknown>) => void, at: string, fn: (signal: AbortSignal) => Promise<void>, timeoutMs?: number): Promise<boolean>;
|
|
248
250
|
/** Question ids an ask envelope raised, as `raiseEnvelopeQuestions` mints them. */
|
|
249
251
|
export declare function envelopeQuestionIds(laneId: string, envelopeRef: string, envelope: LaneEnvelope, questionIndexes?: readonly number[]): string[];
|
|
252
|
+
/** How long a new phase watcher waits for an older phase's watcher to let go. */
|
|
253
|
+
export declare const WATCHER_CLAIM_WAIT_MS = 60000;
|
|
250
254
|
/** Detached and synchronous classifiers share one process-identity claim. */
|
|
251
255
|
export declare function runWatcherLoop(hqRoot: string, laneId: string, deps?: WatcherLoopDeps): Promise<void>;
|
|
252
256
|
export type EnvelopeRepairDecision = {
|