@indigoai-us/hq-cli 5.345.61 → 5.345.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/command-catalog.generated.d.ts +736 -0
- package/dist/command-catalog.generated.js +907 -0
- package/dist/commands/core-hq-session-native.js +5 -1
- package/dist/commands/daemon.js +6 -1
- package/dist/commands/lanes-link.d.ts +3 -0
- package/dist/commands/lanes-link.js +83 -0
- package/dist/commands/lanes-loop.js +5 -6
- package/dist/commands/lanes-questions.js +27 -3
- package/dist/commands/lanes.js +26 -4
- package/dist/lib/daemon/hq-cloud.d.ts +13 -1
- package/dist/lib/daemon/hq-cloud.js +79 -16
- package/dist/lib/lanes/links.d.ts +39 -0
- package/dist/lib/lanes/links.js +450 -0
- package/dist/lib/lanes/loop.d.ts +7 -2
- package/dist/lib/lanes/loop.js +169 -57
- package/dist/utils/client-health-contract.d.ts +12 -0
- package/dist/utils/client-health-contract.js +51 -0
- package/dist/utils/version-gate.js +5 -1
- package/package.json +1 -1
package/dist/lib/lanes/loop.js
CHANGED
|
@@ -305,6 +305,9 @@ function fileEnvelope(file) {
|
|
|
305
305
|
if (!isRecord(value) || value.schema_version !== LOOP_SCHEMA || !nonEmpty(value.envelope_id) || !nonEmpty(value.queued_at)) {
|
|
306
306
|
throw new Error(`Invalid queued loop envelope record: ${file}`);
|
|
307
307
|
}
|
|
308
|
+
if (value.stall_attempts !== undefined && (!Number.isSafeInteger(value.stall_attempts) || Number(value.stall_attempts) < 0)) {
|
|
309
|
+
throw new Error(`Invalid stall_attempts in queued loop envelope record: ${file}`);
|
|
310
|
+
}
|
|
308
311
|
return { ...value, envelope: validateLoopEnvelope(value.envelope) };
|
|
309
312
|
}
|
|
310
313
|
/** Queue depth and FIFO ordering are based on the timestamp-prefixed filename. */
|
|
@@ -329,34 +332,41 @@ export function enqueueLoopEnvelope(input) {
|
|
|
329
332
|
fs.mkdirSync(dir, { recursive: true });
|
|
330
333
|
const release = acquireEnqueueLock(input.runDir);
|
|
331
334
|
try {
|
|
332
|
-
const
|
|
333
|
-
|
|
334
|
-
|
|
335
|
+
const withdrawnDir = path.join(input.runDir, "loop", "withdrawn");
|
|
336
|
+
const stalledDir = path.join(input.runDir, "loop", "stalled");
|
|
337
|
+
const records = [pendingDir(input.runDir), activeDir(input.runDir), claimedDir(input.runDir), stalledDir, withdrawnDir]
|
|
338
|
+
.flatMap((dir) => envelopeFiles(dir).map((file) => ({ file, dir })))
|
|
339
|
+
.map(({ file, dir }) => {
|
|
335
340
|
try {
|
|
336
|
-
return fileEnvelope(file);
|
|
341
|
+
return { file, dir, record: fileEnvelope(file) };
|
|
337
342
|
}
|
|
338
343
|
catch (error) {
|
|
339
344
|
process.stderr.write(`hq lanes: ignored unreadable loop record ${file}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
340
345
|
return undefined;
|
|
341
346
|
}
|
|
342
347
|
})
|
|
343
|
-
.filter((
|
|
344
|
-
const
|
|
345
|
-
const stalledDir = path.join(input.runDir, "loop", "stalled");
|
|
346
|
-
const reusableRecords = records.filter((record) => record.envelope_id === envelopeId);
|
|
348
|
+
.filter((entry) => entry !== undefined);
|
|
349
|
+
const reusableRecords = records.filter(({ record }) => record.envelope_id === envelopeId);
|
|
347
350
|
const terminalResult = fs.existsSync(path.join(resultsDir(input.runDir), resultName(envelopeId)))
|
|
348
351
|
? readJson(path.join(resultsDir(input.runDir), resultName(envelopeId))) : undefined;
|
|
349
|
-
const reusable = reusableRecords.length > 0 && reusableRecords.every((
|
|
352
|
+
const reusable = reusableRecords.length > 0 && reusableRecords.every(({ dir }) => dir === stalledDir || dir === withdrawnDir) &&
|
|
353
|
+
!reusableRecords.some(({ dir }) => dir === pendingDir(input.runDir) || dir === activeDir(input.runDir) || dir === claimedDir(input.runDir)) &&
|
|
354
|
+
(!isRecord(terminalResult) || terminalResult.status === "stalled");
|
|
355
|
+
if ((reusableRecords.length > 0 && !reusable) ||
|
|
356
|
+
(terminalResult !== undefined && (!reusable || !isRecord(terminalResult) || terminalResult.status !== "stalled"))) {
|
|
357
|
+
throw new Error(`envelope id ${envelopeId} was already used in this loop lane`);
|
|
358
|
+
}
|
|
359
|
+
const incomingStop = envelope.kind === "stop";
|
|
360
|
+
const blockingStop = [pendingDir(input.runDir), activeDir(input.runDir), claimedDir(input.runDir)].some((scanDir) => envelopeFiles(scanDir).some((file) => {
|
|
350
361
|
try {
|
|
351
|
-
return fileEnvelope(file).
|
|
362
|
+
return fileEnvelope(file).envelope.kind === "stop";
|
|
352
363
|
}
|
|
353
364
|
catch {
|
|
354
365
|
return false;
|
|
355
366
|
}
|
|
356
|
-
}))
|
|
357
|
-
if (
|
|
358
|
-
(
|
|
359
|
-
throw new Error(`envelope id ${envelopeId} was already used in this loop lane`);
|
|
367
|
+
}));
|
|
368
|
+
if (blockingStop) {
|
|
369
|
+
throw new Error(incomingStop ? "a stop envelope is already queued or active" : "cannot enqueue a phase after a stop envelope");
|
|
360
370
|
}
|
|
361
371
|
if (reusable) {
|
|
362
372
|
for (const file of envelopeFiles(withdrawnDir).concat(envelopeFiles(stalledDir))) {
|
|
@@ -374,23 +384,11 @@ export function enqueueLoopEnvelope(input) {
|
|
|
374
384
|
fs.renameSync(resultFile, asidePath(resultFile, "stalled"));
|
|
375
385
|
}
|
|
376
386
|
}
|
|
377
|
-
const incomingStop = envelope.kind === "stop";
|
|
378
|
-
const blockingStop = [pendingDir(input.runDir), activeDir(input.runDir)].some((scanDir) => envelopeFiles(scanDir).some((file) => {
|
|
379
|
-
try {
|
|
380
|
-
return fileEnvelope(file).envelope.kind === "stop";
|
|
381
|
-
}
|
|
382
|
-
catch {
|
|
383
|
-
return false;
|
|
384
|
-
}
|
|
385
|
-
}));
|
|
386
|
-
if (blockingStop) {
|
|
387
|
-
throw new Error(incomingStop ? "a stop envelope is already queued or active" : "cannot enqueue a phase after a stop envelope");
|
|
388
|
-
}
|
|
389
387
|
const sequence = nextQueueSequence(input.runDir);
|
|
390
388
|
const prefix = `${String(sequence).padStart(16, "0")}-${String(now.getTime()).padStart(13, "0")}-${randomUUID()}`;
|
|
391
389
|
const destination = path.join(dir, `${prefix}-${envelopeId.replace(/[^A-Za-z0-9._-]/g, "_")}.json`);
|
|
392
390
|
const temporary = `${destination}.${process.pid}.tmp`;
|
|
393
|
-
const record = { schema_version: LOOP_SCHEMA, envelope_id: envelopeId, queued_at: queuedAt, envelope };
|
|
391
|
+
const record = { schema_version: LOOP_SCHEMA, envelope_id: envelopeId, queued_at: queuedAt, stall_attempts: 0, envelope };
|
|
394
392
|
try {
|
|
395
393
|
fs.writeFileSync(temporary, `${JSON.stringify(record)}\n`, { flag: "wx", mode: 0o600 });
|
|
396
394
|
fs.renameSync(temporary, destination);
|
|
@@ -417,6 +415,12 @@ export function enqueueLoopEnvelope(input) {
|
|
|
417
415
|
export function claimNextLoopEnvelope(runDir) {
|
|
418
416
|
fs.mkdirSync(activeDir(runDir), { recursive: true });
|
|
419
417
|
fs.mkdirSync(claimedDir(runDir), { recursive: true });
|
|
418
|
+
// A previous controller may have written a stop result but failed before moving the active record to claimed.
|
|
419
|
+
for (const active of envelopeFiles(activeDir(runDir))) {
|
|
420
|
+
const record = fileEnvelope(active);
|
|
421
|
+
if (record.envelope.kind === "stop")
|
|
422
|
+
return { file: active, record };
|
|
423
|
+
}
|
|
420
424
|
for (const pending of envelopeFiles(pendingDir(runDir))) {
|
|
421
425
|
const active = path.join(activeDir(runDir), path.basename(pending));
|
|
422
426
|
try {
|
|
@@ -453,7 +457,7 @@ export function interruptLoopQueue(runDir) {
|
|
|
453
457
|
}
|
|
454
458
|
return { withdrawn, already_picked_up: alreadyPickedUp };
|
|
455
459
|
}
|
|
456
|
-
export function interruptLoopQueueForStory(runDir, storyId, phase) {
|
|
460
|
+
export function interruptLoopQueueForStory(runDir, storyId, phase, hooks = {}) {
|
|
457
461
|
const withdrawn = [];
|
|
458
462
|
const alreadyPickedUp = [];
|
|
459
463
|
const matches = (record) => {
|
|
@@ -461,28 +465,51 @@ export function interruptLoopQueueForStory(runDir, storyId, phase) {
|
|
|
461
465
|
return envelope.story_id === storyId && (phase === undefined || envelope.phase === phase);
|
|
462
466
|
};
|
|
463
467
|
for (const file of envelopeFiles(pendingDir(runDir))) {
|
|
464
|
-
|
|
468
|
+
let record;
|
|
469
|
+
hooks.beforePendingRead?.(file);
|
|
470
|
+
try {
|
|
471
|
+
record = fileEnvelope(file);
|
|
472
|
+
}
|
|
473
|
+
catch (error) {
|
|
474
|
+
if (error.code === "ENOENT")
|
|
475
|
+
continue;
|
|
476
|
+
throw error;
|
|
477
|
+
}
|
|
465
478
|
if (!matches(record))
|
|
466
479
|
continue;
|
|
467
480
|
const destination = path.join(runDir, "loop", "withdrawn", path.basename(file));
|
|
468
481
|
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
469
|
-
|
|
470
|
-
|
|
482
|
+
hooks.beforePendingRename?.(file);
|
|
483
|
+
try {
|
|
484
|
+
fs.renameSync(file, destination);
|
|
485
|
+
withdrawn.push(record.envelope_id);
|
|
486
|
+
}
|
|
487
|
+
catch (error) {
|
|
488
|
+
if (error.code !== "ENOENT")
|
|
489
|
+
throw error;
|
|
490
|
+
alreadyPickedUp.push(record.envelope_id);
|
|
491
|
+
}
|
|
471
492
|
}
|
|
472
493
|
for (const file of envelopeFiles(activeDir(runDir))) {
|
|
473
494
|
const record = fileEnvelope(file);
|
|
474
495
|
if (matches(record))
|
|
475
496
|
alreadyPickedUp.push(record.envelope_id);
|
|
476
497
|
}
|
|
477
|
-
return { withdrawn, already_picked_up: alreadyPickedUp };
|
|
498
|
+
return { withdrawn, already_picked_up: [...new Set(alreadyPickedUp)] };
|
|
478
499
|
}
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
? fs.readdirSync(resultsDir(
|
|
482
|
-
|
|
500
|
+
function stalledResultArchives(runDir) {
|
|
501
|
+
return fs.existsSync(resultsDir(runDir))
|
|
502
|
+
? fs.readdirSync(resultsDir(runDir)).filter((name) => /\.stalled\.\d+\.json$/.test(name))
|
|
503
|
+
: [];
|
|
504
|
+
}
|
|
505
|
+
export function findLoopStallResult(runDir, questionId) {
|
|
506
|
+
const archivedNames = new Set(stalledResultArchives(runDir));
|
|
507
|
+
const results = fs.existsSync(resultsDir(runDir))
|
|
508
|
+
? fs.readdirSync(resultsDir(runDir)).filter((name) => name.endsWith(".json") && !archivedNames.has(name)) : [];
|
|
509
|
+
const resultPath = results.map((name) => path.join(resultsDir(runDir), name)).find((file) => {
|
|
483
510
|
try {
|
|
484
511
|
const result = readJson(file);
|
|
485
|
-
return isRecord(result) && result.status === "stalled" && result.question_id ===
|
|
512
|
+
return isRecord(result) && result.status === "stalled" && result.question_id === questionId;
|
|
486
513
|
}
|
|
487
514
|
catch (error) {
|
|
488
515
|
if (error instanceof SyntaxError)
|
|
@@ -490,9 +517,35 @@ export function applyLoopStallAnswer(input) {
|
|
|
490
517
|
throw error;
|
|
491
518
|
}
|
|
492
519
|
});
|
|
493
|
-
|
|
520
|
+
return resultPath ? readJson(resultPath) : undefined;
|
|
521
|
+
}
|
|
522
|
+
export function applyLoopStallAnswer(input) {
|
|
523
|
+
const result = findLoopStallResult(input.runDir, input.questionId);
|
|
524
|
+
if (!result) {
|
|
525
|
+
for (const name of stalledResultArchives(input.runDir)) {
|
|
526
|
+
try {
|
|
527
|
+
const archived = readJson(path.join(resultsDir(input.runDir), name));
|
|
528
|
+
if (!isRecord(archived) || archived.status !== "stalled" || archived.question_id !== input.questionId)
|
|
529
|
+
continue;
|
|
530
|
+
const envelopeId = String(archived.envelope_id);
|
|
531
|
+
for (const dir of [pendingDir(input.runDir), activeDir(input.runDir), claimedDir(input.runDir)]) {
|
|
532
|
+
const match = envelopeFiles(dir).find((file) => fileEnvelope(file).envelope_id === envelopeId);
|
|
533
|
+
if (!match)
|
|
534
|
+
continue;
|
|
535
|
+
const isAnswer = path.basename(match).startsWith("0000000000000-answer-");
|
|
536
|
+
return { lane_id: path.basename(input.runDir), envelope_id: envelopeId, action: isAnswer && dir === pendingDir(input.runDir) ? "requeued" : "already_applied" };
|
|
537
|
+
}
|
|
538
|
+
return { lane_id: path.basename(input.runDir), envelope_id: envelopeId, action: "already_applied" };
|
|
539
|
+
}
|
|
540
|
+
catch (error) {
|
|
541
|
+
if (error instanceof SyntaxError || error.code === "ENOENT")
|
|
542
|
+
continue;
|
|
543
|
+
throw error;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
494
546
|
return undefined;
|
|
495
|
-
|
|
547
|
+
}
|
|
548
|
+
const resultPath = path.join(resultsDir(input.runDir), resultName(result.envelope_id));
|
|
496
549
|
const envelopeFile = envelopeFiles(path.join(input.runDir, "loop", "stalled")).find((file) => {
|
|
497
550
|
try {
|
|
498
551
|
return fileEnvelope(file).envelope_id === result.envelope_id;
|
|
@@ -506,15 +559,17 @@ export function applyLoopStallAnswer(input) {
|
|
|
506
559
|
const record = fileEnvelope(envelopeFile);
|
|
507
560
|
const action = input.choice === "Retry with a longer deadline" ? "requeued" : "parked";
|
|
508
561
|
if (action === "requeued") {
|
|
509
|
-
record.deadline_budget_seconds =
|
|
562
|
+
record.deadline_budget_seconds = (record.deadline_budget_seconds ?? phaseDeadlineBudget(record.envelope, 3600)) * 2;
|
|
563
|
+
record.stall_attempts = 0;
|
|
510
564
|
const head = path.join(pendingDir(input.runDir), `0000000000000-answer-${path.basename(envelopeFile)}`);
|
|
511
565
|
fs.mkdirSync(pendingDir(input.runDir), { recursive: true });
|
|
512
566
|
fs.renameSync(envelopeFile, head);
|
|
513
567
|
fs.renameSync(resultPath, asidePath(resultPath, "stalled"));
|
|
514
568
|
const pending = fileEnvelope(head);
|
|
515
569
|
pending.deadline_budget_seconds = record.deadline_budget_seconds;
|
|
570
|
+
pending.stall_attempts = 0;
|
|
516
571
|
atomicJson(head, pending);
|
|
517
|
-
|
|
572
|
+
persistStallCount(input.runDir, path.basename(input.runDir), record.envelope_id, 0);
|
|
518
573
|
}
|
|
519
574
|
else {
|
|
520
575
|
atomicJson(resultPath, {
|
|
@@ -613,6 +668,15 @@ function phaseDeadlineBudget(envelope, defaultSeconds) {
|
|
|
613
668
|
const configured = Number(process.env.HQ_LANE_LOOP_PHASE_DEADLINE_SECS);
|
|
614
669
|
return Number.isFinite(configured) && configured > 0 ? configured : defaultSeconds;
|
|
615
670
|
}
|
|
671
|
+
function persistStallCount(runDir, laneId, envelopeId, count) {
|
|
672
|
+
const status = loopStatus(runDir, laneId);
|
|
673
|
+
persistStatus(runDir, laneId, { stall_counts: { ...status.stall_counts, [envelopeId]: count } });
|
|
674
|
+
}
|
|
675
|
+
function clearPersistedStallCount(runDir, laneId, envelopeId) {
|
|
676
|
+
const counts = loopStatus(runDir, laneId).stall_counts;
|
|
677
|
+
delete counts[envelopeId];
|
|
678
|
+
persistStatus(runDir, laneId, { stall_counts: counts });
|
|
679
|
+
}
|
|
616
680
|
function persistStatus(runDir, laneId, patch) {
|
|
617
681
|
let current = {};
|
|
618
682
|
try {
|
|
@@ -658,7 +722,7 @@ export async function runLoopLane(options) {
|
|
|
658
722
|
const stalls = new Map();
|
|
659
723
|
let completed = 0;
|
|
660
724
|
fs.mkdirSync(options.runDir, { recursive: true });
|
|
661
|
-
|
|
725
|
+
let releaseRunner = acquireRunnerLease(options.runDir);
|
|
662
726
|
persistStatus(options.runDir, options.lane.lane_id, { pid: process.pid });
|
|
663
727
|
try {
|
|
664
728
|
const previous = readJson(loopPath(options.runDir));
|
|
@@ -673,7 +737,7 @@ export async function runLoopLane(options) {
|
|
|
673
737
|
if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
|
|
674
738
|
throw error;
|
|
675
739
|
}
|
|
676
|
-
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null
|
|
740
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
677
741
|
while (!options.signal?.aborted) {
|
|
678
742
|
const claimed = claimNextLoopEnvelope(options.runDir);
|
|
679
743
|
if (!claimed) {
|
|
@@ -685,8 +749,29 @@ export async function runLoopLane(options) {
|
|
|
685
749
|
const envelope = record.envelope;
|
|
686
750
|
if (envelope.kind === "stop") {
|
|
687
751
|
const stopped = { schema_version: LOOP_SCHEMA, envelope_id: record.envelope_id, status: "stopped", started_at: record.queued_at, finished_at: now().toISOString() };
|
|
688
|
-
|
|
689
|
-
|
|
752
|
+
try {
|
|
753
|
+
writeResultAt(options.runDir, file, record, stopped);
|
|
754
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "stopped", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
755
|
+
}
|
|
756
|
+
catch (error) {
|
|
757
|
+
const cleanupErrors = [];
|
|
758
|
+
try {
|
|
759
|
+
releaseRunner();
|
|
760
|
+
}
|
|
761
|
+
catch (cleanupError) {
|
|
762
|
+
cleanupErrors.push(cleanupError);
|
|
763
|
+
}
|
|
764
|
+
releaseRunner = () => { };
|
|
765
|
+
try {
|
|
766
|
+
clearLoopControllerPid(options.runDir, options.lane.lane_id, process.pid);
|
|
767
|
+
}
|
|
768
|
+
catch (cleanupError) {
|
|
769
|
+
cleanupErrors.push(cleanupError);
|
|
770
|
+
}
|
|
771
|
+
if (cleanupErrors.length)
|
|
772
|
+
throw new AggregateError([error, ...cleanupErrors], "Could not recover after committing a stop envelope");
|
|
773
|
+
throw error;
|
|
774
|
+
}
|
|
690
775
|
releaseRunner();
|
|
691
776
|
return { state: "stopped", completed };
|
|
692
777
|
}
|
|
@@ -700,12 +785,12 @@ export async function runLoopLane(options) {
|
|
|
700
785
|
const absoluteDeadline = envelope.deadline;
|
|
701
786
|
const budget = record.deadline_budget_seconds ?? firstDeadline.get(record.envelope_id) ??
|
|
702
787
|
(typeof absoluteDeadline === "string"
|
|
703
|
-
? Math.max(1, Math.round((Date.parse(absoluteDeadline) - startedAtMs) / 1000))
|
|
788
|
+
? Math.max(1, Math.round((Date.parse(absoluteDeadline) - startedAtMs) / 1000)) // An absolute deadline already in the past still gets a 1 s execution window.
|
|
704
789
|
: phaseDeadlineBudget(envelope, defaultDeadlineSeconds));
|
|
705
790
|
firstDeadline.set(record.envelope_id, budget);
|
|
706
791
|
record.deadline_budget_seconds = budget;
|
|
707
792
|
atomicJson(file, record);
|
|
708
|
-
persistStatus(options.runDir, options.lane.lane_id, { state: "running", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: record.envelope_id, active_phase: phaseName(envelope, record.envelope_id)
|
|
793
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "running", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: record.envelope_id, active_phase: phaseName(envelope, record.envelope_id) });
|
|
709
794
|
const timer = abortAfter(budget * 1000, options.signal);
|
|
710
795
|
try {
|
|
711
796
|
let result;
|
|
@@ -751,6 +836,7 @@ export async function runLoopLane(options) {
|
|
|
751
836
|
atomicJson(sessionsPath(options.runDir), Object.fromEntries(sessions));
|
|
752
837
|
}
|
|
753
838
|
stalls.delete(record.envelope_id);
|
|
839
|
+
clearPersistedStallCount(options.runDir, options.lane.lane_id, record.envelope_id);
|
|
754
840
|
completed += 1;
|
|
755
841
|
}
|
|
756
842
|
catch (error) {
|
|
@@ -759,49 +845,75 @@ export async function runLoopLane(options) {
|
|
|
759
845
|
const pending = path.join(pendingDir(options.runDir), path.basename(file));
|
|
760
846
|
fs.mkdirSync(pendingDir(options.runDir), { recursive: true });
|
|
761
847
|
fs.renameSync(file, pending);
|
|
762
|
-
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null, stall_counts: Object.fromEntries(stalls) });
|
|
763
848
|
releaseRunner();
|
|
849
|
+
releaseRunner = () => { };
|
|
850
|
+
clearLoopControllerPid(options.runDir, options.lane.lane_id, process.pid);
|
|
851
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
764
852
|
return { state: "aborted", completed };
|
|
765
853
|
}
|
|
766
854
|
if (!timer.signal.aborted) {
|
|
767
855
|
const failure = error instanceof Error ? error.message : String(error);
|
|
768
856
|
writeResultAt(options.runDir, file, record, { schema_version: LOOP_SCHEMA, envelope_id: record.envelope_id, status: "failed", started_at: startedAt, finished_at: now().toISOString(), error: failure });
|
|
769
857
|
completed += 1;
|
|
770
|
-
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null
|
|
858
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
771
859
|
continue;
|
|
772
860
|
}
|
|
773
|
-
const
|
|
861
|
+
const legacyCount = stalls.get(record.envelope_id) ?? 0;
|
|
862
|
+
const attempt = (record.stall_attempts ?? legacyCount) + 1;
|
|
774
863
|
stalls.set(record.envelope_id, attempt);
|
|
775
|
-
|
|
864
|
+
persistStallCount(options.runDir, options.lane.lane_id, record.envelope_id, attempt);
|
|
865
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "running", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: record.envelope_id, active_phase: phaseName(envelope, record.envelope_id) });
|
|
776
866
|
appendJournal(options.runDir, { event: "phase_stalled", envelope_id: record.envelope_id, phase: phaseName(envelope, record.envelope_id), attempt, at: now().toISOString() });
|
|
777
867
|
if (attempt === 1) {
|
|
778
868
|
timer.clear();
|
|
779
869
|
sessions.delete(sessionKey);
|
|
780
870
|
atomicJson(sessionsPath(options.runDir), Object.fromEntries(sessions));
|
|
781
871
|
// Restore the same envelope at the FIFO head. Rename preserves a single claim.
|
|
872
|
+
record.stall_attempts = 1;
|
|
873
|
+
atomicJson(file, record);
|
|
782
874
|
const head = path.join(pendingDir(options.runDir), `0000000000000-retry-${path.basename(file)}`);
|
|
783
875
|
fs.mkdirSync(pendingDir(options.runDir), { recursive: true });
|
|
784
876
|
fs.renameSync(file, head);
|
|
785
|
-
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null
|
|
877
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
786
878
|
continue;
|
|
787
879
|
}
|
|
788
880
|
const message = error instanceof Error ? error.message : String(error);
|
|
789
881
|
const questionId = options.raiseQuestion ? randomUUID() : undefined;
|
|
790
882
|
const result = { schema_version: LOOP_SCHEMA, envelope_id: record.envelope_id, status: "stalled", started_at: startedAt, finished_at: now().toISOString(), error: message, ...(questionId ? { question_id: questionId } : {}) };
|
|
791
883
|
writeResultAt(options.runDir, file, record, result);
|
|
792
|
-
|
|
793
|
-
await options.raiseQuestion?.({ envelope, envelopeId: record.envelope_id, attempt, error: message, questionId });
|
|
884
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "parked", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
794
885
|
completed += 1;
|
|
795
|
-
|
|
796
|
-
|
|
886
|
+
try {
|
|
887
|
+
if (questionId)
|
|
888
|
+
await options.raiseQuestion?.({ envelope, envelopeId: record.envelope_id, attempt, error: message, questionId });
|
|
889
|
+
}
|
|
890
|
+
finally {
|
|
891
|
+
releaseRunner();
|
|
892
|
+
releaseRunner = () => { };
|
|
893
|
+
clearLoopControllerPid(options.runDir, options.lane.lane_id, process.pid);
|
|
894
|
+
}
|
|
895
|
+
const answeredRetryIsPending = envelopeFiles(pendingDir(options.runDir)).some((pendingFile) => path.basename(pendingFile).startsWith("0000000000000-answer-") && fileEnvelope(pendingFile).envelope_id === record.envelope_id);
|
|
896
|
+
if (answeredRetryIsPending) {
|
|
897
|
+
try {
|
|
898
|
+
releaseRunner = acquireRunnerLease(options.runDir);
|
|
899
|
+
persistStatus(options.runDir, options.lane.lane_id, { pid: process.pid, state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
900
|
+
continue;
|
|
901
|
+
}
|
|
902
|
+
catch (leaseError) {
|
|
903
|
+
if (!(leaseError instanceof Error) || !(leaseError.message.includes("already active") || leaseError.message.includes("claimed by another controller")))
|
|
904
|
+
throw leaseError;
|
|
905
|
+
}
|
|
906
|
+
}
|
|
797
907
|
return { state: "parked", completed };
|
|
798
908
|
}
|
|
799
909
|
finally {
|
|
800
910
|
timer.clear();
|
|
801
911
|
}
|
|
802
|
-
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null
|
|
912
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
803
913
|
}
|
|
804
914
|
releaseRunner();
|
|
915
|
+
clearLoopControllerPid(options.runDir, options.lane.lane_id, process.pid);
|
|
916
|
+
persistStatus(options.runDir, options.lane.lane_id, { state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
805
917
|
return { state: "aborted", completed };
|
|
806
918
|
}
|
|
807
919
|
function appendJournal(runDir, value) {
|
|
@@ -53,6 +53,13 @@ export type ClientHealthSyncState = (typeof CLIENT_HEALTH_SYNC_STATES)[number];
|
|
|
53
53
|
*/
|
|
54
54
|
export declare const CLIENT_HEALTH_UPDATER_STATES: readonly ["unchecked", "up_to_date", "update_available", "update_downloading", "update_ready", "update_failed", "unsupported"];
|
|
55
55
|
export type ClientHealthUpdaterState = (typeof CLIENT_HEALTH_UPDATER_STATES)[number];
|
|
56
|
+
/** Origin of the runner currently used by desktop sync. */
|
|
57
|
+
export declare const CLIENT_HEALTH_SYNC_RUNNER_SOURCES: readonly ["npx", "daemon_managed", "daemon_staged", "daemon_bundled", "unknown"];
|
|
58
|
+
export type ClientHealthSyncRunnerSource = (typeof CLIENT_HEALTH_SYNC_RUNNER_SOURCES)[number];
|
|
59
|
+
export declare const CLIENT_HEALTH_DAEMON_HQ_CLOUD_UPDATE_OUTCOMES: readonly ["updated", "up_to_date", "pinned", "failed"];
|
|
60
|
+
export type ClientHealthDaemonHqCloudUpdateOutcome = (typeof CLIENT_HEALTH_DAEMON_HQ_CLOUD_UPDATE_OUTCOMES)[number];
|
|
61
|
+
export declare const CLIENT_HEALTH_DAEMON_HQ_CLOUD_UPDATE_ERROR_CLASSES: readonly ["registry", "install", "convergence", "filesystem", "unknown"];
|
|
62
|
+
export type ClientHealthDaemonHqCloudUpdateErrorClass = (typeof CLIENT_HEALTH_DAEMON_HQ_CLOUD_UPDATE_ERROR_CLASSES)[number];
|
|
56
63
|
/** Closed failure/blocker reason codes — the ONLY reasons that cross the wire. */
|
|
57
64
|
export declare const CLIENT_HEALTH_FAILURE_REASONS: readonly ["SYNC_PAUSED", "CONFLICT_BLOCKED", "DESKTOP_OUTDATED", "CLI_OUTDATED", "CORE_OUTDATED", "AUTH_EXPIRED", "UPDATE_FAILED", "RUNNER_FAILED", "PERMISSION_DENIED", "DISK_FULL", "HEARTBEAT_STALE"];
|
|
58
65
|
export type ClientHealthFailureReason = (typeof CLIENT_HEALTH_FAILURE_REASONS)[number];
|
|
@@ -139,6 +146,11 @@ export interface ClientHealthHeartbeat {
|
|
|
139
146
|
conflictCount?: number;
|
|
140
147
|
updaterState?: ClientHealthUpdaterState;
|
|
141
148
|
failureReason?: ClientHealthFailureReason;
|
|
149
|
+
syncRunnerSource?: ClientHealthSyncRunnerSource;
|
|
150
|
+
daemonHqCloudUpdateLastCheckAt?: string;
|
|
151
|
+
daemonHqCloudUpdateOutcome?: ClientHealthDaemonHqCloudUpdateOutcome;
|
|
152
|
+
daemonHqCloudUpdateTarget?: string;
|
|
153
|
+
daemonHqCloudUpdateErrorClass?: ClientHealthDaemonHqCloudUpdateErrorClass;
|
|
142
154
|
/** Absent on older clients / non-sync-outcome events. See {@link ClientHealthLocalFilesOverview}. */
|
|
143
155
|
localFilesOverview?: ClientHealthLocalFilesOverview;
|
|
144
156
|
}
|
|
@@ -65,6 +65,27 @@ export const CLIENT_HEALTH_UPDATER_STATES = [
|
|
|
65
65
|
"update_failed",
|
|
66
66
|
"unsupported",
|
|
67
67
|
];
|
|
68
|
+
/** Origin of the runner currently used by desktop sync. */
|
|
69
|
+
export const CLIENT_HEALTH_SYNC_RUNNER_SOURCES = [
|
|
70
|
+
"npx",
|
|
71
|
+
"daemon_managed",
|
|
72
|
+
"daemon_staged",
|
|
73
|
+
"daemon_bundled",
|
|
74
|
+
"unknown",
|
|
75
|
+
];
|
|
76
|
+
export const CLIENT_HEALTH_DAEMON_HQ_CLOUD_UPDATE_OUTCOMES = [
|
|
77
|
+
"updated",
|
|
78
|
+
"up_to_date",
|
|
79
|
+
"pinned",
|
|
80
|
+
"failed",
|
|
81
|
+
];
|
|
82
|
+
export const CLIENT_HEALTH_DAEMON_HQ_CLOUD_UPDATE_ERROR_CLASSES = [
|
|
83
|
+
"registry",
|
|
84
|
+
"install",
|
|
85
|
+
"convergence",
|
|
86
|
+
"filesystem",
|
|
87
|
+
"unknown",
|
|
88
|
+
];
|
|
68
89
|
/** Closed failure/blocker reason codes — the ONLY reasons that cross the wire. */
|
|
69
90
|
export const CLIENT_HEALTH_FAILURE_REASONS = [
|
|
70
91
|
"SYNC_PAUSED",
|
|
@@ -193,6 +214,23 @@ function assertEnum(field, value, allowed) {
|
|
|
193
214
|
}
|
|
194
215
|
return safe;
|
|
195
216
|
}
|
|
217
|
+
function copyValidOptionalDiagnostic(field, value, validate) {
|
|
218
|
+
try {
|
|
219
|
+
return validate(field, value);
|
|
220
|
+
}
|
|
221
|
+
catch (error) {
|
|
222
|
+
if (error instanceof ClientHealthContractError)
|
|
223
|
+
return undefined;
|
|
224
|
+
throw error;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
function copyKnownOptionalEnum(target, field, value, allowed) {
|
|
228
|
+
if (value === undefined)
|
|
229
|
+
return;
|
|
230
|
+
const parsed = copyValidOptionalDiagnostic(field, value, (name, input) => assertEnum(name, input, allowed));
|
|
231
|
+
if (parsed !== undefined)
|
|
232
|
+
Object.assign(target, { [field]: parsed });
|
|
233
|
+
}
|
|
196
234
|
function assertBoundedInt(field, value, max) {
|
|
197
235
|
if (value === undefined || value === null)
|
|
198
236
|
throw new ClientHealthContractError("MISSING_FIELD", field);
|
|
@@ -264,6 +302,19 @@ export function parseClientHealthHeartbeat(input) {
|
|
|
264
302
|
if (raw.failureReason !== undefined) {
|
|
265
303
|
heartbeat.failureReason = assertEnum("failureReason", raw.failureReason, CLIENT_HEALTH_FAILURE_REASONS);
|
|
266
304
|
}
|
|
305
|
+
copyKnownOptionalEnum(heartbeat, "syncRunnerSource", raw.syncRunnerSource, CLIENT_HEALTH_SYNC_RUNNER_SOURCES);
|
|
306
|
+
if (raw.daemonHqCloudUpdateLastCheckAt !== undefined) {
|
|
307
|
+
const lastCheckAt = copyValidOptionalDiagnostic("daemonHqCloudUpdateLastCheckAt", raw.daemonHqCloudUpdateLastCheckAt, assertIsoUtc);
|
|
308
|
+
if (lastCheckAt !== undefined)
|
|
309
|
+
heartbeat.daemonHqCloudUpdateLastCheckAt = lastCheckAt;
|
|
310
|
+
}
|
|
311
|
+
copyKnownOptionalEnum(heartbeat, "daemonHqCloudUpdateOutcome", raw.daemonHqCloudUpdateOutcome, CLIENT_HEALTH_DAEMON_HQ_CLOUD_UPDATE_OUTCOMES);
|
|
312
|
+
if (raw.daemonHqCloudUpdateTarget !== undefined) {
|
|
313
|
+
const target = copyValidOptionalDiagnostic("daemonHqCloudUpdateTarget", raw.daemonHqCloudUpdateTarget, assertVersion);
|
|
314
|
+
if (target !== undefined)
|
|
315
|
+
heartbeat.daemonHqCloudUpdateTarget = target;
|
|
316
|
+
}
|
|
317
|
+
copyKnownOptionalEnum(heartbeat, "daemonHqCloudUpdateErrorClass", raw.daemonHqCloudUpdateErrorClass, CLIENT_HEALTH_DAEMON_HQ_CLOUD_UPDATE_ERROR_CLASSES);
|
|
267
318
|
if (raw.localFilesOverview !== undefined) {
|
|
268
319
|
heartbeat.localFilesOverview = parseClientHealthLocalFilesOverview("localFilesOverview", raw.localFilesOverview);
|
|
269
320
|
}
|
|
@@ -1947,7 +1947,11 @@ export async function enforceVersionGate(onUpdateRecommended, options = {}) {
|
|
|
1947
1947
|
* force-upgraded.
|
|
1948
1948
|
*/
|
|
1949
1949
|
export function shouldSkipGate(argv) {
|
|
1950
|
-
|
|
1950
|
+
// Hook-time deliveries must finish inside the provider's timeout budget.
|
|
1951
|
+
// process.argv slots 0 and 1 are the runtime and CLI entry point. Inspect
|
|
1952
|
+
// command positions only; user arguments may legitimately contain these words.
|
|
1953
|
+
const hookTimeLinkDelivery = argv[2] === "lanes" && argv[3] === "link" && argv[4] === "_deliver";
|
|
1954
|
+
return hookTimeLinkDelivery || isVersionRequest(argv) || argv.some((a) => a === "--version" ||
|
|
1951
1955
|
a === "-V" ||
|
|
1952
1956
|
a === "-v" ||
|
|
1953
1957
|
a === "--help" ||
|