@adhdev/daemon-core 1.0.18-rc.13 → 1.0.18-rc.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-adapters/provider-cli-shared.d.ts +33 -0
- package/dist/index.js +176 -36
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +176 -36
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-remote-event-pull.d.ts +3 -0
- package/dist/providers/cli-provider-instance.d.ts +27 -0
- package/package.json +3 -3
- package/src/cli-adapters/cli-state-engine.ts +17 -1
- package/src/cli-adapters/provider-cli-adapter.ts +2 -6
- package/src/cli-adapters/provider-cli-shared.ts +41 -0
- package/src/mesh/mesh-reconcile-loop.ts +131 -2
- package/src/mesh/mesh-remote-event-pull.ts +68 -47
- package/src/providers/cli-provider-instance.ts +92 -0
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
2
|
import type { LocalMeshEntry } from '../repo-mesh-types.js';
|
|
3
3
|
export declare function pullRemoteNodeQueues(components: DaemonComponents, mesh: LocalMeshEntry, localDaemonId: string | undefined, candidateDaemonIds: string[]): Promise<void>;
|
|
4
|
+
export declare function pullPendingEventsFromNode(components: DaemonComponents, meshId: string, node: {
|
|
5
|
+
daemonId?: string;
|
|
6
|
+
}, localDaemonId: string | undefined, candidateDaemonIds: string[], pulls: Array<Record<string, unknown>>): Promise<void>;
|
|
4
7
|
export declare function unwrapReadChatPayload(raw: unknown): Record<string, unknown> | null;
|
|
5
8
|
export declare function readChatPayloadStatus(payload: Record<string, unknown> | null): string;
|
|
6
9
|
export declare function realTerminalEmitPendingForTask(meshId: string, taskId: string): boolean;
|
|
@@ -619,6 +619,33 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
619
619
|
* call unconditionally from the cleanup path.
|
|
620
620
|
*/
|
|
621
621
|
flushMeshCompletionBeforeCleanup(): boolean;
|
|
622
|
+
/**
|
|
623
|
+
* KIMI-PURE-PTY-COMPLETION-EMIT (Fix 3): stall-path completion reconcile for the
|
|
624
|
+
* PURE-PTY transcript class (kimi and kin — no native transcript, no provider
|
|
625
|
+
* authority; see isPurePtyTranscriptProvider). Such a worker whose
|
|
626
|
+
* generating_completed never emitted (the onTurnStarted idle→idle collapse Fix 1
|
|
627
|
+
* fixes at the source) sits at a STATIC idle prompt: its PTY output stops the
|
|
628
|
+
* instant the answer is rendered, so the status-agnostic no-progress watchdog
|
|
629
|
+
* (checkMeshWorkerStall) reads the finished-but-quiet session as a stall and would
|
|
630
|
+
* false-fire monitor:no_progress. The native-transcript reconcile
|
|
631
|
+
* (sampleNativeTranscriptProgress) does NOT cover this class (no native source →
|
|
632
|
+
* null sample), so the stall path needs its own guard.
|
|
633
|
+
*
|
|
634
|
+
* Called from checkMeshWorkerStall just before the fire. Returns true when the
|
|
635
|
+
* session is a finished pure-PTY turn — idle, no pending response, and a PTY-parsed
|
|
636
|
+
* in-turn final assistant summary — in which case it emits the missing
|
|
637
|
+
* generating_completed (idempotent: a real/late emit writes the terminal ledger and
|
|
638
|
+
* the coordinator's reconcile makes any duplicate a no-op) and the caller SUPPRESSES
|
|
639
|
+
* the stall. Returns false for every other class/state (native-source provider, a
|
|
640
|
+
* genuinely mid-turn or wedged worker with no final assistant), so a real stall still
|
|
641
|
+
* fires unchanged.
|
|
642
|
+
*
|
|
643
|
+
* Evidence bar is identical to flushMeshCompletionBeforeCleanup: injected task's turn
|
|
644
|
+
* genuinely started + an in-turn final assistant summary. Conservative by
|
|
645
|
+
* construction — no summary ⇒ no proof of completion ⇒ return false and let the real
|
|
646
|
+
* stall fire.
|
|
647
|
+
*/
|
|
648
|
+
private tryReconcilePurePtyCompletionForStall;
|
|
622
649
|
/**
|
|
623
650
|
* AUTOAPPROVE-FLAP-RECUR (Fix A+B): how long a busy blip / modal scroll-out may
|
|
624
651
|
* persist before the in-progress settle gate is torn down. For a delegated
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "1.0.18-rc.
|
|
3
|
+
"version": "1.0.18-rc.15",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -47,8 +47,8 @@
|
|
|
47
47
|
"author": "vilmire",
|
|
48
48
|
"license": "AGPL-3.0-or-later",
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@adhdev/mesh-shared": "1.0.18-rc.
|
|
51
|
-
"@adhdev/session-host-core": "1.0.18-rc.
|
|
50
|
+
"@adhdev/mesh-shared": "1.0.18-rc.15",
|
|
51
|
+
"@adhdev/session-host-core": "1.0.18-rc.15",
|
|
52
52
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
53
53
|
"ajv": "^8.20.0",
|
|
54
54
|
"ajv-formats": "^3.0.1",
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
import {
|
|
16
16
|
buildCliScreenSnapshot,
|
|
17
17
|
compactPromptText,
|
|
18
|
+
isPurePtyTranscriptProvider,
|
|
18
19
|
normalizePromptText,
|
|
19
20
|
promptLikelyVisible,
|
|
20
21
|
type CliChatMessage,
|
|
@@ -361,7 +362,22 @@ export class CliStateEngine {
|
|
|
361
362
|
// idle transition, unchanged. Scoped to transcriptAuthority:'provider'
|
|
362
363
|
// only, so PTY-authoritative providers (whose spinner/settled parsing
|
|
363
364
|
// already drives generating promptly) are unaffected.
|
|
364
|
-
|
|
365
|
+
//
|
|
366
|
+
// (fix: kimi pure-PTY completion-emit) The pure-PTY full-buffer class
|
|
367
|
+
// (kimi and kin — see isPurePtyTranscriptProvider) is NOT
|
|
368
|
+
// transcriptAuthority:'provider', so without this it stays idle when a
|
|
369
|
+
// prompt is submitted from idle: the FSM never crosses generating→idle,
|
|
370
|
+
// detectStatusTransition's generating|waiting_approval→idle arm never
|
|
371
|
+
// runs, and agent:generating_completed is never emitted — the mesh
|
|
372
|
+
// coordinator leaves the task 'assigned' and the status-agnostic stall
|
|
373
|
+
// watchdog then false-fires task_stalled on the finished-but-idle
|
|
374
|
+
// session. Promoting this class to generating on turn-start makes the
|
|
375
|
+
// existing PTY-parsed final-assistant idle gate produce a real
|
|
376
|
+
// generating→idle completion edge. Same idle safety as above: applyIdle
|
|
377
|
+
// / finishResponse still own the actual idle transition.
|
|
378
|
+
const promoteOnTurnStart = this.provider.transcriptAuthority === 'provider'
|
|
379
|
+
|| isPurePtyTranscriptProvider(this.provider);
|
|
380
|
+
if (promoteOnTurnStart && this.currentStatus !== 'waiting_approval') {
|
|
365
381
|
this.setStatus('generating', 'turn_started');
|
|
366
382
|
this.callbacks.onStatusChange();
|
|
367
383
|
}
|
|
@@ -39,6 +39,7 @@ import {
|
|
|
39
39
|
compactPromptText,
|
|
40
40
|
estimatePromptDisplayLines,
|
|
41
41
|
extractPromptRetrySnippet,
|
|
42
|
+
isPurePtyTranscriptProvider,
|
|
42
43
|
listCliScriptNames,
|
|
43
44
|
normalizePromptText,
|
|
44
45
|
normalizeScreenSnapshot,
|
|
@@ -456,12 +457,7 @@ export class ProviderCliAdapter implements CliAdapter {
|
|
|
456
457
|
* provider-owned) so no other provider's turn-scoped parse changes.
|
|
457
458
|
*/
|
|
458
459
|
private parsesFullPtyTranscriptFromBuffer(): boolean {
|
|
459
|
-
|
|
460
|
-
// nativeHistory is a top-level provider field not surfaced on
|
|
461
|
-
// CliProviderModule; read it via the same structural cast used for `tui`.
|
|
462
|
-
if ((this.provider as { nativeHistory?: unknown }).nativeHistory) return false;
|
|
463
|
-
const transcriptPty = (this.provider.tui as { transcriptPty?: { scope?: unknown } } | undefined)?.transcriptPty;
|
|
464
|
-
return transcriptPty?.scope === 'buffer';
|
|
460
|
+
return isPurePtyTranscriptProvider(this.provider);
|
|
465
461
|
}
|
|
466
462
|
|
|
467
463
|
/**
|
|
@@ -254,6 +254,47 @@ export interface CliProviderModule {
|
|
|
254
254
|
_versionWarning?: string | null;
|
|
255
255
|
}
|
|
256
256
|
|
|
257
|
+
/**
|
|
258
|
+
* PURE-PTY TRANSCRIPT CLASS predicate (kimi and kin).
|
|
259
|
+
*
|
|
260
|
+
* A provider is "pure-PTY full-buffer" when it reconstructs its ENTIRE transcript
|
|
261
|
+
* from the rendered PTY buffer on every read and has NO alternate transcript
|
|
262
|
+
* source:
|
|
263
|
+
* - transcriptAuthority !== 'provider' (the daemon PTY parser owns the transcript,
|
|
264
|
+
* not a provider-side canonical source)
|
|
265
|
+
* - NO nativeHistory (no on-disk / native-source history to fall back to)
|
|
266
|
+
* - tui.transcriptPty.scope === 'buffer' (the parser walks the full rendered buffer)
|
|
267
|
+
*
|
|
268
|
+
* This class is invisible to two provider-authority-keyed code paths that other
|
|
269
|
+
* providers rely on for mesh completion semantics:
|
|
270
|
+
* 1. CliStateEngine.onTurnStarted only promotes to 'generating' for
|
|
271
|
+
* transcriptAuthority==='provider' providers — so a pure-PTY session that
|
|
272
|
+
* submits a prompt while already idle collapses idle→idle, the
|
|
273
|
+
* generating→idle edge never occurs, and agent:generating_completed is never
|
|
274
|
+
* emitted (the coordinator ledger leaves the task 'assigned' forever).
|
|
275
|
+
* 2. checkMeshWorkerStall's native-transcript completion reconcile is gated on
|
|
276
|
+
* the native-source shape, so a finished pure-PTY worker's static idle is
|
|
277
|
+
* misread as monitor:no_progress (a false task_stalled).
|
|
278
|
+
*
|
|
279
|
+
* The runtime capability, NOT any single spec field value, is authoritative: a
|
|
280
|
+
* given kimi manifest checkout may declare nativeHistory/transcriptAuthority, but
|
|
281
|
+
* the live-loaded pure-PTY session has none. Callers that only hold a
|
|
282
|
+
* CliProviderModule (the engine) share this exact predicate with the adapter
|
|
283
|
+
* (parsesFullPtyTranscriptFromBuffer) so the two never drift.
|
|
284
|
+
*/
|
|
285
|
+
export function isPurePtyTranscriptProvider(provider: {
|
|
286
|
+
transcriptAuthority?: 'provider' | 'daemon';
|
|
287
|
+
nativeHistory?: unknown;
|
|
288
|
+
tui?: Record<string, unknown>;
|
|
289
|
+
}): boolean {
|
|
290
|
+
if (provider.transcriptAuthority === 'provider') return false;
|
|
291
|
+
// nativeHistory is a top-level provider field not surfaced on
|
|
292
|
+
// CliProviderModule; read it via the same structural shape the adapter uses.
|
|
293
|
+
if (provider.nativeHistory) return false;
|
|
294
|
+
const transcriptPty = (provider.tui as { transcriptPty?: { scope?: unknown } } | undefined)?.transcriptPty;
|
|
295
|
+
return transcriptPty?.scope === 'buffer';
|
|
296
|
+
}
|
|
297
|
+
|
|
257
298
|
function stripAnsi(str: string): string {
|
|
258
299
|
// eslint-disable-next-line no-control-regex
|
|
259
300
|
return str
|
|
@@ -75,7 +75,7 @@ import {
|
|
|
75
75
|
resolvePendingHeldDrainEscalateMs,
|
|
76
76
|
resolveReconcileIntervalMs,
|
|
77
77
|
} from './mesh-reconcile-config.js';
|
|
78
|
-
import { pullRemoteNodeQueues } from './mesh-remote-event-pull.js';
|
|
78
|
+
import { pullRemoteNodeQueues, pullPendingEventsFromNode } from './mesh-remote-event-pull.js';
|
|
79
79
|
import { runDiskRetentionSweep, detectAndSignalOrphanWorktrees } from './mesh-disk-retention.js';
|
|
80
80
|
import {
|
|
81
81
|
reconcileUnterminatedDirectDispatches,
|
|
@@ -732,10 +732,32 @@ const deliveredNoTurnUnknownStreak = new Map<string, number>();
|
|
|
732
732
|
// immediately; UNKNOWN accrues a streak and re-drives only after RECLAIM_UNKNOWN_GRACE_TICKS.
|
|
733
733
|
const deliveredUnconsumedUnknownStreak = new Map<string, number>();
|
|
734
734
|
|
|
735
|
+
// KIMI-PURE-PTY-COMPLETION-EMIT (Fix 2): how long a delivered assigned row whose worker is
|
|
736
|
+
// CONTINUOUSLY idle-WITH-a-final-assistant-message must stay so before we short-circuit it to
|
|
737
|
+
// 'completed' from transcript evidence — WITHOUT waiting the full DELIVERED_NO_TURN_DEADLINE_MS
|
|
738
|
+
// (15min). A pure-PTY worker (kimi and kin — no native transcript, no provider authority) whose
|
|
739
|
+
// generating_completed never emitted (the onTurnStarted idle→idle collapse Fix 1 addresses at the
|
|
740
|
+
// source) leaves the row 'assigned' with an idle worker that already rendered its answer. The
|
|
741
|
+
// F3 long-deadline reclaim's pollAssignedTaskTerminalEvidence would eventually recover it — but only
|
|
742
|
+
// after 15min, during which the coordinator ledger wrongly shows the task in flight. This EARLY
|
|
743
|
+
// reconcile applies the SAME idle-with-final-assistant-after-dispatch evidence bar (via
|
|
744
|
+
// pollAssignedTaskTerminalEvidence) after a few continuous-idle ticks, so a finished worker's task
|
|
745
|
+
// is marked complete promptly. Conservative by construction: the streak resets the instant a read is
|
|
746
|
+
// non-idle / lacks a final summary / is unreadable, and the poll itself enforces the after-dispatch
|
|
747
|
+
// stale-summary guard, so a mid-turn or warming-up worker is never falsely completed.
|
|
748
|
+
const ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS = 8_000;
|
|
749
|
+
|
|
750
|
+
// Per-row continuous idle-with-final-assistant streak for the early transcript-evidence completion,
|
|
751
|
+
// keyed `${meshId}::${taskId}`, storing the wall-clock ms when the continuous run began. Pruned each
|
|
752
|
+
// pass to the currently-assigned rows (same as the UNKNOWN streaks). A non-idle / no-summary /
|
|
753
|
+
// unreadable tick clears the entry so the grace must re-accumulate from scratch.
|
|
754
|
+
const assignedIdleFinalAssistantSince = new Map<string, number>();
|
|
755
|
+
|
|
735
756
|
// Test hook: clear the delivered-no-turn UNKNOWN streaks between cases.
|
|
736
757
|
export function __resetReclaimUnknownStreakForTests(): void {
|
|
737
758
|
deliveredNoTurnUnknownStreak.clear();
|
|
738
759
|
deliveredUnconsumedUnknownStreak.clear();
|
|
760
|
+
assignedIdleFinalAssistantSince.clear();
|
|
739
761
|
}
|
|
740
762
|
|
|
741
763
|
// APPROVAL-INBOX-BLINDSPOT (Fix A.3): true when the assigned row's bound session is, per the
|
|
@@ -781,6 +803,8 @@ async function recoverStrandedAssignedDispatches(
|
|
|
781
803
|
components: DaemonComponents,
|
|
782
804
|
mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
|
|
783
805
|
store: MeshRuntimeStore,
|
|
806
|
+
localDaemonId?: string,
|
|
807
|
+
selfIds: string[] = [],
|
|
784
808
|
): Promise<void> {
|
|
785
809
|
const meshId = mesh.id;
|
|
786
810
|
const assigned = getQueue(meshId, { status: ['assigned'] });
|
|
@@ -797,10 +821,76 @@ async function recoverStrandedAssignedDispatches(
|
|
|
797
821
|
for (const key of [...deliveredUnconsumedUnknownStreak.keys()]) {
|
|
798
822
|
if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredUnconsumedUnknownStreak.delete(key);
|
|
799
823
|
}
|
|
824
|
+
for (const key of [...assignedIdleFinalAssistantSince.keys()]) {
|
|
825
|
+
if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) assignedIdleFinalAssistantSince.delete(key);
|
|
826
|
+
}
|
|
800
827
|
for (const row of assigned) {
|
|
801
828
|
const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
|
|
802
829
|
if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
|
|
803
830
|
const ageMs = nowMs - dispatchedAtMs;
|
|
831
|
+
const idleTranscriptStreakKey = `${meshId}::${row.id}`;
|
|
832
|
+
// KIMI-PURE-PTY-COMPLETION-EMIT (Fix 2): EARLY transcript-evidence completion for a
|
|
833
|
+
// delivered assigned row whose worker is already idle with a final assistant answer, so a
|
|
834
|
+
// pure-PTY worker whose generating_completed never emitted is marked done PROMPTLY instead of
|
|
835
|
+
// sitting 'assigned' until the 15-min DELIVERED_NO_TURN_DEADLINE_MS reclaim. Gate on a
|
|
836
|
+
// CONFIRMED delivery (a never-delivered row is handled by dispatch/redrive, not completion)
|
|
837
|
+
// and a bound session, then require the worker to read idle-WITH-final-assistant continuously
|
|
838
|
+
// for ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS before polling the decisive transcript evidence.
|
|
839
|
+
// The streak is the cheap gate (avoids a read every tick); the read only runs once the grace
|
|
840
|
+
// elapses. pollAssignedTaskTerminalEvidence enforces idle + final-assistant-after-dispatch, so
|
|
841
|
+
// a mid-turn / stale-tail / unreadable worker is never falsely completed — a non-qualifying
|
|
842
|
+
// tick clears the streak below.
|
|
843
|
+
if (
|
|
844
|
+
store.taskHasConfirmedDelivery(meshId, row.id)
|
|
845
|
+
&& !findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })
|
|
846
|
+
&& readNonEmptyString(row.assignedSessionId)
|
|
847
|
+
&& resolveSessionBusyVerdict(components, row.assignedSessionId!) !== 'GENERATING'
|
|
848
|
+
) {
|
|
849
|
+
const since = assignedIdleFinalAssistantSince.get(idleTranscriptStreakKey);
|
|
850
|
+
if (since === undefined) {
|
|
851
|
+
assignedIdleFinalAssistantSince.set(idleTranscriptStreakKey, nowMs);
|
|
852
|
+
} else if (nowMs - since >= ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS) {
|
|
853
|
+
const terminalEvidence = await pollAssignedTaskTerminalEvidence(components, mesh, row);
|
|
854
|
+
if (terminalEvidence) {
|
|
855
|
+
assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
|
|
856
|
+
updateTaskStatus(meshId, row.id, terminalEvidence);
|
|
857
|
+
if (!findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })) {
|
|
858
|
+
try {
|
|
859
|
+
appendLedgerEntry(meshId, {
|
|
860
|
+
kind: terminalEvidence === 'completed' ? 'task_completed' : 'task_failed',
|
|
861
|
+
nodeId: row.assignedNodeId,
|
|
862
|
+
sessionId: row.assignedSessionId,
|
|
863
|
+
providerType: row.assignedProviderType,
|
|
864
|
+
payload: {
|
|
865
|
+
taskId: row.id,
|
|
866
|
+
event: 'agent:generating_completed',
|
|
867
|
+
source: 'early_idle_transcript_evidence',
|
|
868
|
+
},
|
|
869
|
+
});
|
|
870
|
+
} catch { /* best-effort ledger write */ }
|
|
871
|
+
}
|
|
872
|
+
LOG.warn('MeshReconcile', `Early-completed assigned task ${row.id} on mesh ${meshId} `
|
|
873
|
+
+ `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}): worker idle with a `
|
|
874
|
+
+ `final assistant message after dispatch for ≥${Math.round(ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS / 1000)}s — `
|
|
875
|
+
+ `the completion event was lost/late (pure-PTY provider), task is ${terminalEvidence} without waiting the 15-min deadline`);
|
|
876
|
+
traceMeshEventDrop('assigned_early_transcript_completed', {
|
|
877
|
+
taskId: row.id,
|
|
878
|
+
sessionId: row.assignedSessionId,
|
|
879
|
+
nodeId: row.assignedNodeId,
|
|
880
|
+
meshId,
|
|
881
|
+
event: 'agent:generating_completed',
|
|
882
|
+
}, terminalEvidence);
|
|
883
|
+
continue;
|
|
884
|
+
}
|
|
885
|
+
// Poll was inconclusive (mid-turn re-check / stale tail / unreadable) — reset the
|
|
886
|
+
// streak so it must re-accumulate a fresh continuous-idle run before the next poll.
|
|
887
|
+
assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
|
|
888
|
+
}
|
|
889
|
+
} else {
|
|
890
|
+
// Not a qualifying tick (no confirmed delivery, generating, terminal ledger present, or
|
|
891
|
+
// no bound session) — break the continuous-idle run.
|
|
892
|
+
assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
|
|
893
|
+
}
|
|
804
894
|
// DELIVERED-NOT-CONSUMED short-grace re-drive (remote autoLaunch delivered≠consumed gap).
|
|
805
895
|
// Runs BEFORE the ASSIGNED_STRANDED_DEADLINE_MS confirm-window gate below because its whole
|
|
806
896
|
// point is to recover a delivered-but-unconsumed row well inside that window. A remote
|
|
@@ -832,6 +922,45 @@ async function recoverStrandedAssignedDispatches(
|
|
|
832
922
|
updateTaskStatus(meshId, row.id, status);
|
|
833
923
|
continue;
|
|
834
924
|
}
|
|
925
|
+
// GENERATING-STARTED-CONSUME-RACE (fix B): the delivery-consume that clears this
|
|
926
|
+
// gate (delivered → acked) runs ONLY when the coordinator pulls the worker's queued
|
|
927
|
+
// agent:generating_started in PHASE 1 (handleMeshForwardEvent → consumeSessionDelivery).
|
|
928
|
+
// A worker may have emitted generating_started ALREADY, but if PHASE 1 has not yet
|
|
929
|
+
// pulled that specific event (a skipped/slow peer tick, or the event was queued after
|
|
930
|
+
// this tick's pull ran), the delivery row still reads 'delivered' here and the redrive
|
|
931
|
+
// below tears a genuinely-generating remote worker off its task and re-injects the
|
|
932
|
+
// prompt (the delivered_not_consumed_redrive symptom). Close the race by issuing a
|
|
933
|
+
// TARGETED, in-process last-chance pull of THIS node's queue right now, then
|
|
934
|
+
// re-reading taskDeliveryConsumed(): if the worker's generating_started was waiting to
|
|
935
|
+
// be pulled, the pull consumes the delivery this tick and we skip the redrive entirely.
|
|
936
|
+
// Best-effort and self-scoped (the same skip/peer guards as PHASE 1) — a miss just
|
|
937
|
+
// falls through to the existing UNKNOWN-streak grace, so this only ever removes false
|
|
938
|
+
// redrives, never adds one. A local (co-hosted) worker has nothing to pull (the node's
|
|
939
|
+
// daemon is a self id) and this is a fast no-op.
|
|
940
|
+
const assignedNode = row.assignedNodeId
|
|
941
|
+
? mesh.nodes?.find(n => meshNodeIdMatches(n, row.assignedNodeId))
|
|
942
|
+
: undefined;
|
|
943
|
+
if (assignedNode?.daemonId) {
|
|
944
|
+
try {
|
|
945
|
+
await pullPendingEventsFromNode(
|
|
946
|
+
components,
|
|
947
|
+
meshId,
|
|
948
|
+
assignedNode,
|
|
949
|
+
localDaemonId,
|
|
950
|
+
selfIds,
|
|
951
|
+
selfIds.length > 0
|
|
952
|
+
? selfIds.map(id => ({ meshId, coordinatorDaemonId: id }))
|
|
953
|
+
: [{ meshId }],
|
|
954
|
+
);
|
|
955
|
+
} catch { /* best-effort last-chance pull */ }
|
|
956
|
+
if (store.taskDeliveryConsumed(meshId, row.id)) {
|
|
957
|
+
// The pull delivered the worker's generating_started (or a terminal event) and
|
|
958
|
+
// consumed the delivery — the task is genuinely live/handled. Reset any accrued
|
|
959
|
+
// streak and leave the row to PHASE 4's completion reconcile.
|
|
960
|
+
deliveredUnconsumedUnknownStreak.delete(`${meshId}::${row.id}`);
|
|
961
|
+
continue;
|
|
962
|
+
}
|
|
963
|
+
}
|
|
835
964
|
const shortStreakKey = `${meshId}::${row.id}`;
|
|
836
965
|
const verdict = row.assignedSessionId
|
|
837
966
|
? resolveSessionBusyVerdict(components, row.assignedSessionId)
|
|
@@ -1227,7 +1356,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
1227
1356
|
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
1228
1357
|
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
1229
1358
|
try {
|
|
1230
|
-
await recoverStrandedAssignedDispatches(components, mesh, store);
|
|
1359
|
+
await recoverStrandedAssignedDispatches(components, mesh, store, localDaemonId, selfIds);
|
|
1231
1360
|
} catch (e: any) {
|
|
1232
1361
|
LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
1233
1362
|
}
|
|
@@ -60,57 +60,78 @@ export async function pullRemoteNodeQueues(
|
|
|
60
60
|
// block the other nodes for the rest of the tick. Each node callback is fully
|
|
61
61
|
// self-contained (local/candidate skip, peer-connected pre-check, per-candidate
|
|
62
62
|
// pulls, extract→re-inject) and best-effort — allSettled swallows per-node errors.
|
|
63
|
-
await Promise.allSettled(mesh.nodes.map(
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
// already in the local queue drained in PHASE 2). "This daemon" is matched
|
|
67
|
-
// against the full self-identity set (candidateDaemonIds), not just the bare
|
|
68
|
-
// localDaemonId — a self node can be registered under the config-form daemonId
|
|
69
|
-
// (`daemon_<machineId>`) which would NOT equal bare localDaemonId, and pulling
|
|
70
|
-
// from ourselves over P2P is both wasteful and a self-dispatch hazard.
|
|
71
|
-
if (!nodeDaemonId) return;
|
|
72
|
-
if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) return;
|
|
73
|
-
if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) return;
|
|
63
|
+
await Promise.allSettled(mesh.nodes.map(node =>
|
|
64
|
+
pullPendingEventsFromNode(components, meshId, node, localDaemonId, candidateDaemonIds, pulls)));
|
|
65
|
+
}
|
|
74
66
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
67
|
+
// Pull one node's pending coordinator events and re-inject them locally via
|
|
68
|
+
// handleMeshForwardEvent (which runs the delivery-consume + re-queue paths). Factored
|
|
69
|
+
// out of pullRemoteNodeQueues so the reconcile loop can also issue a TARGETED single-node
|
|
70
|
+
// pull on demand — specifically, the DELIVERED-NOT-CONSUMED redrive gate calls this for
|
|
71
|
+
// one node right before it would re-drive, so a worker's already-emitted (but not-yet-
|
|
72
|
+
// pulled) agent:generating_started is consumed IN-PROCESS this tick instead of waiting for
|
|
73
|
+
// the next PHASE 1 pull. That closes the redrive-vs-consume race: the redrive gate reads
|
|
74
|
+
// taskDeliveryConsumed() AFTER this pull has had its chance to flip the delivery row.
|
|
75
|
+
// Returns silently on any skip/error — every caller treats it as best-effort.
|
|
76
|
+
export async function pullPendingEventsFromNode(
|
|
77
|
+
components: DaemonComponents,
|
|
78
|
+
meshId: string,
|
|
79
|
+
node: { daemonId?: string },
|
|
80
|
+
localDaemonId: string | undefined,
|
|
81
|
+
candidateDaemonIds: string[],
|
|
82
|
+
pulls: Array<Record<string, unknown>>,
|
|
83
|
+
): Promise<void> {
|
|
84
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
85
|
+
if (!dispatchMeshCommand) return;
|
|
86
|
+
const nodeDaemonId = readNonEmptyString(node.daemonId);
|
|
87
|
+
// Skip nodes without a daemon, and nodes on THIS daemon (their events are
|
|
88
|
+
// already in the local queue drained in PHASE 2). "This daemon" is matched
|
|
89
|
+
// against the full self-identity set (candidateDaemonIds), not just the bare
|
|
90
|
+
// localDaemonId — a self node can be registered under the config-form daemonId
|
|
91
|
+
// (`daemon_<machineId>`) which would NOT equal bare localDaemonId, and pulling
|
|
92
|
+
// from ourselves over P2P is both wasteful and a self-dispatch hazard.
|
|
93
|
+
if (!nodeDaemonId) return;
|
|
94
|
+
if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) return;
|
|
95
|
+
if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) return;
|
|
95
96
|
|
|
96
|
-
|
|
97
|
-
|
|
97
|
+
// Peer-connected pre-check (EVENT-DELIVERY-DELAY fix(a) + OFFLINE-NODE-FANOUT):
|
|
98
|
+
// a degraded peer whose DataChannel is not open would sink this pull into
|
|
99
|
+
// peer.connectQueue and stall until CONNECT_TIMEOUT_MS (90s), formerly freezing
|
|
100
|
+
// the whole serial loop and delaying completion-event recovery from healthy
|
|
101
|
+
// nodes. Skip such a node THIS tick and retry next tick — LOSSLESS: an
|
|
102
|
+
// unconnected peer has not drained anything (drained=0 preserved), so its events
|
|
103
|
+
// are recovered whole on the next successful tick. Skip = delay, never loss.
|
|
104
|
+
// • getter WIRED (cloud) → a null/undefined snapshot means "no peer object
|
|
105
|
+
// right now" = NOT connected (a powered-off node whose failPeer just deleted
|
|
106
|
+
// the peer each cycle). Treat it EXACTLY like state !== 'connected' and skip;
|
|
107
|
+
// dialing here would re-queue for another 90s (the null-race the guard is
|
|
108
|
+
// meant to prevent). Only a snapshot with state === 'connected' proceeds.
|
|
109
|
+
// • getter UNWIRED (standalone) → DO NOT skip; fall through to the legacy path
|
|
110
|
+
// so this stays regression-free (the standalone case the guard's history
|
|
111
|
+
// references).
|
|
112
|
+
const getPeerStatus = components.getMeshPeerConnectionStatus;
|
|
113
|
+
if (getPeerStatus) {
|
|
114
|
+
const peerSnapshot = getPeerStatus(nodeDaemonId);
|
|
115
|
+
if (!peerSnapshot || String(peerSnapshot.state) !== 'connected') return;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
for (const pendingEventArgs of pulls) {
|
|
119
|
+
let events: unknown;
|
|
120
|
+
try {
|
|
121
|
+
events = await dispatchMeshCommand(nodeDaemonId, 'get_pending_mesh_events', pendingEventArgs);
|
|
122
|
+
} catch {
|
|
123
|
+
// Remote pull is best-effort; the node may be offline. Retry next tick.
|
|
124
|
+
break; // node unreachable — don't bother with the other id form this tick.
|
|
125
|
+
}
|
|
126
|
+
const list = extractPendingEvents(events).filter(e => readNonEmptyString(e?.meshId) === meshId);
|
|
127
|
+
for (const event of list) {
|
|
128
|
+
const payload = buildForwardPayloadFromPending(event);
|
|
129
|
+
if (!payload.event || !payload.meshId) continue;
|
|
98
130
|
try {
|
|
99
|
-
|
|
100
|
-
} catch {
|
|
101
|
-
// Remote pull is best-effort; the node may be offline. Retry next tick.
|
|
102
|
-
break; // node unreachable — don't bother with the other id form this tick.
|
|
103
|
-
}
|
|
104
|
-
const list = extractPendingEvents(events).filter(e => readNonEmptyString(e?.meshId) === meshId);
|
|
105
|
-
for (const event of list) {
|
|
106
|
-
const payload = buildForwardPayloadFromPending(event);
|
|
107
|
-
if (!payload.event || !payload.meshId) continue;
|
|
108
|
-
try {
|
|
109
|
-
handleMeshForwardEvent(components, payload);
|
|
110
|
-
} catch { /* best-effort re-inject */ }
|
|
111
|
-
}
|
|
131
|
+
handleMeshForwardEvent(components, payload);
|
|
132
|
+
} catch { /* best-effort re-inject */ }
|
|
112
133
|
}
|
|
113
|
-
}
|
|
134
|
+
}
|
|
114
135
|
}
|
|
115
136
|
|
|
116
137
|
// Pull the read_chat payload out of whatever envelope the transport returned.
|
|
@@ -17,6 +17,7 @@ import { ProviderCliAdapter } from '../cli-adapters/provider-cli-adapter.js';
|
|
|
17
17
|
import { shortHash } from '../system/hash.js';
|
|
18
18
|
import type { CliProviderModule } from '../cli-adapters/provider-cli-adapter.js';
|
|
19
19
|
import type { MeshSendKeyItem, MeshSendKeyName } from '../cli-adapters/provider-cli-shared.js';
|
|
20
|
+
import { isPurePtyTranscriptProvider } from '../cli-adapters/provider-cli-shared.js';
|
|
20
21
|
import { createCliAdapter } from './spec/route.js';
|
|
21
22
|
import type { PtyRuntimeMetadata, PtyTransportFactory } from '../cli-adapters/pty-transport.js';
|
|
22
23
|
import { StatusMonitor } from './status-monitor.js';
|
|
@@ -2457,6 +2458,20 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
2457
2458
|
}
|
|
2458
2459
|
}
|
|
2459
2460
|
|
|
2461
|
+
// KIMI-PURE-PTY-COMPLETION-EMIT (Fix 3): before firing a false stall, check the
|
|
2462
|
+
// pure-PTY class (no native transcript → nativeSample was null above). A finished
|
|
2463
|
+
// pure-PTY worker sits at a static idle prompt with an in-turn final assistant
|
|
2464
|
+
// message but never emitted generating_completed — the status-agnostic watchdog
|
|
2465
|
+
// would misread that quiet as a wedge. If the PTY transcript proves a finished
|
|
2466
|
+
// turn, emit the missing completion and SUPPRESS the stall (mark the anchor
|
|
2467
|
+
// emitted so this static idle is not re-checked every tick). No-op for
|
|
2468
|
+
// native-source providers and for a genuinely mid-turn / wedged worker with no
|
|
2469
|
+
// final assistant → the real stall fires below unchanged.
|
|
2470
|
+
if (this.tryReconcilePurePtyCompletionForStall(observedStatus)) {
|
|
2471
|
+
this.meshStallEmittedForAnchor = true;
|
|
2472
|
+
return;
|
|
2473
|
+
}
|
|
2474
|
+
|
|
2460
2475
|
// FALSE-STALL-WATCHDOG-OVERFIRE (fix E): per-session refire cooldown. Mark
|
|
2461
2476
|
// this anchor emitted regardless so a still-static anchor does not re-check
|
|
2462
2477
|
// every tick, but suppress the actual coordinator notification (and the
|
|
@@ -2619,6 +2634,83 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
2619
2634
|
return true;
|
|
2620
2635
|
}
|
|
2621
2636
|
|
|
2637
|
+
/**
|
|
2638
|
+
* KIMI-PURE-PTY-COMPLETION-EMIT (Fix 3): stall-path completion reconcile for the
|
|
2639
|
+
* PURE-PTY transcript class (kimi and kin — no native transcript, no provider
|
|
2640
|
+
* authority; see isPurePtyTranscriptProvider). Such a worker whose
|
|
2641
|
+
* generating_completed never emitted (the onTurnStarted idle→idle collapse Fix 1
|
|
2642
|
+
* fixes at the source) sits at a STATIC idle prompt: its PTY output stops the
|
|
2643
|
+
* instant the answer is rendered, so the status-agnostic no-progress watchdog
|
|
2644
|
+
* (checkMeshWorkerStall) reads the finished-but-quiet session as a stall and would
|
|
2645
|
+
* false-fire monitor:no_progress. The native-transcript reconcile
|
|
2646
|
+
* (sampleNativeTranscriptProgress) does NOT cover this class (no native source →
|
|
2647
|
+
* null sample), so the stall path needs its own guard.
|
|
2648
|
+
*
|
|
2649
|
+
* Called from checkMeshWorkerStall just before the fire. Returns true when the
|
|
2650
|
+
* session is a finished pure-PTY turn — idle, no pending response, and a PTY-parsed
|
|
2651
|
+
* in-turn final assistant summary — in which case it emits the missing
|
|
2652
|
+
* generating_completed (idempotent: a real/late emit writes the terminal ledger and
|
|
2653
|
+
* the coordinator's reconcile makes any duplicate a no-op) and the caller SUPPRESSES
|
|
2654
|
+
* the stall. Returns false for every other class/state (native-source provider, a
|
|
2655
|
+
* genuinely mid-turn or wedged worker with no final assistant), so a real stall still
|
|
2656
|
+
* fires unchanged.
|
|
2657
|
+
*
|
|
2658
|
+
* Evidence bar is identical to flushMeshCompletionBeforeCleanup: injected task's turn
|
|
2659
|
+
* genuinely started + an in-turn final assistant summary. Conservative by
|
|
2660
|
+
* construction — no summary ⇒ no proof of completion ⇒ return false and let the real
|
|
2661
|
+
* stall fire.
|
|
2662
|
+
*/
|
|
2663
|
+
private tryReconcilePurePtyCompletionForStall(observedStatus: string): boolean {
|
|
2664
|
+
// Only the pure-PTY class — native-source providers are handled by
|
|
2665
|
+
// sampleNativeTranscriptProgress above.
|
|
2666
|
+
if (!isPurePtyTranscriptProvider(this.provider)) return false;
|
|
2667
|
+
// A finished turn is idle with nothing pending. A generating/waiting session is
|
|
2668
|
+
// mid-turn (the real stall path should evaluate it), so never complete it here.
|
|
2669
|
+
if (observedStatus !== 'idle') return false;
|
|
2670
|
+
if (this.hasAdapterPendingResponse()) return false;
|
|
2671
|
+
|
|
2672
|
+
const taskId = this.completingTurnTaskId();
|
|
2673
|
+
// DOUBLE-EMIT guard: this turn's completion already fired — never re-emit.
|
|
2674
|
+
if (this.lastEmittedCompletion && this.lastEmittedCompletion.taskId === (taskId ?? '')) {
|
|
2675
|
+
return false;
|
|
2676
|
+
}
|
|
2677
|
+
// The injected task's own turn must have genuinely started (guards against a
|
|
2678
|
+
// reused session's stale tail before this task ran).
|
|
2679
|
+
if (!this.injectedTaskHasStartedGenerating()) return false;
|
|
2680
|
+
|
|
2681
|
+
const turnStartedAt = typeof (this.adapter as any)?.currentTurnStartedAt === 'number'
|
|
2682
|
+
? (this.adapter as any).currentTurnStartedAt as number
|
|
2683
|
+
: this.meshTaskInjectedAt || undefined;
|
|
2684
|
+
let parsedMessages: unknown;
|
|
2685
|
+
try {
|
|
2686
|
+
parsedMessages = this.adapter?.getScriptParsedStatus?.()?.messages;
|
|
2687
|
+
} catch { parsedMessages = undefined; }
|
|
2688
|
+
let finalSummary: string | undefined;
|
|
2689
|
+
try {
|
|
2690
|
+
finalSummary = this.completionFinalSummary(parsedMessages, turnStartedAt);
|
|
2691
|
+
} catch { finalSummary = undefined; }
|
|
2692
|
+
// No in-turn final assistant evidence → not a proven turn-end. Let the real
|
|
2693
|
+
// stall fire so a genuinely-wedged worker is still surfaced.
|
|
2694
|
+
if (!finalSummary) return false;
|
|
2695
|
+
|
|
2696
|
+
LOG.warn('CLI', `[${this.type}] reconciling pure-PTY mesh completion from the stall path for session ${this.instanceId} `
|
|
2697
|
+
+ `task=${taskId ?? '(none)'} — PTY is idle-quiet with an in-turn final assistant message but the completion `
|
|
2698
|
+
+ `event never fired (pure-PTY provider); emitting it instead of a false monitor:no_progress.`);
|
|
2699
|
+
if (this.isMeshWorkerSession()) {
|
|
2700
|
+
traceMeshEventStage('fired', this.meshTraceCtx(), 'stall_pure_pty_transcript_completion');
|
|
2701
|
+
}
|
|
2702
|
+
this.emitGeneratingCompleted({
|
|
2703
|
+
chatTitle: '',
|
|
2704
|
+
duration: undefined,
|
|
2705
|
+
timestamp: Date.now(),
|
|
2706
|
+
taskId,
|
|
2707
|
+
finalSummary,
|
|
2708
|
+
evidenceLevel: 'transcript',
|
|
2709
|
+
completionDiagnostic: { source: 'stall_pure_pty_transcript_completion' },
|
|
2710
|
+
});
|
|
2711
|
+
return true;
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2622
2714
|
/**
|
|
2623
2715
|
* AUTOAPPROVE-FLAP-RECUR (Fix A+B): how long a busy blip / modal scroll-out may
|
|
2624
2716
|
* persist before the in-progress settle gate is torn down. For a delegated
|