@adhdev/daemon-core 0.9.82-rc.458 → 0.9.82-rc.459
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +562 -178
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +563 -179
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-queue-assignment.d.ts +28 -0
- package/dist/mesh/mesh-reconcile-loop.d.ts +1 -0
- package/dist/providers/chat-message-normalization.d.ts +26 -0
- package/dist/providers/cli-provider-instance.d.ts +7 -0
- package/dist/providers/native-history/antigravity-claim-registry.d.ts +28 -0
- package/dist/providers/native-history/antigravity-cli-transcript.d.ts +11 -0
- package/dist/providers/native-history/dispatcher.d.ts +4 -0
- package/package.json +3 -3
- package/src/mesh/mesh-events-stale.ts +55 -4
- package/src/mesh/mesh-queue-assignment.ts +220 -3
- package/src/mesh/mesh-reconcile-loop.ts +83 -10
- package/src/providers/chat-message-normalization.ts +44 -10
- package/src/providers/cli-provider-instance.ts +46 -0
- package/src/providers/native-history/antigravity-claim-registry.ts +131 -0
- package/src/providers/native-history/antigravity-cli-transcript.ts +154 -4
- package/src/providers/native-history/dispatcher.ts +150 -20
|
@@ -3,6 +3,12 @@ import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
|
3
3
|
export declare function __resetIdleAutoFastForwardForTests(): void;
|
|
4
4
|
export declare function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined;
|
|
5
5
|
export declare function tryAssignQueueTask(components: DaemonComponents, meshId: string, nodeId: string, sessionId: string, providerType: string): boolean;
|
|
6
|
+
interface AwaitClaimBackoffState {
|
|
7
|
+
cycles: number;
|
|
8
|
+
nextAttemptAtMs: number;
|
|
9
|
+
}
|
|
10
|
+
export declare function __resetAutoLaunchAwaitClaimBackoffForTests(): void;
|
|
11
|
+
export declare function __seedAutoLaunchAwaitClaimBackoffForTests(meshId: string, taskId: string, state: AwaitClaimBackoffState): void;
|
|
6
12
|
/** Active assignments that hold the one-active-per-node / global-parallel invariant
|
|
7
13
|
* (everything except read-only diagnoses, which run unbounded by the write cap). */
|
|
8
14
|
export declare function activeWriteAssignedCount(meshId: string): number;
|
|
@@ -65,6 +71,28 @@ export declare function sessionHasActiveAssignment(meshId: string, sessionId: st
|
|
|
65
71
|
* dead/stale session is not generating → returns false → the requeue proceeds as before.
|
|
66
72
|
*/
|
|
67
73
|
export declare function isSessionActivelyGenerating(components: DaemonComponents, sessionId: string): boolean;
|
|
74
|
+
/**
|
|
75
|
+
* RECLAIM-FALSEPOS tri-state busy verdict for a session id.
|
|
76
|
+
*
|
|
77
|
+
* The binary isSessionActivelyGenerating() folds "absence of a positive generating
|
|
78
|
+
* signal" into a definitive NEGATIVE (returns false when the instance is absent). But a
|
|
79
|
+
* REMOTE session (never in THIS daemon's instanceManager) — or a locally-present session
|
|
80
|
+
* looked up under a skewed id form — then looks "not generating" and can be reclaimed out
|
|
81
|
+
* from under a worker that is genuinely mid-turn. This resolves an explicit three-way
|
|
82
|
+
* verdict instead:
|
|
83
|
+
* - GENERATING — a locally-present instance reports an active/streaming state.
|
|
84
|
+
* - IDLE_CONFIRMED — a locally-present instance reports a non-active (idle/terminal)
|
|
85
|
+
* state. Positive local evidence the worker is not working.
|
|
86
|
+
* - UNKNOWN — no locally-present instance matches (remote / gone / id-skew) or
|
|
87
|
+
* the observation failed. NEVER treated as IDLE_CONFIRMED.
|
|
88
|
+
*
|
|
89
|
+
* The lookup scans getByCategory('cli') with sessionIdsEquivalent (the same equivalence
|
|
90
|
+
* matching nodeHasActiveMeshWork / liveSessionCountForNode use) rather than a raw
|
|
91
|
+
* instanceManager.getInstance(id) Map.get, so an id-form-skewed but present session is
|
|
92
|
+
* found (closing the same id-form-skew hole class e245c2f9's F1 fixed elsewhere).
|
|
93
|
+
*/
|
|
94
|
+
export type SessionBusyVerdict = 'GENERATING' | 'IDLE_CONFIRMED' | 'UNKNOWN';
|
|
95
|
+
export declare function resolveSessionBusyVerdict(components: DaemonComponents, sessionId: string): SessionBusyVerdict;
|
|
68
96
|
export interface MeshQueueTriggerResult {
|
|
69
97
|
success: true;
|
|
70
98
|
meshId: string;
|
|
@@ -70,6 +70,7 @@ export declare function resolveCoordinatorDrainDeliverability(components: Pick<D
|
|
|
70
70
|
export declare function shouldHoldPendingDrainForBusyLocalCoordinator(components: Pick<DaemonComponents, 'instanceManager'> & {
|
|
71
71
|
statusInstanceId?: string;
|
|
72
72
|
}, meshId: string, requestedCoordinatorDaemonId?: string | null, callerIsSelfCoordinatorInboxRead?: boolean): boolean;
|
|
73
|
+
export declare function __resetReclaimUnknownStreakForTests(): void;
|
|
73
74
|
export declare function runMeshReconcileTick(components: DaemonComponents): Promise<void>;
|
|
74
75
|
export declare function __resetUnresolvedForwardRejectionCountsForTests(): void;
|
|
75
76
|
interface ReconcileLoopHandle {
|
|
@@ -32,6 +32,32 @@ export declare function extractFinalAssistantSummaryEvidence(messages: ChatMessa
|
|
|
32
32
|
finalSummary: string;
|
|
33
33
|
transcriptMessageAt?: string;
|
|
34
34
|
};
|
|
35
|
+
/**
|
|
36
|
+
* EARLYNOTIFY-GATEBYPASS (a)/(b) — the shared turn-finality selector for the completion
|
|
37
|
+
* final-assistant judgement, so the ~duplicated "which bubble is the turn's final answer"
|
|
38
|
+
* logic is decided ONE way (UNIFY A-6).
|
|
39
|
+
*
|
|
40
|
+
* Returns the message that qualifies as the turn's FINAL assistant bubble, or null when the
|
|
41
|
+
* transcript does not (yet) prove a turn end. The rule is a NON-EMPTY LATEST user-facing
|
|
42
|
+
* assistant/model bubble:
|
|
43
|
+
* - Scanning from the end, the FIRST user-facing assistant/model bubble encountered IS the
|
|
44
|
+
* turn-end candidate. If it is EMPTY (a streaming placeholder / mid-turn narration whose
|
|
45
|
+
* text has not landed), the turn is still in flight → return null. Crucially we do NOT walk
|
|
46
|
+
* back past that empty bubble to promote an EARLIER assistant narration to "final" (the
|
|
47
|
+
* Defect-B walk-back).
|
|
48
|
+
* - Trailing activity/internal bubbles (tool/thought/status) are skipped — they are not the
|
|
49
|
+
* assistant's user-facing answer.
|
|
50
|
+
* - A trailing user-facing USER message (a freshly dispatched task with no reply yet) means the
|
|
51
|
+
* assistant did not have the last word — no earlier bubble is promoted here; callers that must
|
|
52
|
+
* still reach a prior turn's tail use the timestamp-scoped extractor instead.
|
|
53
|
+
*
|
|
54
|
+
* A bare snapshot-idle with an arbitrary non-empty tail therefore does NOT qualify as a turn end;
|
|
55
|
+
* only a genuine latest-assistant bubble does. Turn-finality signals that live OUTSIDE the
|
|
56
|
+
* transcript (a committed generating→idle FSM transition, a self-attributing final_summary_json,
|
|
57
|
+
* or a continuous-idle streak) are enforced by the callers (the CLI completion gate, the reconcile
|
|
58
|
+
* grace gate) on top of this structural check.
|
|
59
|
+
*/
|
|
60
|
+
export declare function selectFinalAssistantTurnEndMessage(messages: ChatMessage[] | null | undefined): ChatMessage | null;
|
|
35
61
|
export declare const BUILTIN_CHAT_MESSAGE_KINDS: readonly ["standard", "thought", "tool", "terminal", "system"];
|
|
36
62
|
export type BuiltinChatMessageKind = typeof BUILTIN_CHAT_MESSAGE_KINDS[number];
|
|
37
63
|
export type ChatMessageKind = BuiltinChatMessageKind | (string & {});
|
|
@@ -268,6 +268,13 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
268
268
|
recordAcknowledgedUserInput(input: InputEnvelope | string): void;
|
|
269
269
|
/** Drop user-input ack entries older than the dedup window so the map can't grow unbounded. */
|
|
270
270
|
private pruneRecentUserInputAcks;
|
|
271
|
+
/**
|
|
272
|
+
* Owner token for this session in the antigravity conversation-claim
|
|
273
|
+
* registry. Derived identically to the dispatcher's read-side token
|
|
274
|
+
* (workspace + spawn time) so the claims the dispatcher records under this
|
|
275
|
+
* session are the ones dispose() releases.
|
|
276
|
+
*/
|
|
277
|
+
private antigravityClaimOwner;
|
|
271
278
|
dispose(): void;
|
|
272
279
|
private completedDebounceTimer;
|
|
273
280
|
private completedDebouncePending;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** A claim older than this with no refresh is reclaimable (owner presumed dead). */
|
|
2
|
+
export declare const CLAIM_STALE_MS: number;
|
|
3
|
+
/**
|
|
4
|
+
* Derive the per-session owner token. Both the dispatcher (from the read input:
|
|
5
|
+
* workspace + sessionStartedAtMs) and the provider instance (from its
|
|
6
|
+
* workingDir + startedAt, or its instanceId) call this with the same inputs so
|
|
7
|
+
* claims and releases line up. Returns '' when there is no stable identity to
|
|
8
|
+
* key on (e.g. a workspace-less discovery with no spawn time) — the caller then
|
|
9
|
+
* skips claiming but the exclusion checks still run against existing claims.
|
|
10
|
+
*/
|
|
11
|
+
export declare function antigravityOwnerToken(workspace: string, sessionStartedAtMs: number, instanceId?: string): string;
|
|
12
|
+
/**
|
|
13
|
+
* Claim `uuid` for `owner`. Succeeds (and refreshes) when the conversation is
|
|
14
|
+
* unclaimed, already owned by this owner, or held by a stale (abandoned) owner.
|
|
15
|
+
* Fails when a DIFFERENT live owner holds it. Returns whether `owner` holds the
|
|
16
|
+
* claim after the call.
|
|
17
|
+
*/
|
|
18
|
+
export declare function claimAntigravityConversation(uuid: string, owner: string, now?: number): boolean;
|
|
19
|
+
/** True when `uuid` is held by a live owner OTHER than `owner`. */
|
|
20
|
+
export declare function isAntigravityConversationClaimedByOther(uuid: string, owner: string, now?: number): boolean;
|
|
21
|
+
/** The live owner of `uuid`, or undefined when unclaimed or stale. */
|
|
22
|
+
export declare function antigravityConversationOwner(uuid: string, now?: number): string | undefined;
|
|
23
|
+
/** Release a single conversation, only if held by `owner` (or `owner` empty). */
|
|
24
|
+
export declare function releaseAntigravityConversation(uuid: string, owner?: string): void;
|
|
25
|
+
/** Release every conversation held by `owner` (called on session shutdown). */
|
|
26
|
+
export declare function releaseAntigravityOwner(owner: string): void;
|
|
27
|
+
/** Test-only: wipe all claims so each test starts from a clean registry. */
|
|
28
|
+
export declare function __resetAntigravityClaimRegistry(): void;
|
|
@@ -38,6 +38,17 @@
|
|
|
38
38
|
* pty parser (which only echoes the user's own input) — assistant
|
|
39
39
|
* answers appeared lost even though they were on disk.
|
|
40
40
|
*
|
|
41
|
+
* Schema-drift resilience: the exact field path (20 → 1/8 for the answer,
|
|
42
|
+
* 19 → 2/3 for the prompt) is empirically verified against real stores, but
|
|
43
|
+
* antigravity may move it in a future build. So when the known path yields
|
|
44
|
+
* no text, instead of silently dropping the turn we fall back to a UTF-8
|
|
45
|
+
* printable-run scan of the payload (recoverMessageText) that recovers the
|
|
46
|
+
* answer/prompt even if the field number drifted — while explicitly
|
|
47
|
+
* EXCLUDING the reasoning subtree (20 → 3) so internal reasoning is never
|
|
48
|
+
* surfaced as the answer. Only when even that finds nothing beyond
|
|
49
|
+
* reasoning/metadata is the step dropped, and a content-free DEBUG
|
|
50
|
+
* breadcrumb is logged so a real drift is greppable rather than invisible.
|
|
51
|
+
*
|
|
41
52
|
* This adapter provides:
|
|
42
53
|
* - Full coverage from a per-session .db (current format) — preferred.
|
|
43
54
|
* - Full coverage when a brain transcript exists (legacy authoritative source).
|
|
@@ -4,6 +4,10 @@ export interface NativeHistoryInput {
|
|
|
4
4
|
sessionId?: string;
|
|
5
5
|
providerSessionId?: string;
|
|
6
6
|
historySessionId?: string;
|
|
7
|
+
/** Daemon instance id of the reading session. Used (with workspace +
|
|
8
|
+
* sessionStartedAtMs) to derive the antigravity conversation-claim owner
|
|
9
|
+
* token so two concurrent sessions never bind to the same .db. */
|
|
10
|
+
instanceId?: string;
|
|
7
11
|
workspace?: string;
|
|
8
12
|
sessionStartedAtMs?: number;
|
|
9
13
|
format?: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.459",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,8 +46,8 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
50
|
-
"@adhdev/session-host-core": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.459",
|
|
50
|
+
"@adhdev/session-host-core": "0.9.82-rc.459",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
53
53
|
"ajv-formats": "^3.0.1",
|
|
@@ -4,8 +4,19 @@ import { updateDirectDispatchStatus, cleanupTerminalDirectDispatches } from './m
|
|
|
4
4
|
import { markSessionDeliveriesTerminal } from './mesh-delivery-policy.js';
|
|
5
5
|
import { queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
6
6
|
import { readNonEmptyString, readRecord, resolveEventSessionId, readWorkerResultMetadata, isWeakCompletionEvidence, buildMeshSystemMessage } from './mesh-events-utils.js';
|
|
7
|
+
import { recordDebugTrace } from '../logging/debug-trace.js';
|
|
7
8
|
import { meshNodeIdMatches, sessionIdsEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
8
9
|
|
|
10
|
+
// EARLYNOTIFY-GATEBYPASS (d): every completed-emit producer that bypasses the CLI-provider
|
|
11
|
+
// completion gate (transcript-reconcile synth here, no-progress reconcile below, the fast-collapse
|
|
12
|
+
// synth in cli-provider-instance) records a completion-gate trace so a synthesized "completed and
|
|
13
|
+
// idle" emit can never again be silent. Content-free by construction — keyed by taskId + source
|
|
14
|
+
// only, never worker/screen text. completion-gate is an ALWAYS_ON_TRACE_CATEGORY, so recordDebugTrace
|
|
15
|
+
// self-gates and lands in the ring even on a production daemon.
|
|
16
|
+
function recordSynthCompletionGateTrace(stage: string, payload: Record<string, unknown>): void {
|
|
17
|
+
recordDebugTrace({ category: 'completion-gate', stage, level: 'debug', payload });
|
|
18
|
+
}
|
|
19
|
+
|
|
9
20
|
// ---------------------------------------------------------------------------
|
|
10
21
|
// Stale direct-dispatch detection & transcript reconciliation
|
|
11
22
|
// ---------------------------------------------------------------------------
|
|
@@ -232,6 +243,14 @@ export function reconcileDirectDispatchCompletionFromTranscript(args: {
|
|
|
232
243
|
completedAt,
|
|
233
244
|
});
|
|
234
245
|
const workerResult = evidence.workerResult;
|
|
246
|
+
// EARLYNOTIFY-GATEBYPASS (c): a transcript-reconcile synth is TENTATIVE unless the worker's
|
|
247
|
+
// summary self-attributes to this turn — i.e. it parsed a worker-result-shaped JSON
|
|
248
|
+
// (`final_summary_json`), the same self-attribution the grace gate below exempts. A plain-text
|
|
249
|
+
// transcript tail proves neither turn-finality nor that this reconcile beat the real completion,
|
|
250
|
+
// so it is marked WEAK: buildPendingEventFingerprint then keys it `…::weak`, leaving the
|
|
251
|
+
// `…::genuine` slot free for the worker's own later agent:generating_completed to surface (the
|
|
252
|
+
// CANON-B weak→genuine supersession) instead of being dropped as a duplicate.
|
|
253
|
+
const selfAttributing = workerResult.source === 'final_summary_json';
|
|
235
254
|
const dispatchTime = dispatch?.timestamp ? new Date(dispatch.timestamp).getTime() : Number.NaN;
|
|
236
255
|
const transcriptTime = args.transcriptMessageAt ? new Date(args.transcriptMessageAt).getTime() : Number.NaN;
|
|
237
256
|
const transcriptAfterDispatch = Number.isFinite(dispatchTime) && Number.isFinite(transcriptTime) && transcriptTime >= dispatchTime;
|
|
@@ -281,7 +300,9 @@ export function reconcileDirectDispatchCompletionFromTranscript(args: {
|
|
|
281
300
|
dispatchEntryId: dispatch?.id,
|
|
282
301
|
dispatchTimestamp: dispatch?.timestamp,
|
|
283
302
|
transcriptMessageAt: readNonEmptyString(args.transcriptMessageAt),
|
|
284
|
-
|
|
303
|
+
// Honestly reflect self-attribution: a plain-text tail did NOT prove a turn-final
|
|
304
|
+
// assistant message (only a self-attributing final_summary_json did).
|
|
305
|
+
transcriptFinalAssistantPresent: selfAttributing,
|
|
285
306
|
},
|
|
286
307
|
evidence,
|
|
287
308
|
},
|
|
@@ -308,6 +329,11 @@ export function reconcileDirectDispatchCompletionFromTranscript(args: {
|
|
|
308
329
|
finalSummary,
|
|
309
330
|
taskId: args.taskId,
|
|
310
331
|
workerResult,
|
|
332
|
+
// EARLYNOTIFY-GATEBYPASS (c): mark a non-self-attributing synth WEAK so its pending
|
|
333
|
+
// fingerprint is `…::weak` (isWeakCompletionMetadata reads evidenceLevel), never claiming
|
|
334
|
+
// the genuine dedup slot. evidenceLevel:'weak' is deliberately NOT a false-idle marker
|
|
335
|
+
// (the transcript tail existed) — it keeps the completion superseable, not suppressed.
|
|
336
|
+
...(selfAttributing ? {} : { evidenceLevel: 'weak' as const }),
|
|
311
337
|
completionDiagnostic: {
|
|
312
338
|
reason: 'direct_task_transcript_reconciliation',
|
|
313
339
|
terminalLedgerKind: kind,
|
|
@@ -331,6 +357,16 @@ export function reconcileDirectDispatchCompletionFromTranscript(args: {
|
|
|
331
357
|
...(targetCoordinatorSessionId ? { targetCoordinatorSessionId } : {}),
|
|
332
358
|
});
|
|
333
359
|
|
|
360
|
+
// (d) The synth fired — record it so this gate-bypassing emit is observable.
|
|
361
|
+
recordSynthCompletionGateTrace('synth-fire', {
|
|
362
|
+
producer: 'transcript_reconcile',
|
|
363
|
+
source: args.source || 'direct_task_transcript_reconciliation',
|
|
364
|
+
taskId: args.taskId,
|
|
365
|
+
kind,
|
|
366
|
+
selfAttributing,
|
|
367
|
+
evidenceLevel: selfAttributing ? 'sufficient' : 'weak',
|
|
368
|
+
});
|
|
369
|
+
|
|
334
370
|
return { reconciled: true, kind, workerResult, ledgerEntryId: entry.id };
|
|
335
371
|
}
|
|
336
372
|
|
|
@@ -349,15 +385,29 @@ export function buildNoProgressCompletionReconciliation(args: {
|
|
|
349
385
|
const completionDiagnostic = readRecord(args.metadataEvent.completionDiagnostic);
|
|
350
386
|
const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary);
|
|
351
387
|
const status = readNonEmptyString(args.metadataEvent.status).toLowerCase();
|
|
388
|
+
// EARLYNOTIFY-GATEBYPASS (c): a bare status flag (idle/ready/completed) with NO worker text is
|
|
389
|
+
// the weakest possible "done" evidence — it is exactly the false-idle the no-progress monitor
|
|
390
|
+
// fires on. Only a real assistant summary / worker result / confirmed final-assistant makes this
|
|
391
|
+
// reconcile self-attributing. When it is not, mark the synthesized completion WEAK so a later
|
|
392
|
+
// genuine completion can still supersede it (and buildMeshSystemMessage appends a verify hint).
|
|
393
|
+
const noProgressSelfAttributing = Boolean(
|
|
394
|
+
finalSummary || workerResult || completionDiagnostic?.finalAssistantPresent === true,
|
|
395
|
+
);
|
|
352
396
|
const explicitCompletionEvidence = Boolean(
|
|
353
|
-
|
|
354
|
-
|| workerResult
|
|
355
|
-
|| completionDiagnostic?.finalAssistantPresent === true
|
|
397
|
+
noProgressSelfAttributing
|
|
356
398
|
|| status === 'idle'
|
|
357
399
|
|| status === 'ready'
|
|
358
400
|
|| status === 'completed',
|
|
359
401
|
);
|
|
360
402
|
if (explicitCompletionEvidence) {
|
|
403
|
+
// (d) A no-progress→completion synth bypasses the CLI provider gate — trace it.
|
|
404
|
+
recordSynthCompletionGateTrace('synth-fire', {
|
|
405
|
+
producer: 'no_progress_reconcile',
|
|
406
|
+
source: 'no_progress_reconciliation',
|
|
407
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId),
|
|
408
|
+
selfAttributing: noProgressSelfAttributing,
|
|
409
|
+
evidenceLevel: noProgressSelfAttributing ? 'sufficient' : 'weak',
|
|
410
|
+
});
|
|
361
411
|
return {
|
|
362
412
|
...args.metadataEvent,
|
|
363
413
|
targetSessionId: sessionId,
|
|
@@ -367,6 +417,7 @@ export function buildNoProgressCompletionReconciliation(args: {
|
|
|
367
417
|
source: 'no_progress_reconciliation',
|
|
368
418
|
reconciledFromEvent: 'monitor:no_progress',
|
|
369
419
|
timestamp: args.metadataEvent.timestamp ?? Date.now(),
|
|
420
|
+
...(noProgressSelfAttributing ? {} : { evidenceLevel: 'weak' as const }),
|
|
370
421
|
completionDiagnostic: {
|
|
371
422
|
...(completionDiagnostic || {}),
|
|
372
423
|
reconciliationReason: 'provider_completion_evidence',
|
|
@@ -606,6 +606,80 @@ const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
|
|
|
606
606
|
// seconds) but bounded so a launch that silently never reaches idle is eventually retried.
|
|
607
607
|
const AUTO_LAUNCH_AWAIT_CLAIM_MS = 90_000;
|
|
608
608
|
|
|
609
|
+
// AUTOLAUNCH-CLAIM-CHURN. For a REMOTE node the launch→claim handshake is purely
|
|
610
|
+
// event-sourced: the worker's agent:ready must be pulled (reconcile PHASE 1) to run
|
|
611
|
+
// setRemoteIdleSession before the drain can claim. If that pull is lost, nothing recovers,
|
|
612
|
+
// and after AUTO_LAUNCH_AWAIT_CLAIM_MS the loop used to blindly RESPAWN a new session — whose
|
|
613
|
+
// respawn guards (nodeHasLiveSessionPendingClaim / liveSessionCountForNode) scan only the LOCAL
|
|
614
|
+
// instanceManager, so the remote pending-claim session is invisible and a fresh ghost accumulates
|
|
615
|
+
// every ~90s (observed live 2026-07-04: task 8b188c64, and 7 ghost sessions on this worktree's
|
|
616
|
+
// own task at 11:23-11:34). Instead of respawning on window expiry, we re-drive the claim for the
|
|
617
|
+
// EXISTING session; when its liveness cannot be positively determined we EXTEND the window with
|
|
618
|
+
// exponential backoff (90 → 180 → 360s) and, only after the cap, deliver the task directly into
|
|
619
|
+
// the launched session (the mesh_send_task-equivalent) rather than spawning another worker.
|
|
620
|
+
const AUTO_LAUNCH_AWAIT_CLAIM_BACKOFF_CAP_CYCLES = 2;
|
|
621
|
+
// Local mirror of REMOTE_IDLE_SESSION_TTL_MS (mesh-event-forwarding) — kept here to avoid a
|
|
622
|
+
// cross-module import cycle. Used when (re)registering a launched remote session as an idle
|
|
623
|
+
// claim candidate during the await-claim re-drive.
|
|
624
|
+
const AUTO_LAUNCH_REMOTE_IDLE_TTL_MS = 5 * 60 * 1000;
|
|
625
|
+
|
|
626
|
+
// Per-task await-claim backoff state, keyed `${meshId}::${taskId}`. `cycles` counts how many
|
|
627
|
+
// times the window has been extended; `nextAttemptAtMs` rate-limits the re-drive to the backoff
|
|
628
|
+
// cadence so the 4s reconcile tick does not hammer it. Cleared once the task claims, the direct
|
|
629
|
+
// dispatch fires, or a respawn is authorized. In-memory (per process); a stale entry is harmless
|
|
630
|
+
// (it only defers a respawn) and self-clears on the next resolution.
|
|
631
|
+
interface AwaitClaimBackoffState { cycles: number; nextAttemptAtMs: number; }
|
|
632
|
+
const autoLaunchAwaitClaimBackoff = new Map<string, AwaitClaimBackoffState>();
|
|
633
|
+
|
|
634
|
+
// Test hooks: reset / seed the await-claim backoff state between cases.
|
|
635
|
+
export function __resetAutoLaunchAwaitClaimBackoffForTests(): void {
|
|
636
|
+
autoLaunchAwaitClaimBackoff.clear();
|
|
637
|
+
}
|
|
638
|
+
export function __seedAutoLaunchAwaitClaimBackoffForTests(meshId: string, taskId: string, state: AwaitClaimBackoffState): void {
|
|
639
|
+
autoLaunchAwaitClaimBackoff.set(`${meshId}::${taskId}`, { ...state });
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
// Backoff window for a given cycle count: 90 → 180 → 360s (capped at the cap-cycle multiplier).
|
|
643
|
+
function awaitClaimWindowMs(cycles: number): number {
|
|
644
|
+
return AUTO_LAUNCH_AWAIT_CLAIM_MS * Math.pow(2, Math.min(cycles, AUTO_LAUNCH_AWAIT_CLAIM_BACKOFF_CAP_CYCLES));
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
// Does the coordinator's remote-session view (MeshRuntimeStore remote idle sessions, populated by
|
|
648
|
+
// mesh event forwarding) currently show this session as a live idle claim candidate? Positive
|
|
649
|
+
// evidence the launched remote session is reachable — used to re-drive its claim directly instead
|
|
650
|
+
// of respawning. Absence is NOT proof the session is gone (the agent:ready pull may simply have
|
|
651
|
+
// been lost), so callers treat a false here as UNKNOWN liveness, never a definitive terminal.
|
|
652
|
+
function remoteSessionAppearsLive(meshId: string, sessionId: string): boolean {
|
|
653
|
+
if (!sessionId) return false;
|
|
654
|
+
try {
|
|
655
|
+
return MeshRuntimeStore.getInstance().getRemoteIdleSessions(meshId)
|
|
656
|
+
.some(s => sessionIdsEquivalent(s.sessionId, sessionId));
|
|
657
|
+
} catch {
|
|
658
|
+
return false;
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
// (A) Respawn-guard remote-awareness. The session ids of pending tasks whose auto-launch record
|
|
663
|
+
// targets `nodeId` (status started/completed with a sessionId) and is still inside its await-claim
|
|
664
|
+
// window — the base 90s window OR an active backoff extension. Such a session is ALREADY on its way
|
|
665
|
+
// to claim even when it is REMOTE (invisible to this daemon's instanceManager), so counting it
|
|
666
|
+
// suppresses a duplicate launch that would otherwise spawn a ghost.
|
|
667
|
+
function inWindowAutoLaunchSessionIdsForNode(meshId: string, nodeId: string): string[] {
|
|
668
|
+
const nowMs = Date.now();
|
|
669
|
+
const out: string[] = [];
|
|
670
|
+
for (const task of getQueue(meshId, { status: ['pending'] as any })) {
|
|
671
|
+
const al = task.autoLaunch;
|
|
672
|
+
const sid = al ? readNonEmptyString(al.sessionId) : '';
|
|
673
|
+
if (!al || (al.status !== 'started' && al.status !== 'completed') || !sid) continue;
|
|
674
|
+
if (!daemonIdsEquivalent(al.nodeId, nodeId)) continue;
|
|
675
|
+
const launchedAtMs = Date.parse(al.updatedAt);
|
|
676
|
+
const inBaseWindow = Number.isFinite(launchedAtMs) && nowMs - launchedAtMs < AUTO_LAUNCH_AWAIT_CLAIM_MS;
|
|
677
|
+
const inBackoff = autoLaunchAwaitClaimBackoff.has(`${meshId}::${task.id}`);
|
|
678
|
+
if (inBaseWindow || inBackoff) out.push(sid);
|
|
679
|
+
}
|
|
680
|
+
return out;
|
|
681
|
+
}
|
|
682
|
+
|
|
609
683
|
// De-dup for repeated `skipped` ledger noise: the reconcile loop re-runs the queue
|
|
610
684
|
// trigger every 4s, so a task that can't be claimed (e.g. a remote node with no
|
|
611
685
|
// transport, or a node under cooldown) would otherwise append an identical
|
|
@@ -1110,8 +1184,46 @@ export function isSessionActivelyGenerating(components: DaemonComponents, sessio
|
|
|
1110
1184
|
return sessionStateLooksActive(state);
|
|
1111
1185
|
}
|
|
1112
1186
|
|
|
1187
|
+
/**
|
|
1188
|
+
* RECLAIM-FALSEPOS tri-state busy verdict for a session id.
|
|
1189
|
+
*
|
|
1190
|
+
* The binary isSessionActivelyGenerating() folds "absence of a positive generating
|
|
1191
|
+
* signal" into a definitive NEGATIVE (returns false when the instance is absent). But a
|
|
1192
|
+
* REMOTE session (never in THIS daemon's instanceManager) — or a locally-present session
|
|
1193
|
+
* looked up under a skewed id form — then looks "not generating" and can be reclaimed out
|
|
1194
|
+
* from under a worker that is genuinely mid-turn. This resolves an explicit three-way
|
|
1195
|
+
* verdict instead:
|
|
1196
|
+
* - GENERATING — a locally-present instance reports an active/streaming state.
|
|
1197
|
+
* - IDLE_CONFIRMED — a locally-present instance reports a non-active (idle/terminal)
|
|
1198
|
+
* state. Positive local evidence the worker is not working.
|
|
1199
|
+
* - UNKNOWN — no locally-present instance matches (remote / gone / id-skew) or
|
|
1200
|
+
* the observation failed. NEVER treated as IDLE_CONFIRMED.
|
|
1201
|
+
*
|
|
1202
|
+
* The lookup scans getByCategory('cli') with sessionIdsEquivalent (the same equivalence
|
|
1203
|
+
* matching nodeHasActiveMeshWork / liveSessionCountForNode use) rather than a raw
|
|
1204
|
+
* instanceManager.getInstance(id) Map.get, so an id-form-skewed but present session is
|
|
1205
|
+
* found (closing the same id-form-skew hole class e245c2f9's F1 fixed elsewhere).
|
|
1206
|
+
*/
|
|
1207
|
+
export type SessionBusyVerdict = 'GENERATING' | 'IDLE_CONFIRMED' | 'UNKNOWN';
|
|
1208
|
+
export function resolveSessionBusyVerdict(components: DaemonComponents, sessionId: string): SessionBusyVerdict {
|
|
1209
|
+
if (!sessionId) return 'UNKNOWN';
|
|
1210
|
+
try {
|
|
1211
|
+
const instances = components.instanceManager?.getByCategory?.('cli') || [];
|
|
1212
|
+
const inst = instances.find((i: any) => {
|
|
1213
|
+
const sid = readNonEmptyString(i?.getState?.().instanceId);
|
|
1214
|
+
return sid && sessionIdsEquivalent(sid, sessionId);
|
|
1215
|
+
});
|
|
1216
|
+
if (!inst) return 'UNKNOWN'; // remote / gone / id-form skew not present locally
|
|
1217
|
+
const state = inst.getState?.();
|
|
1218
|
+
if (!state) return 'UNKNOWN';
|
|
1219
|
+
return sessionStateLooksActive(state) ? 'GENERATING' : 'IDLE_CONFIRMED';
|
|
1220
|
+
} catch {
|
|
1221
|
+
return 'UNKNOWN'; // failed observation ⇒ unknown, never a definitive idle
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1113
1225
|
function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
|
|
1114
|
-
|
|
1226
|
+
const localInstances = components.instanceManager.getByCategory('cli').filter((inst: any) => {
|
|
1115
1227
|
const state = inst.getState();
|
|
1116
1228
|
const settings = state.settings as Record<string, unknown> || {};
|
|
1117
1229
|
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
@@ -1122,7 +1234,20 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
|
|
|
1122
1234
|
if (!daemonIdsEquivalent(instNodeId, nodeId)) return false;
|
|
1123
1235
|
const status = readNonEmptyString(state.status).toLowerCase();
|
|
1124
1236
|
return !isTerminalSessionStatus(status);
|
|
1125
|
-
})
|
|
1237
|
+
});
|
|
1238
|
+
let count = localInstances.length;
|
|
1239
|
+
// (A) AUTOLAUNCH-CLAIM-CHURN: also count launched-but-not-yet-claimed sessions targeting this
|
|
1240
|
+
// node whose await-claim window is still open. A REMOTE such session is invisible to the local
|
|
1241
|
+
// instanceManager above, so without this the maxConcurrentSessions cap undercounts it and a
|
|
1242
|
+
// duplicate ghost launch slips through. Exclude any id already represented by a local instance
|
|
1243
|
+
// so a co-located launch is not double-counted.
|
|
1244
|
+
const localSessionIds = localInstances
|
|
1245
|
+
.map((inst: any) => readNonEmptyString(inst.getState().instanceId))
|
|
1246
|
+
.filter(Boolean);
|
|
1247
|
+
for (const sid of inWindowAutoLaunchSessionIdsForNode(meshId, nodeId)) {
|
|
1248
|
+
if (!localSessionIds.some(local => sessionIdsEquivalent(local, sid))) count += 1;
|
|
1249
|
+
}
|
|
1250
|
+
return count;
|
|
1126
1251
|
}
|
|
1127
1252
|
|
|
1128
1253
|
/**
|
|
@@ -1141,6 +1266,12 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
|
|
|
1141
1266
|
* not match, preserving the legitimate first-session spawn.
|
|
1142
1267
|
*/
|
|
1143
1268
|
function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string): boolean {
|
|
1269
|
+
// (A) AUTOLAUNCH-CLAIM-CHURN remote-awareness: a task whose auto-launch record targets this
|
|
1270
|
+
// node and is still inside its await-claim window (base or backoff) already has a session on
|
|
1271
|
+
// its way to claim — even when that session is REMOTE and thus invisible to the local
|
|
1272
|
+
// instanceManager scan below. Treat it as a live pending-claim session so a duplicate launch
|
|
1273
|
+
// is suppressed and no ghost accumulates every ~90s.
|
|
1274
|
+
if (inWindowAutoLaunchSessionIdsForNode(meshId, nodeId).length > 0) return true;
|
|
1144
1275
|
// Session ids currently holding an assigned queue task on this node — those are busy,
|
|
1145
1276
|
// not pending claimers, so they must NOT suppress a (read-only) launch.
|
|
1146
1277
|
const busySessionIds = new Set(
|
|
@@ -1312,6 +1443,69 @@ function readMeshNodeId(node: any): string {
|
|
|
1312
1443
|
return normalizeMeshNodeId(node) ?? '';
|
|
1313
1444
|
}
|
|
1314
1445
|
|
|
1446
|
+
// AUTOLAUNCH-CLAIM-CHURN. The await-claim window for a launched (remote) session has expired
|
|
1447
|
+
// without a claim. Instead of a blind respawn, re-drive the claim for the EXISTING session,
|
|
1448
|
+
// backing off when its liveness is unknown, and only respawning when it is provably unclaimable.
|
|
1449
|
+
// Returns a directive for the caller:
|
|
1450
|
+
// - 'claimed' — the re-drive claimed/dispatched the task into the existing session (progress).
|
|
1451
|
+
// - 'fallback' — the post-cap direct dispatch delivered the task into the existing session.
|
|
1452
|
+
// - 'backoff' — liveness unknown; the window was extended (or is still cooling down). No launch.
|
|
1453
|
+
// - 'respawn' — the session is provably gone/unclaimable; the caller may launch a fresh one.
|
|
1454
|
+
function driveExpiredAwaitClaim(
|
|
1455
|
+
components: DaemonComponents,
|
|
1456
|
+
meshId: string,
|
|
1457
|
+
task: MeshWorkQueueEntry,
|
|
1458
|
+
ctx: { sessionId: string; nodeId: string; providerType: string },
|
|
1459
|
+
): 'claimed' | 'fallback' | 'backoff' | 'respawn' {
|
|
1460
|
+
const { sessionId, nodeId, providerType } = ctx;
|
|
1461
|
+
const backoffKey = `${meshId}::${task.id}`;
|
|
1462
|
+
const nowMs = Date.now();
|
|
1463
|
+
const state = autoLaunchAwaitClaimBackoff.get(backoffKey) || { cycles: 0, nextAttemptAtMs: 0 };
|
|
1464
|
+
// Rate-limit re-drive attempts to the backoff cadence so the 4s reconcile tick does not hammer
|
|
1465
|
+
// a still-cooling-down window. The initial (no-state) expiry proceeds immediately.
|
|
1466
|
+
if (state.nextAttemptAtMs && nowMs < state.nextAttemptAtMs) return 'backoff';
|
|
1467
|
+
|
|
1468
|
+
const atCap = state.cycles >= AUTO_LAUNCH_AWAIT_CLAIM_BACKOFF_CAP_CYCLES;
|
|
1469
|
+
const live = remoteSessionAppearsLive(meshId, sessionId);
|
|
1470
|
+
|
|
1471
|
+
// (B) Re-drive when the remote view shows the session live; (C) after the backoff cap, force the
|
|
1472
|
+
// same direct dispatch unconditionally. Both funnel through tryAssignQueueTask, which
|
|
1473
|
+
// idempotently (re)registers the session, delivers the task message (send_chat), and marks the
|
|
1474
|
+
// row assigned — the exact operation a coordinator performs manually via mesh_send_task. (D)
|
|
1475
|
+
// The setRemoteIdleSession re-register makes this robust to a dropped agent:ready.
|
|
1476
|
+
if ((live || atCap) && nodeId && providerType) {
|
|
1477
|
+
try {
|
|
1478
|
+
MeshRuntimeStore.getInstance().setRemoteIdleSession(meshId, nodeId, sessionId, providerType, nowMs + AUTO_LAUNCH_REMOTE_IDLE_TTL_MS);
|
|
1479
|
+
} catch { /* best-effort re-register */ }
|
|
1480
|
+
const assigned = tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
|
|
1481
|
+
if (assigned) {
|
|
1482
|
+
autoLaunchAwaitClaimBackoff.delete(backoffKey);
|
|
1483
|
+
const isFallback = atCap && !live;
|
|
1484
|
+
recordAutoLaunchEvent(meshId, {
|
|
1485
|
+
phase: 'completed',
|
|
1486
|
+
taskId: task.id,
|
|
1487
|
+
reason: isFallback ? 'await_claim_direct_dispatch_fallback' : 'await_claim_redriven',
|
|
1488
|
+
nodeId,
|
|
1489
|
+
sessionId,
|
|
1490
|
+
});
|
|
1491
|
+
// Content-free progress line (ids only).
|
|
1492
|
+
LOG.info('MeshQueue', `Auto-launch await-claim ${isFallback ? 'direct-dispatch fallback' : 're-drive'} claimed task ${task.id} into existing session ${sessionId} on node ${nodeId} (mesh ${meshId})`);
|
|
1493
|
+
return isFallback ? 'fallback' : 'claimed';
|
|
1494
|
+
}
|
|
1495
|
+
if (atCap) {
|
|
1496
|
+
// The forced dispatch could not claim — the session is genuinely gone/unclaimable.
|
|
1497
|
+
// Authorize a fresh respawn (ghosts were already prevented through the backoff window).
|
|
1498
|
+
autoLaunchAwaitClaimBackoff.delete(backoffKey);
|
|
1499
|
+
return 'respawn';
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
// Liveness unknown (or live-but-not-claimable) and not at cap → extend the window with backoff.
|
|
1503
|
+
const cycles = Math.min(state.cycles + 1, AUTO_LAUNCH_AWAIT_CLAIM_BACKOFF_CAP_CYCLES);
|
|
1504
|
+
autoLaunchAwaitClaimBackoff.set(backoffKey, { cycles, nextAttemptAtMs: nowMs + awaitClaimWindowMs(cycles) });
|
|
1505
|
+
recordAutoLaunchEvent(meshId, { phase: 'skipped', taskId: task.id, reason: 'awaiting_launched_session_claim_backoff', nodeId, sessionId });
|
|
1506
|
+
return 'backoff';
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1315
1509
|
async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
|
|
1316
1510
|
const queue = getQueue(meshId);
|
|
1317
1511
|
// DEPENDSON-GATE-SYMMETRY: status index over the FULL queue (incl. completed)
|
|
@@ -1319,6 +1513,15 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1319
1513
|
// dependency, not just the still-active rows.
|
|
1320
1514
|
const statusById = new Map(queue.map(task => [task.id, task.status] as const));
|
|
1321
1515
|
const pending = queue.filter(task => task.status === 'pending');
|
|
1516
|
+
// AUTOLAUNCH-CLAIM-CHURN: prune await-claim backoff state for tasks of this mesh that are no
|
|
1517
|
+
// longer pending (claimed/completed/cancelled) so the map cannot grow without bound.
|
|
1518
|
+
{
|
|
1519
|
+
const pendingIds = new Set(pending.map(t => t.id));
|
|
1520
|
+
const prefix = `${meshId}::`;
|
|
1521
|
+
for (const key of [...autoLaunchAwaitClaimBackoff.keys()]) {
|
|
1522
|
+
if (key.startsWith(prefix) && !pendingIds.has(key.slice(prefix.length))) autoLaunchAwaitClaimBackoff.delete(key);
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1322
1525
|
if (!pending.length) return false;
|
|
1323
1526
|
|
|
1324
1527
|
// Write cap + read-only cap resolved through the shared helpers from the
|
|
@@ -1367,14 +1570,28 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1367
1570
|
// session never reaches idle within the window, a later tick retries.
|
|
1368
1571
|
if (task.autoLaunch?.status === 'completed' && task.autoLaunch.sessionId) {
|
|
1369
1572
|
const launchedAtMs = Date.parse(task.autoLaunch.updatedAt);
|
|
1573
|
+
const alSessionId = readNonEmptyString(task.autoLaunch.sessionId);
|
|
1574
|
+
const alNodeId = readNonEmptyString(task.autoLaunch.nodeId);
|
|
1575
|
+
const alProvider = readNonEmptyString(task.autoLaunch.providerType);
|
|
1370
1576
|
if (Number.isFinite(launchedAtMs) && Date.now() - launchedAtMs < AUTO_LAUNCH_AWAIT_CLAIM_MS) {
|
|
1371
1577
|
// Record the skip in the ledger ONLY (dedup'd). Do NOT call markAutoLaunch
|
|
1372
1578
|
// here: recordTaskAutoLaunch overwrites task.autoLaunch wholesale, which would
|
|
1373
1579
|
// erase the very `completed` record (status + sessionId + updatedAt) this guard
|
|
1374
1580
|
// reads on the next tick, reopening the duplicate-launch hole it closes.
|
|
1375
|
-
recordAutoLaunchEvent(meshId, { phase: 'skipped', taskId: task.id, reason: 'awaiting_launched_session_claim', nodeId:
|
|
1581
|
+
recordAutoLaunchEvent(meshId, { phase: 'skipped', taskId: task.id, reason: 'awaiting_launched_session_claim', nodeId: alNodeId, sessionId: alSessionId });
|
|
1376
1582
|
continue;
|
|
1377
1583
|
}
|
|
1584
|
+
// AUTOLAUNCH-CLAIM-CHURN: the initial await-claim window expired. Rather than a blind
|
|
1585
|
+
// respawn (which the local-only respawn guards can't dedup for a remote pending-claim
|
|
1586
|
+
// session → ghost accumulation), re-drive the claim for the EXISTING launched session,
|
|
1587
|
+
// backing off on unknown liveness and direct-dispatching after the cap. Only a
|
|
1588
|
+
// 'respawn' directive falls through to a fresh launch below.
|
|
1589
|
+
if (Number.isFinite(launchedAtMs) && alSessionId && alNodeId) {
|
|
1590
|
+
const outcome = driveExpiredAwaitClaim(components, meshId, task, { sessionId: alSessionId, nodeId: alNodeId, providerType: alProvider });
|
|
1591
|
+
if (outcome === 'claimed' || outcome === 'fallback') return true; // progress; suppress a duplicate launch
|
|
1592
|
+
if (outcome === 'backoff') continue; // window extended; no respawn
|
|
1593
|
+
// outcome === 'respawn' → session provably gone; proceed to a fresh launch below.
|
|
1594
|
+
}
|
|
1378
1595
|
}
|
|
1379
1596
|
|
|
1380
1597
|
const candidateNodes = Array.isArray(mesh?.nodes)
|