chatroom-cli 1.128.0 → 1.128.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +388 -36
- package/dist/index.js.map +9 -7
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -20251,6 +20251,16 @@ var init_agent_process_manager_service = __esm(() => {
|
|
|
20251
20251
|
init_check_agent_slot();
|
|
20252
20252
|
});
|
|
20253
20253
|
|
|
20254
|
+
// ../../services/backend/config/reliability.ts
|
|
20255
|
+
var HARNESS_SESSION_READY_TIMEOUT_MS = 5000, NATIVE_DELIVERY_RECONCILE_MS = 1e4, DAEMON_HEARTBEAT_INTERVAL_MS, DAEMON_HEARTBEAT_TTL_MS, AGENT_REQUEST_DEADLINE_MS = 120000, MACHINE_COMMAND_LEASE_RENEWAL_INTERVAL_MS = 20000, MACHINE_COMMAND_DAEMON_ROUTINE_TTL_MS, SCOPE_TARGET_STOP_TIMEOUT_MS = 1e4, AGENT_LIFECYCLE_OPERATION_TIMEOUT_MS = 120000, CONNECTION_CLOSE_REQUEST_TTL_MS, ENHANCER_TERMINAL_JOB_RETENTION_MS;
|
|
20256
|
+
var init_reliability = __esm(() => {
|
|
20257
|
+
DAEMON_HEARTBEAT_INTERVAL_MS = 5 * 60000;
|
|
20258
|
+
DAEMON_HEARTBEAT_TTL_MS = 6 * DAEMON_HEARTBEAT_INTERVAL_MS;
|
|
20259
|
+
MACHINE_COMMAND_DAEMON_ROUTINE_TTL_MS = 5 * 60000;
|
|
20260
|
+
CONNECTION_CLOSE_REQUEST_TTL_MS = 10 * 60000;
|
|
20261
|
+
ENHANCER_TERMINAL_JOB_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
|
20262
|
+
});
|
|
20263
|
+
|
|
20254
20264
|
// ../shared/src/domain/team-kind.ts
|
|
20255
20265
|
var TEAM_PRESET_IDS;
|
|
20256
20266
|
var init_team_kind = __esm(() => {
|
|
@@ -35229,8 +35239,10 @@ var init_agent_lifecycle_fact = __esm(() => {
|
|
|
35229
35239
|
function isDeliverableTaskStatus(status) {
|
|
35230
35240
|
return status === "pending" || status === "acknowledged";
|
|
35231
35241
|
}
|
|
35242
|
+
var ACTIVE_TASK_STATUSES;
|
|
35232
35243
|
var init_assigned_task = __esm(() => {
|
|
35233
35244
|
init_agent_role();
|
|
35245
|
+
ACTIVE_TASK_STATUSES = ["pending", "acknowledged", "in_progress"];
|
|
35234
35246
|
});
|
|
35235
35247
|
|
|
35236
35248
|
// src/daemon/infrastructure/local/harness/services/spawn-prompt.ts
|
|
@@ -35292,6 +35304,17 @@ var init_native_task_injector_logic = __esm(() => {
|
|
|
35292
35304
|
};
|
|
35293
35305
|
});
|
|
35294
35306
|
|
|
35307
|
+
// src/daemon/services/task-service/domain/usecase/uncovered-turn-nudge.ts
|
|
35308
|
+
function buildUncoveredTurnNudgePrompt(args2) {
|
|
35309
|
+
return [
|
|
35310
|
+
`Your previous turn ended, but task ${args2.taskId} has no handoff recorded.`,
|
|
35311
|
+
"If the work is complete, run `chatroom handoff` for this task now, as instructed in the task.",
|
|
35312
|
+
"If you are waiting on something (for example a background command or CI), do not end your turn: wait for it in the foreground, then run `chatroom handoff`.",
|
|
35313
|
+
"If you cannot do either, reply with one line saying what you are waiting on."
|
|
35314
|
+
].join(`
|
|
35315
|
+
`);
|
|
35316
|
+
}
|
|
35317
|
+
|
|
35295
35318
|
// ../../services/backend/src/domain/entities/participant.ts
|
|
35296
35319
|
var NATIVE_WAITING_ACTION = "native:waiting", NATIVE_TASK_INJECTED_ACTION = "native:task-injected", GET_NEXT_TASK_STARTED_ACTION = "get-next-task:started";
|
|
35297
35320
|
|
|
@@ -36324,16 +36347,6 @@ var init_parse_session_augmentation = __esm(() => {
|
|
|
36324
36347
|
init_team_agent_settings();
|
|
36325
36348
|
});
|
|
36326
36349
|
|
|
36327
|
-
// ../../services/backend/config/reliability.ts
|
|
36328
|
-
var HARNESS_SESSION_READY_TIMEOUT_MS = 5000, NATIVE_DELIVERY_RECONCILE_MS = 1e4, DAEMON_HEARTBEAT_INTERVAL_MS, DAEMON_HEARTBEAT_TTL_MS, AGENT_REQUEST_DEADLINE_MS = 120000, MACHINE_COMMAND_LEASE_RENEWAL_INTERVAL_MS = 20000, MACHINE_COMMAND_DAEMON_ROUTINE_TTL_MS, SCOPE_TARGET_STOP_TIMEOUT_MS = 1e4, AGENT_LIFECYCLE_OPERATION_TIMEOUT_MS = 120000, CONNECTION_CLOSE_REQUEST_TTL_MS, ENHANCER_TERMINAL_JOB_RETENTION_MS;
|
|
36329
|
-
var init_reliability = __esm(() => {
|
|
36330
|
-
DAEMON_HEARTBEAT_INTERVAL_MS = 5 * 60000;
|
|
36331
|
-
DAEMON_HEARTBEAT_TTL_MS = 6 * DAEMON_HEARTBEAT_INTERVAL_MS;
|
|
36332
|
-
MACHINE_COMMAND_DAEMON_ROUTINE_TTL_MS = 5 * 60000;
|
|
36333
|
-
CONNECTION_CLOSE_REQUEST_TTL_MS = 10 * 60000;
|
|
36334
|
-
ENHANCER_TERMINAL_JOB_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
|
36335
|
-
});
|
|
36336
|
-
|
|
36337
36350
|
// src/daemon/services/task-service/service/native-cold-session-before-inject.ts
|
|
36338
36351
|
function sleep3(ms) {
|
|
36339
36352
|
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
@@ -36603,7 +36616,7 @@ function resumeHarnessWithPrompt(task, deps, prompt, harnessSessionId) {
|
|
|
36603
36616
|
taskId,
|
|
36604
36617
|
error: error51
|
|
36605
36618
|
});
|
|
36606
|
-
return;
|
|
36619
|
+
return yield* exports_Effect.fail(resumeResult.failure);
|
|
36607
36620
|
}
|
|
36608
36621
|
yield* exports_Effect.tryPromise({
|
|
36609
36622
|
try: () => deps.audit.emit({
|
|
@@ -36732,6 +36745,8 @@ class AgentWorkManager {
|
|
|
36732
36745
|
nativeTaskDeliveryQueue;
|
|
36733
36746
|
deliveryTaskService;
|
|
36734
36747
|
recoveryGates = new Map;
|
|
36748
|
+
respawnStartRetries = new Map;
|
|
36749
|
+
respawnRestartSeq = 0;
|
|
36735
36750
|
disposed = false;
|
|
36736
36751
|
constructor(deps) {
|
|
36737
36752
|
this.deps = deps;
|
|
@@ -36763,7 +36778,10 @@ class AgentWorkManager {
|
|
|
36763
36778
|
releaseTaskAfterTurnFailure: deps.taskService.releaseTaskAfterTurnFailure,
|
|
36764
36779
|
recordDeliveryFailure: deps.taskService.recordDeliveryFailure,
|
|
36765
36780
|
clearDeliveryFailure: deps.taskService.clearDeliveryFailure,
|
|
36766
|
-
|
|
36781
|
+
recordTurnFailure: deps.taskService.recordTurnFailure,
|
|
36782
|
+
decideUncoveredTurnEnd: deps.taskService.decideUncoveredTurnEnd,
|
|
36783
|
+
confirmNudgeSent: deps.taskService.confirmNudgeSent,
|
|
36784
|
+
resetTurnTracking: deps.taskService.resetTurnTracking,
|
|
36767
36785
|
isRedeliveryExhausted: deps.taskService.isRedeliveryExhausted,
|
|
36768
36786
|
clearRedeliveryTracking: deps.taskService.clearRedeliveryTracking,
|
|
36769
36787
|
forgetStaleTask: deps.taskService.forgetStaleTask,
|
|
@@ -36789,6 +36807,7 @@ class AgentWorkManager {
|
|
|
36789
36807
|
this.unsubscribeTaskService = deps.taskService.subscribe((notification) => this.handleTaskServiceNotification(notification));
|
|
36790
36808
|
}
|
|
36791
36809
|
async handleAgentStarted(event) {
|
|
36810
|
+
this.cancelRespawnStart(recoveryKey(event.chatroomId, event.role), `agent_started:${event.reason ?? "unknown"}`);
|
|
36792
36811
|
await this.waitForRoleRecovery(event.chatroomId, event.role);
|
|
36793
36812
|
this.deps.agentTaskState.clear({ chatroomId: event.chatroomId, role: event.role });
|
|
36794
36813
|
if (event.reason === "user.start") {
|
|
@@ -36906,9 +36925,9 @@ class AgentWorkManager {
|
|
|
36906
36925
|
});
|
|
36907
36926
|
}, delay3);
|
|
36908
36927
|
}
|
|
36909
|
-
async
|
|
36928
|
+
async recoverTaskAfterTurnFailure(event, activeTask, phase) {
|
|
36910
36929
|
const { chatroomId, role } = event;
|
|
36911
|
-
const { exceeded } = await this.deps.taskService.
|
|
36930
|
+
const { exceeded } = await this.deps.taskService.recordTurnFailure({
|
|
36912
36931
|
chatroomId,
|
|
36913
36932
|
role,
|
|
36914
36933
|
taskId: activeTask.taskId
|
|
@@ -36937,7 +36956,7 @@ class AgentWorkManager {
|
|
|
36937
36956
|
const errorDetail = completion.error;
|
|
36938
36957
|
console.error(`[NativeDelivery:turn-failed] chatroom=${event.chatroomId} role=${event.role} task=${activeTask2?.taskId ?? "none"} turn=${completion.turnId} status=${completion.status} source=${completion.source} error=${errorDetail ?? "none"}`);
|
|
36939
36958
|
if (activeTask2) {
|
|
36940
|
-
const recovery = await this.
|
|
36959
|
+
const recovery = await this.recoverTaskAfterTurnFailure(event, activeTask2, "turn-failed");
|
|
36941
36960
|
if (recovery === "exceeded") {
|
|
36942
36961
|
this.deps.agentTaskState.clear({ chatroomId: event.chatroomId, role: event.role });
|
|
36943
36962
|
return { kind: "release-slot" };
|
|
@@ -36977,14 +36996,41 @@ class AgentWorkManager {
|
|
|
36977
36996
|
});
|
|
36978
36997
|
if (activeTask) {
|
|
36979
36998
|
const handoff = await this.deps.taskService.getLatestHandoff(event.chatroomId, event.role);
|
|
36980
|
-
|
|
36981
|
-
|
|
36982
|
-
|
|
36983
|
-
|
|
36999
|
+
const covered = handoff?.taskIds.includes(activeTask.taskId) ?? false;
|
|
37000
|
+
if (covered) {
|
|
37001
|
+
this.deps.taskService.resetTurnTracking({
|
|
37002
|
+
chatroomId: event.chatroomId,
|
|
37003
|
+
role: event.role,
|
|
37004
|
+
taskId: activeTask.taskId
|
|
37005
|
+
});
|
|
37006
|
+
} else {
|
|
37007
|
+
const decision = await this.deps.taskService.decideUncoveredTurnEnd({
|
|
37008
|
+
chatroomId: event.chatroomId,
|
|
37009
|
+
role: event.role,
|
|
37010
|
+
taskId: activeTask.taskId,
|
|
37011
|
+
sessionKey: sessionKeyFor(event.pid, event.slot.harnessSessionId)
|
|
37012
|
+
});
|
|
37013
|
+
if (decision.action === "nudge") {
|
|
37014
|
+
await this.enqueueTurnEndedWaitingFact(event);
|
|
37015
|
+
this.scheduleUncoveredAction(event, activeTask, "nudge");
|
|
37016
|
+
return { kind: "release-slot" };
|
|
37017
|
+
}
|
|
37018
|
+
if (decision.action === "restart-session") {
|
|
37019
|
+
await this.enqueueTurnEndedWaitingFact(event);
|
|
37020
|
+
this.scheduleUncoveredAction(event, activeTask, "restart-session");
|
|
37021
|
+
return { kind: "release-slot" };
|
|
36984
37022
|
}
|
|
37023
|
+
await this.enqueueTurnEndedWaitingFact(event);
|
|
37024
|
+
this.applyParkAction(event);
|
|
37025
|
+
return { kind: "release-slot" };
|
|
36985
37026
|
}
|
|
36986
37027
|
}
|
|
36987
37028
|
this.deps.agentTaskState.clear({ chatroomId: event.chatroomId, role: event.role });
|
|
37029
|
+
await this.enqueueTurnEndedWaitingFact(event);
|
|
37030
|
+
this.scheduleRoleDelivery(event.chatroomId, event.role);
|
|
37031
|
+
return { kind: "release-slot" };
|
|
37032
|
+
}
|
|
37033
|
+
async enqueueTurnEndedWaitingFact(event) {
|
|
36988
37034
|
try {
|
|
36989
37035
|
await this.deps.lifecycleOutbox.enqueue(buildAgentStatusFact({
|
|
36990
37036
|
chatroomId: event.chatroomId,
|
|
@@ -36994,8 +37040,244 @@ class AgentWorkManager {
|
|
|
36994
37040
|
} catch (error51) {
|
|
36995
37041
|
console.error(`[NativeDelivery:turn-ended-outbox-error] chatroom=${event.chatroomId} role=${event.role} error=${error51 instanceof Error ? error51.stack ?? error51.message : String(error51)}`);
|
|
36996
37042
|
}
|
|
37043
|
+
}
|
|
37044
|
+
scheduleUncoveredAction(event, task, action, attempt = 0, delayMs = 0) {
|
|
37045
|
+
setTimeout(() => {
|
|
37046
|
+
if (this.disposed)
|
|
37047
|
+
return;
|
|
37048
|
+
const run = action === "nudge" ? this.sendUncoveredNudge(event, task, attempt) : this.restartSessionForUncoveredTask(event, task);
|
|
37049
|
+
run.catch((error51) => {
|
|
37050
|
+
console.error(`[NativeDelivery:uncovered-action-error] action=${action} chatroom=${event.chatroomId} role=${event.role} task=${task.taskId} error=${error51 instanceof Error ? error51.stack ?? error51.message : String(error51)}`);
|
|
37051
|
+
});
|
|
37052
|
+
}, delayMs);
|
|
37053
|
+
}
|
|
37054
|
+
async sendUncoveredNudge(event, task, attempt = 0) {
|
|
37055
|
+
const { chatroomId, role } = event;
|
|
37056
|
+
const sessionKey = sessionKeyFor(event.pid, event.slot.harnessSessionId);
|
|
37057
|
+
const deliveryState = getRoleDeliveryState();
|
|
37058
|
+
if (!deliveryState.tryAcquireDelivery(chatroomId, role)) {
|
|
37059
|
+
console.log(`[NativeDelivery:uncovered-nudge-skipped] chatroom=${chatroomId} role=${role} task=${task.taskId} reason=delivery_busy`);
|
|
37060
|
+
if (attempt + 1 < MAX_UNCOVERED_NUDGE_SEND_ATTEMPTS) {
|
|
37061
|
+
this.scheduleUncoveredAction(event, task, "nudge", attempt + 1, UNCOVERED_NUDGE_RETRY_MS);
|
|
37062
|
+
return;
|
|
37063
|
+
}
|
|
37064
|
+
await this.escalateUndeliverableNudge(event, task, sessionKey);
|
|
37065
|
+
return;
|
|
37066
|
+
}
|
|
37067
|
+
const generation = deliveryState.getGeneration(chatroomId, role);
|
|
37068
|
+
try {
|
|
37069
|
+
const skipReason = this.uncoveredNudgeSkipReason(event, task);
|
|
37070
|
+
if (skipReason) {
|
|
37071
|
+
console.log(`[NativeDelivery:uncovered-nudge-skipped] chatroom=${chatroomId} role=${role} task=${task.taskId} reason=${skipReason}`);
|
|
37072
|
+
return;
|
|
37073
|
+
}
|
|
37074
|
+
try {
|
|
37075
|
+
await exports_Effect.runPromise(this.deps.agentMgr.resumeTurnForSlot({
|
|
37076
|
+
chatroomId,
|
|
37077
|
+
role,
|
|
37078
|
+
prompt: buildUncoveredTurnNudgePrompt({ taskId: task.taskId })
|
|
37079
|
+
}));
|
|
37080
|
+
} catch (error51) {
|
|
37081
|
+
console.error(`[NativeDelivery:uncovered-nudge] chatroom=${chatroomId} role=${role} task=${task.taskId} error=${error51 instanceof Error ? error51.stack ?? error51.message : String(error51)}`);
|
|
37082
|
+
await this.escalateUndeliverableNudge(event, task, sessionKey);
|
|
37083
|
+
return;
|
|
37084
|
+
}
|
|
37085
|
+
this.deps.taskService.confirmNudgeSent({ chatroomId, role, taskId: task.taskId, sessionKey });
|
|
37086
|
+
} finally {
|
|
37087
|
+
deliveryState.releaseDelivery(chatroomId, role, generation);
|
|
37088
|
+
}
|
|
37089
|
+
}
|
|
37090
|
+
async escalateUndeliverableNudge(event, task, sessionKey) {
|
|
37091
|
+
const decision = await this.deps.taskService.decideUncoveredTurnEnd({
|
|
37092
|
+
chatroomId: event.chatroomId,
|
|
37093
|
+
role: event.role,
|
|
37094
|
+
taskId: task.taskId,
|
|
37095
|
+
sessionKey,
|
|
37096
|
+
nudgeUnavailable: true
|
|
37097
|
+
});
|
|
37098
|
+
if (decision.action === "restart-session") {
|
|
37099
|
+
this.scheduleUncoveredAction(event, task, "restart-session");
|
|
37100
|
+
} else {
|
|
37101
|
+
this.applyParkAction(event);
|
|
37102
|
+
}
|
|
37103
|
+
}
|
|
37104
|
+
async restartSessionForUncoveredTask(event, task) {
|
|
37105
|
+
const { chatroomId, role, pid } = event;
|
|
37106
|
+
const config3 = this.deps.configurationService.get(chatroomId, role);
|
|
37107
|
+
const slot = this.deps.agentMgr.getSlot(chatroomId, role);
|
|
37108
|
+
const harness = config3?.agentHarness ?? slot?.harness;
|
|
37109
|
+
const model = config3?.model ?? slot?.model;
|
|
37110
|
+
const workingDir = config3?.workingDir ?? slot?.workingDir;
|
|
37111
|
+
if (!harness || !model || !workingDir) {
|
|
37112
|
+
console.error(`[NativeDelivery:uncovered-restart-failed] chatroom=${chatroomId} role=${role} task=${task.taskId} error=missing_agent_config`);
|
|
37113
|
+
const sessionKey = sessionKeyFor(event.pid, event.slot.harnessSessionId);
|
|
37114
|
+
await this.deps.taskService.decideUncoveredTurnEnd({
|
|
37115
|
+
chatroomId,
|
|
37116
|
+
role,
|
|
37117
|
+
taskId: task.taskId,
|
|
37118
|
+
sessionKey,
|
|
37119
|
+
nudgeUnavailable: true
|
|
37120
|
+
});
|
|
37121
|
+
this.applyParkAction(event);
|
|
37122
|
+
return;
|
|
37123
|
+
}
|
|
37124
|
+
const startInput = {
|
|
37125
|
+
chatroomId,
|
|
37126
|
+
role,
|
|
37127
|
+
agentHarness: harness,
|
|
37128
|
+
model,
|
|
37129
|
+
workingDir,
|
|
37130
|
+
reason: DaemonStartReasonCode.RESPAWN,
|
|
37131
|
+
wantResume: false
|
|
37132
|
+
};
|
|
37133
|
+
let outcome;
|
|
37134
|
+
try {
|
|
37135
|
+
outcome = await this.deps.runSerializedForAgent({ chatroomId, role, workingDir }, { timeoutMs: AGENT_LIFECYCLE_OPERATION_TIMEOUT_MS }, async (ops, context3) => {
|
|
37136
|
+
const current = this.deps.agentMgr.getSlot(chatroomId, role);
|
|
37137
|
+
if (this.deps.agentMgr.isStopRequested(chatroomId, role) || current?.pid !== pid || current?.harnessSessionId !== event.slot.harnessSessionId) {
|
|
37138
|
+
console.log(`[NativeDelivery:uncovered-restart-stale] chatroom=${chatroomId} role=${role} task=${task.taskId}`);
|
|
37139
|
+
return { kind: "skipped", reason: "stale_slot" };
|
|
37140
|
+
}
|
|
37141
|
+
await this.prepareRoleRecovery({ chatroomId, role });
|
|
37142
|
+
await ops.stopAgent({ chatroomId, role, reason: AgentStopReasonEnum["daemon.respawn"], pid, workingDir }, context3.signal);
|
|
37143
|
+
await this.recoverStoppedRole({ chatroomId, role, mode: "automatic" });
|
|
37144
|
+
return this.attemptRespawnStart(ops, startInput, context3.signal);
|
|
37145
|
+
});
|
|
37146
|
+
} catch (error51) {
|
|
37147
|
+
const message = error51 instanceof Error ? error51.message : String(error51);
|
|
37148
|
+
console.error(`[NativeDelivery:uncovered-restart-failed] chatroom=${chatroomId} role=${role} task=${task.taskId} error=${message}`);
|
|
37149
|
+
if (this.recoveryGates.has(recoveryKey(chatroomId, role))) {
|
|
37150
|
+
this.scheduleRecoveryRetry(chatroomId, role);
|
|
37151
|
+
}
|
|
37152
|
+
return;
|
|
37153
|
+
}
|
|
37154
|
+
if (outcome.kind === "start-failed") {
|
|
37155
|
+
console.error(`[NativeDelivery:uncovered-restart-failed] chatroom=${chatroomId} role=${role} task=${task.taskId} attempt=1 of ${MAX_RESPAWN_START_ATTEMPTS} error=${outcome.error}`);
|
|
37156
|
+
this.beginRespawnStartRetries({ chatroomId, role, workingDir, startInput });
|
|
37157
|
+
}
|
|
37158
|
+
}
|
|
37159
|
+
async attemptRespawnStart(ops, startInput, signal) {
|
|
37160
|
+
try {
|
|
37161
|
+
const started = await ops.startAgent(startInput, signal);
|
|
37162
|
+
return started.success ? { kind: "started" } : { kind: "start-failed", error: started.error ?? "start_rejected" };
|
|
37163
|
+
} catch (error51) {
|
|
37164
|
+
return {
|
|
37165
|
+
kind: "start-failed",
|
|
37166
|
+
error: error51 instanceof Error ? error51.message : String(error51)
|
|
37167
|
+
};
|
|
37168
|
+
}
|
|
37169
|
+
}
|
|
37170
|
+
beginRespawnStartRetries(request3) {
|
|
37171
|
+
if (this.disposed)
|
|
37172
|
+
return;
|
|
37173
|
+
const key = recoveryKey(request3.chatroomId, request3.role);
|
|
37174
|
+
this.cancelRespawnStart(key, "superseded_by_new_restart");
|
|
37175
|
+
this.respawnRestartSeq += 1;
|
|
37176
|
+
const pending = {
|
|
37177
|
+
restartId: this.respawnRestartSeq,
|
|
37178
|
+
request: request3,
|
|
37179
|
+
attempts: 1
|
|
37180
|
+
};
|
|
37181
|
+
this.respawnStartRetries.set(key, pending);
|
|
37182
|
+
this.armRespawnStartRetry(key, pending);
|
|
37183
|
+
}
|
|
37184
|
+
armRespawnStartRetry(key, pending) {
|
|
37185
|
+
if (this.disposed)
|
|
37186
|
+
return;
|
|
37187
|
+
const delay3 = Math.min(30000, 1000 * 2 ** (pending.attempts - 1));
|
|
37188
|
+
pending.timer = setTimeout(() => {
|
|
37189
|
+
pending.timer = undefined;
|
|
37190
|
+
this.retryRespawnStart(key, pending.restartId).catch((error51) => {
|
|
37191
|
+
console.error(`[NativeDelivery:uncovered-restart-error] chatroom=${pending.request.chatroomId} role=${pending.request.role} error=${error51 instanceof Error ? error51.stack ?? error51.message : String(error51)}`);
|
|
37192
|
+
});
|
|
37193
|
+
}, delay3);
|
|
37194
|
+
}
|
|
37195
|
+
async retryRespawnStart(key, restartId) {
|
|
37196
|
+
const pending = this.respawnStartRetries.get(key);
|
|
37197
|
+
if (!pending || pending.restartId !== restartId || this.disposed)
|
|
37198
|
+
return;
|
|
37199
|
+
const { chatroomId, role, workingDir, startInput } = pending.request;
|
|
37200
|
+
if (this.deps.agentMgr.isStopRequested(chatroomId, role)) {
|
|
37201
|
+
this.cancelRespawnStart(key, "stop_requested");
|
|
37202
|
+
return;
|
|
37203
|
+
}
|
|
37204
|
+
const slot = this.deps.agentMgr.getSlot(chatroomId, role);
|
|
37205
|
+
if (slot && isSlotRunning(slot.state)) {
|
|
37206
|
+
this.cancelRespawnStart(key, "agent_already_running");
|
|
37207
|
+
return;
|
|
37208
|
+
}
|
|
37209
|
+
const outcome = await this.deps.runSerializedForAgent({ chatroomId, role, workingDir }, { timeoutMs: AGENT_LIFECYCLE_OPERATION_TIMEOUT_MS }, async (ops, context3) => {
|
|
37210
|
+
if (this.respawnStartRetries.get(key)?.restartId !== restartId) {
|
|
37211
|
+
return { kind: "skipped", reason: "superseded" };
|
|
37212
|
+
}
|
|
37213
|
+
if (this.deps.agentMgr.isStopRequested(chatroomId, role)) {
|
|
37214
|
+
return { kind: "skipped", reason: "stop_requested" };
|
|
37215
|
+
}
|
|
37216
|
+
const current2 = this.deps.agentMgr.getSlot(chatroomId, role);
|
|
37217
|
+
if (current2 && isSlotRunning(current2.state)) {
|
|
37218
|
+
return { kind: "skipped", reason: "agent_already_running" };
|
|
37219
|
+
}
|
|
37220
|
+
return this.attemptRespawnStart(ops, startInput, context3.signal);
|
|
37221
|
+
});
|
|
37222
|
+
if (outcome.kind === "skipped") {
|
|
37223
|
+
this.cancelRespawnStart(key, outcome.reason);
|
|
37224
|
+
return;
|
|
37225
|
+
}
|
|
37226
|
+
const current = this.respawnStartRetries.get(key);
|
|
37227
|
+
if (current?.restartId !== restartId)
|
|
37228
|
+
return;
|
|
37229
|
+
if (outcome.kind === "started") {
|
|
37230
|
+
this.cancelRespawnStart(key, "started");
|
|
37231
|
+
return;
|
|
37232
|
+
}
|
|
37233
|
+
current.attempts += 1;
|
|
37234
|
+
if (current.attempts >= MAX_RESPAWN_START_ATTEMPTS) {
|
|
37235
|
+
this.respawnStartRetries.delete(key);
|
|
37236
|
+
console.error(`[NativeDelivery:uncovered-restart-failed] attempts exhausted chatroom=${chatroomId} role=${role} attempts=${current.attempts} error=${outcome.error}`);
|
|
37237
|
+
return;
|
|
37238
|
+
}
|
|
37239
|
+
console.error(`[NativeDelivery:uncovered-restart-failed] chatroom=${chatroomId} role=${role} attempt=${current.attempts} of ${MAX_RESPAWN_START_ATTEMPTS} error=${outcome.error}`);
|
|
37240
|
+
this.armRespawnStartRetry(key, current);
|
|
37241
|
+
}
|
|
37242
|
+
cancelRespawnStart(key, reason) {
|
|
37243
|
+
const pending = this.respawnStartRetries.get(key);
|
|
37244
|
+
if (!pending)
|
|
37245
|
+
return;
|
|
37246
|
+
if (pending.timer)
|
|
37247
|
+
clearTimeout(pending.timer);
|
|
37248
|
+
pending.timer = undefined;
|
|
37249
|
+
this.respawnStartRetries.delete(key);
|
|
37250
|
+
if (reason === "started")
|
|
37251
|
+
return;
|
|
37252
|
+
console.log(`[NativeDelivery:uncovered-restart-cancelled] chatroom=${pending.request.chatroomId} role=${pending.request.role} attempts=${pending.attempts} reason=${reason}`);
|
|
37253
|
+
}
|
|
37254
|
+
uncoveredNudgeSkipReason(event, task) {
|
|
37255
|
+
const { chatroomId, role } = event;
|
|
37256
|
+
const slot = this.deps.agentMgr.getSlot(chatroomId, role);
|
|
37257
|
+
if (!slot || !isSlotRunning(slot.state))
|
|
37258
|
+
return "slot_not_running";
|
|
37259
|
+
if (slot.pid !== event.pid)
|
|
37260
|
+
return "pid_changed";
|
|
37261
|
+
if (slot.harnessSessionId !== event.slot.harnessSessionId)
|
|
37262
|
+
return "session_changed";
|
|
37263
|
+
if (!isTurnPhaseIdle(slot.nativeTurnPhase ?? "idle"))
|
|
37264
|
+
return "turn_not_idle";
|
|
37265
|
+
if (this.deps.agentMgr.isStopRequested(chatroomId, role))
|
|
37266
|
+
return "stop_requested";
|
|
37267
|
+
if (this.recoveryGates.has(recoveryKey(chatroomId, role)))
|
|
37268
|
+
return "recovery_in_progress";
|
|
37269
|
+
if (this.deps.agentTaskState.get({ chatroomId, role })?.taskId !== task.taskId) {
|
|
37270
|
+
return "task_not_active";
|
|
37271
|
+
}
|
|
37272
|
+
const current = this.deps.taskInboxState.getForRole(chatroomId, role, task.taskId);
|
|
37273
|
+
if (!current || !ACTIVE_TASK_STATUSES.includes(current.status)) {
|
|
37274
|
+
return "task_not_in_inbox";
|
|
37275
|
+
}
|
|
37276
|
+
return;
|
|
37277
|
+
}
|
|
37278
|
+
applyParkAction(event) {
|
|
37279
|
+
this.deps.agentTaskState.clear({ chatroomId: event.chatroomId, role: event.role });
|
|
36997
37280
|
this.scheduleRoleDelivery(event.chatroomId, event.role);
|
|
36998
|
-
return { kind: "release-slot" };
|
|
36999
37281
|
}
|
|
37000
37282
|
scheduleRoleDelivery(chatroomId, role) {
|
|
37001
37283
|
setTimeout(() => {
|
|
@@ -37006,6 +37288,12 @@ class AgentWorkManager {
|
|
|
37006
37288
|
}
|
|
37007
37289
|
dispose() {
|
|
37008
37290
|
this.disposed = true;
|
|
37291
|
+
for (const pending of this.respawnStartRetries.values()) {
|
|
37292
|
+
if (pending.timer)
|
|
37293
|
+
clearTimeout(pending.timer);
|
|
37294
|
+
pending.timer = undefined;
|
|
37295
|
+
}
|
|
37296
|
+
this.respawnStartRetries.clear();
|
|
37009
37297
|
for (const gate of this.recoveryGates.values()) {
|
|
37010
37298
|
if (gate.retryTimer)
|
|
37011
37299
|
clearTimeout(gate.retryTimer);
|
|
@@ -37217,13 +37505,16 @@ class AgentWorkManager {
|
|
|
37217
37505
|
this.deps.agentTaskState.start(args2);
|
|
37218
37506
|
}
|
|
37219
37507
|
}
|
|
37220
|
-
var recoveryKey = (chatroomId, role) => `${chatroomId}:${role.toLowerCase()}`;
|
|
37508
|
+
var MAX_RESPAWN_START_ATTEMPTS = 3, MAX_UNCOVERED_NUDGE_SEND_ATTEMPTS = 3, UNCOVERED_NUDGE_RETRY_MS = 250, recoveryKey = (chatroomId, role) => `${chatroomId}:${role.toLowerCase()}`, sessionKeyFor = (pid, harnessSessionId) => `${pid}:${harnessSessionId ?? "none"}`;
|
|
37221
37509
|
var init_agent_work_manager = __esm(() => {
|
|
37510
|
+
init_reliability();
|
|
37222
37511
|
init_agent();
|
|
37223
37512
|
init_chatroom_workspace_task_inbox();
|
|
37224
37513
|
init_dist();
|
|
37225
37514
|
init_agent_lifecycle_fact();
|
|
37226
37515
|
init_agent_slot();
|
|
37516
|
+
init_assigned_task();
|
|
37517
|
+
init_check_agent_slot();
|
|
37227
37518
|
init_native_task_injector_logic();
|
|
37228
37519
|
init_convex_native_task_delivery_gateway();
|
|
37229
37520
|
init_daemon_audit_port();
|
|
@@ -40322,6 +40613,27 @@ class TaskInboxState {
|
|
|
40322
40613
|
}
|
|
40323
40614
|
}
|
|
40324
40615
|
|
|
40616
|
+
// src/daemon/services/task-service/domain/usecase/uncovered-turn-ladder.ts
|
|
40617
|
+
function decideUncoveredTurnEnd(entry, sessionKey, opts = {}) {
|
|
40618
|
+
const current = entry ?? FRESH_ENTRY;
|
|
40619
|
+
if (isNudgeDue(current, sessionKey, opts))
|
|
40620
|
+
return { action: "nudge" };
|
|
40621
|
+
return current.sessionRestarts < MAX_SESSION_RESTARTS_PER_TASK ? { action: "restart-session" } : { action: "park" };
|
|
40622
|
+
}
|
|
40623
|
+
function isNudgeDue(entry, sessionKey, opts) {
|
|
40624
|
+
return !opts.nudgeUnavailable && entry.nudgedSessionKey !== sessionKey;
|
|
40625
|
+
}
|
|
40626
|
+
function withNudgeSent(entry, sessionKey) {
|
|
40627
|
+
return { nudgedSessionKey: sessionKey, sessionRestarts: entry?.sessionRestarts ?? 0 };
|
|
40628
|
+
}
|
|
40629
|
+
function withSessionRestart(entry) {
|
|
40630
|
+
return { nudgedSessionKey: undefined, sessionRestarts: (entry?.sessionRestarts ?? 0) + 1 };
|
|
40631
|
+
}
|
|
40632
|
+
var MAX_SESSION_RESTARTS_PER_TASK = 1, FRESH_ENTRY;
|
|
40633
|
+
var init_uncovered_turn_ladder = __esm(() => {
|
|
40634
|
+
FRESH_ENTRY = { sessionRestarts: 0 };
|
|
40635
|
+
});
|
|
40636
|
+
|
|
40325
40637
|
// src/daemon/services/task-service/service/task-service.ts
|
|
40326
40638
|
function createTaskService(deps) {
|
|
40327
40639
|
const gateway = createConvexNativeTaskDeliveryGateway(deps.backend);
|
|
@@ -40339,18 +40651,13 @@ function createTaskService(deps) {
|
|
|
40339
40651
|
let bootstrapRecoveryAttempts = 0;
|
|
40340
40652
|
let bootstrapRecoveryTimer;
|
|
40341
40653
|
let bootstrapRecoveryPromise;
|
|
40342
|
-
const
|
|
40343
|
-
const
|
|
40654
|
+
const TURN_FAILURE_RETRY_CAP = 3;
|
|
40655
|
+
const turnFailureCounts = new Map;
|
|
40656
|
+
const uncoveredLadder = new Map;
|
|
40344
40657
|
const redeliveryExhaustedKeys = new Set;
|
|
40345
40658
|
const redeliveryKey = (chatroomId, role, taskId) => `${chatroomId}:${role.toLowerCase()}:${taskId}`;
|
|
40346
|
-
const
|
|
40659
|
+
const parkTask = async (args2, attempts) => {
|
|
40347
40660
|
const key = redeliveryKey(args2.chatroomId, args2.role, args2.taskId);
|
|
40348
|
-
if (redeliveryExhaustedKeys.has(key))
|
|
40349
|
-
return { exceeded: true };
|
|
40350
|
-
const attempts = (turnEndAttemptCounts.get(key) ?? 0) + 1;
|
|
40351
|
-
turnEndAttemptCounts.set(key, attempts);
|
|
40352
|
-
if (attempts < REDIVERY_ATTEMPT_CAP)
|
|
40353
|
-
return { exceeded: false };
|
|
40354
40661
|
redeliveryExhaustedKeys.add(key);
|
|
40355
40662
|
try {
|
|
40356
40663
|
await gateway.recordDeliveryFailure({
|
|
@@ -40362,13 +40669,50 @@ function createTaskService(deps) {
|
|
|
40362
40669
|
} catch (error51) {
|
|
40363
40670
|
console.warn(`[TaskService] redelivery-exhausted failure record failed chatroom=${args2.chatroomId} role=${args2.role} task=${args2.taskId}: ${error51 instanceof Error ? error51.message : String(error51)}`);
|
|
40364
40671
|
}
|
|
40365
|
-
|
|
40672
|
+
const attemptsSuffix = attempts === undefined ? "" : ` attempts=${attempts}`;
|
|
40673
|
+
console.log(`[NativeDelivery:redelivery-exhausted] chatroom=${args2.chatroomId} role=${args2.role} task=${args2.taskId}${attemptsSuffix} — release/redelivery stopped; agent restart (user.start) resets`);
|
|
40674
|
+
};
|
|
40675
|
+
const recordTurnFailure = async (args2) => {
|
|
40676
|
+
const key = redeliveryKey(args2.chatroomId, args2.role, args2.taskId);
|
|
40677
|
+
if (redeliveryExhaustedKeys.has(key))
|
|
40678
|
+
return { exceeded: true };
|
|
40679
|
+
const attempts = (turnFailureCounts.get(key) ?? 0) + 1;
|
|
40680
|
+
turnFailureCounts.set(key, attempts);
|
|
40681
|
+
if (attempts < TURN_FAILURE_RETRY_CAP)
|
|
40682
|
+
return { exceeded: false };
|
|
40683
|
+
await parkTask(args2, attempts);
|
|
40366
40684
|
return { exceeded: true };
|
|
40367
40685
|
};
|
|
40686
|
+
const decideUncoveredTurnEnd2 = async (args2) => {
|
|
40687
|
+
const key = redeliveryKey(args2.chatroomId, args2.role, args2.taskId);
|
|
40688
|
+
if (redeliveryExhaustedKeys.has(key))
|
|
40689
|
+
return { action: "park" };
|
|
40690
|
+
turnFailureCounts.delete(key);
|
|
40691
|
+
const entry = uncoveredLadder.get(key);
|
|
40692
|
+
const decision = decideUncoveredTurnEnd(entry, args2.sessionKey, {
|
|
40693
|
+
nudgeUnavailable: args2.nudgeUnavailable ?? false
|
|
40694
|
+
});
|
|
40695
|
+
if (decision.action === "restart-session") {
|
|
40696
|
+
uncoveredLadder.set(key, withSessionRestart(entry));
|
|
40697
|
+
} else if (decision.action === "park") {
|
|
40698
|
+
await parkTask(args2);
|
|
40699
|
+
}
|
|
40700
|
+
return decision;
|
|
40701
|
+
};
|
|
40702
|
+
const confirmNudgeSent = (args2) => {
|
|
40703
|
+
const key = redeliveryKey(args2.chatroomId, args2.role, args2.taskId);
|
|
40704
|
+
uncoveredLadder.set(key, withNudgeSent(uncoveredLadder.get(key), args2.sessionKey));
|
|
40705
|
+
};
|
|
40706
|
+
const resetTurnTracking = (args2) => {
|
|
40707
|
+
const key = redeliveryKey(args2.chatroomId, args2.role, args2.taskId);
|
|
40708
|
+
turnFailureCounts.delete(key);
|
|
40709
|
+
uncoveredLadder.delete(key);
|
|
40710
|
+
};
|
|
40368
40711
|
const isRedeliveryExhausted = (args2) => redeliveryExhaustedKeys.has(redeliveryKey(args2.chatroomId, args2.role, args2.taskId));
|
|
40369
40712
|
const clearRedeliveryTrackingForTask = (chatroomId, role, taskId) => {
|
|
40370
40713
|
const key = redeliveryKey(chatroomId, role, taskId);
|
|
40371
|
-
|
|
40714
|
+
turnFailureCounts.delete(key);
|
|
40715
|
+
uncoveredLadder.delete(key);
|
|
40372
40716
|
redeliveryExhaustedKeys.delete(key);
|
|
40373
40717
|
};
|
|
40374
40718
|
const forgetStaleTask = ({
|
|
@@ -40398,9 +40742,13 @@ function createTaskService(deps) {
|
|
|
40398
40742
|
};
|
|
40399
40743
|
const clearRoleRedeliveryTracking = (chatroomId, role) => {
|
|
40400
40744
|
const prefix = `${chatroomId}:${role.toLowerCase()}:`;
|
|
40401
|
-
for (const key of [...
|
|
40745
|
+
for (const key of [...turnFailureCounts.keys()]) {
|
|
40746
|
+
if (key.startsWith(prefix))
|
|
40747
|
+
turnFailureCounts.delete(key);
|
|
40748
|
+
}
|
|
40749
|
+
for (const key of [...uncoveredLadder.keys()]) {
|
|
40402
40750
|
if (key.startsWith(prefix))
|
|
40403
|
-
|
|
40751
|
+
uncoveredLadder.delete(key);
|
|
40404
40752
|
}
|
|
40405
40753
|
for (const key of [...redeliveryExhaustedKeys]) {
|
|
40406
40754
|
if (key.startsWith(prefix))
|
|
@@ -40802,7 +41150,10 @@ function createTaskService(deps) {
|
|
|
40802
41150
|
taskId,
|
|
40803
41151
|
...expectedReason ? { expectedReason } : {}
|
|
40804
41152
|
}),
|
|
40805
|
-
|
|
41153
|
+
recordTurnFailure,
|
|
41154
|
+
decideUncoveredTurnEnd: decideUncoveredTurnEnd2,
|
|
41155
|
+
confirmNudgeSent,
|
|
41156
|
+
resetTurnTracking,
|
|
40806
41157
|
isRedeliveryExhausted,
|
|
40807
41158
|
clearRedeliveryTracking: ({ chatroomId, role }) => {
|
|
40808
41159
|
clearRoleRedeliveryTracking(chatroomId, role);
|
|
@@ -40817,6 +41168,7 @@ var init_task_service = __esm(() => {
|
|
|
40817
41168
|
init_task_inbox_acknowledgement_retry_watcher();
|
|
40818
41169
|
init_task_inbox_delivery_retry_watcher();
|
|
40819
41170
|
init_api3();
|
|
41171
|
+
init_uncovered_turn_ladder();
|
|
40820
41172
|
init_convex_native_task_delivery_gateway();
|
|
40821
41173
|
});
|
|
40822
41174
|
|
|
@@ -137918,4 +138270,4 @@ program2.hook("preAction", async (_thisCommand, actionCommand) => {
|
|
|
137918
138270
|
});
|
|
137919
138271
|
program2.parse();
|
|
137920
138272
|
|
|
137921
|
-
//# debugId=
|
|
138273
|
+
//# debugId=7C84BEF97B16C1D064756E2164756E21
|