instar 1.3.607 → 1.3.608
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +297 -31
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +22 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +18 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +6 -0
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/types.d.ts +37 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/monitoring/AutonomousLivenessReconciler.d.ts +278 -0
- package/dist/monitoring/AutonomousLivenessReconciler.d.ts.map +1 -0
- package/dist/monitoring/AutonomousLivenessReconciler.js +660 -0
- package/dist/monitoring/AutonomousLivenessReconciler.js.map +1 -0
- package/dist/monitoring/ResumeQueue.d.ts +10 -0
- package/dist/monitoring/ResumeQueue.d.ts.map +1 -1
- package/dist/monitoring/ResumeQueue.js +12 -0
- package/dist/monitoring/ResumeQueue.js.map +1 -1
- package/dist/scaffold/templates.d.ts.map +1 -1
- package/dist/scaffold/templates.js +4 -0
- package/dist/scaffold/templates.js.map +1 -1
- package/dist/server/AgentServer.d.ts +1 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +6 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/routes.d.ts +4 -0
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +10 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +64 -64
- package/src/scaffold/templates.ts +4 -0
- package/upgrades/1.3.608.md +29 -0
- package/upgrades/side-effects/autonomous-liveness-reconciler.md +39 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/commands/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAoCH,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAS3D,OAAO,EAAE,eAAe,EAAiC,MAAM,iCAAiC,CAAC;AAwBjG,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAkH7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAsBtD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAC1C,OAAO,CAUT;AAyID,UAAU,YAAY;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;2DACuD;IACvD,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAg4CD,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,eAAe,EACzB,cAAc,EAAE,cAAc,EAC9B,YAAY,CAAC,EAAE,YAAY,EAC3B,WAAW,CAAC,EAAE,WAAW,EACzB,WAAW,CAAC,EAAE,WAAW,EACzB,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,EAGvE,UAAU,CAAC,EAAE,MAAM,OAAO,8BAA8B,EAAE,WAAW,GAAG,IAAI,EAK5E,qBAAqB,CAAC,EAAE,MAAM,OAAO,gCAAgC,EAAE,kBAAkB,GAAG,IAAI,EAKhG,mBAAmB,CAAC,EAAE,MAAM,MAAM,GAAG,IAAI,GAAG,SAAS,GACpD,IAAI,CA8eN;AA2lBD,wBAAsB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/commands/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAoCH,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAS3D,OAAO,EAAE,eAAe,EAAiC,MAAM,iCAAiC,CAAC;AAwBjG,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAkH7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAsBtD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAC1C,OAAO,CAUT;AAyID,UAAU,YAAY;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;2DACuD;IACvD,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAg4CD,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,eAAe,EACzB,cAAc,EAAE,cAAc,EAC9B,YAAY,CAAC,EAAE,YAAY,EAC3B,WAAW,CAAC,EAAE,WAAW,EACzB,WAAW,CAAC,EAAE,WAAW,EACzB,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,EAGvE,UAAU,CAAC,EAAE,MAAM,OAAO,8BAA8B,EAAE,WAAW,GAAG,IAAI,EAK5E,qBAAqB,CAAC,EAAE,MAAM,OAAO,gCAAgC,EAAE,kBAAkB,GAAG,IAAI,EAKhG,mBAAmB,CAAC,EAAE,MAAM,MAAM,GAAG,IAAI,GAAG,SAAS,GACpD,IAAI,CA8eN;AA2lBD,wBAAsB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA+yetE;AAED,wBAAsB,UAAU,CAAC,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAsDzE;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAuD5E"}
|
package/dist/commands/server.js
CHANGED
|
@@ -24,7 +24,7 @@ import { TopicProfileOrchestrator, resolvedToApplied, } from '../core/TopicProfi
|
|
|
24
24
|
import { CodexResumeMap } from '../core/CodexResumeMap.js';
|
|
25
25
|
import { paneIdleWithEmptyInput } from '../core/ModelSwapService.js';
|
|
26
26
|
import { escalatedModelIds, normalizeTierEscalationConfig } from '../core/ModelTierEscalation.js';
|
|
27
|
-
import { activeAutonomousJobs, autonomousRunRemainingForTopic } from '../core/AutonomousSessions.js';
|
|
27
|
+
import { activeAutonomousJobs, autonomousRunRemainingForTopic, listAutonomousJobs } from '../core/AutonomousSessions.js';
|
|
28
28
|
import { AGE_LIMIT_ACTIVE_RUN_REASON, COMMITMENT_ACTIVE_RUN_REASON } from '../core/WorkEvidence.js';
|
|
29
29
|
import { gapBEligibleForTopic, recentUserMessageFromHistory, resolveGapBInjectionGate, decideGapBInjection } from '../core/gapBCommitmentEvidence.js';
|
|
30
30
|
import { TopicProfileTransferCarrier, createTopicProfilePullHandler } from '../core/TopicProfileTransferCarrier.js';
|
|
@@ -6419,6 +6419,7 @@ export async function startServer(options) {
|
|
|
6419
6419
|
const rqCfg = config.monitoring?.resumeQueue ?? {};
|
|
6420
6420
|
let resumeQueue = null;
|
|
6421
6421
|
let resumeDrainer = null;
|
|
6422
|
+
let autonomousLivenessReconciler = null;
|
|
6422
6423
|
let prHandLease = null;
|
|
6423
6424
|
// GAP-B Part A surface (spec: autonomous-registration-guarantee.md) — the
|
|
6424
6425
|
// aggregated-attention chokepoint reference, hoisted to the handler's scope.
|
|
@@ -6607,31 +6608,50 @@ export async function startServer(options) {
|
|
|
6607
6608
|
return null;
|
|
6608
6609
|
}
|
|
6609
6610
|
};
|
|
6611
|
+
// ── Shared closures (spec §"Shared closures"): extracted to named consts
|
|
6612
|
+
// and passed by the SAME reference into BOTH the ResumeQueueDrainer and the
|
|
6613
|
+
// AutonomousLivenessReconciler, so "the reaper, the drainer, and the
|
|
6614
|
+
// reconciler agree" is structurally true (no divergent re-created copy —
|
|
6615
|
+
// the operator-stop file-mtime arm in particular cannot be forgotten). ──
|
|
6616
|
+
const sharedCanSpawnSession = () => quotaManager ? quotaManager.canSpawnSession().allowed : true;
|
|
6617
|
+
const sharedSessionCountOk = () => sessionManager.listRunningSessions().length <
|
|
6618
|
+
(config.maxSessions ?? 10);
|
|
6619
|
+
const sharedMigrationInFlight = () => quotaManager?.isMigrationInFlight() ?? false;
|
|
6620
|
+
const sharedLiveSessionForTopic = (topicId) => sessionManager.listRunningSessions().some((s) => resolveTopicForTmux(s.tmuxSession) === topicId);
|
|
6621
|
+
const sharedTopicOwnerElsewhere = (topicId) => {
|
|
6622
|
+
// Pool not wired → single-machine → always local.
|
|
6623
|
+
const reg = sessionOwnershipRegistry;
|
|
6624
|
+
const self = _meshSelfId;
|
|
6625
|
+
if (!reg || !self)
|
|
6626
|
+
return false;
|
|
6627
|
+
const owner = reg.ownerOf(String(topicId));
|
|
6628
|
+
return !!owner && owner !== self;
|
|
6629
|
+
};
|
|
6630
|
+
const sharedOperatorStopSince = (topicId, sinceIso) => {
|
|
6631
|
+
const since = Date.parse(sinceIso);
|
|
6632
|
+
const perTopic = operatorStopsByTopic.get(topicId) ?? 0;
|
|
6633
|
+
let flagAt = 0;
|
|
6634
|
+
try {
|
|
6635
|
+
flagAt = fs.statSync(path.join(_projectDir, '.instar', 'autonomous-emergency-stop')).mtimeMs;
|
|
6636
|
+
}
|
|
6637
|
+
catch { /* no flag */ }
|
|
6638
|
+
return Math.max(perTopic, globalOperatorStopAt, flagAt) > since;
|
|
6639
|
+
};
|
|
6610
6640
|
resumeDrainer = new ResumeQueueDrainer({
|
|
6611
6641
|
queue: rq,
|
|
6612
6642
|
pressureTier: sharedPressureTier,
|
|
6613
|
-
canSpawnSession:
|
|
6614
|
-
sessionCountOk:
|
|
6615
|
-
(config.maxSessions ?? 10),
|
|
6643
|
+
canSpawnSession: sharedCanSpawnSession,
|
|
6644
|
+
sessionCountOk: sharedSessionCountOk,
|
|
6616
6645
|
// No catch: a throwing dep resolves to the SAFE side (blocked)
|
|
6617
6646
|
// inside the drainer's gate — wrapping it here would flip the
|
|
6618
6647
|
// failure to the lenient side.
|
|
6619
|
-
migrationInFlight:
|
|
6620
|
-
liveSessionForTopic:
|
|
6621
|
-
.listRunningSessions()
|
|
6622
|
-
.some((s) => resolveTopicForTmux(s.tmuxSession) === topicId),
|
|
6648
|
+
migrationInFlight: sharedMigrationInFlight,
|
|
6649
|
+
liveSessionForTopic: sharedLiveSessionForTopic,
|
|
6623
6650
|
currentResumeUuid: (topicId) => _topicResumeMap?.get(topicId) ?? null,
|
|
6624
|
-
|
|
6625
|
-
|
|
6626
|
-
|
|
6627
|
-
|
|
6628
|
-
const reg = sessionOwnershipRegistry;
|
|
6629
|
-
const self = _meshSelfId;
|
|
6630
|
-
if (!reg || !self)
|
|
6631
|
-
return false;
|
|
6632
|
-
const owner = reg.ownerOf(String(topicId));
|
|
6633
|
-
return !!owner && owner !== self;
|
|
6634
|
-
},
|
|
6651
|
+
// Shared reference — the reconciler uses the SAME closure (no catch: a
|
|
6652
|
+
// registry error propagates to the drainer's validateReality, which
|
|
6653
|
+
// resolves a throwing dep to the SAFE side (invalidated)).
|
|
6654
|
+
topicOwnerElsewhere: sharedTopicOwnerElsewhere,
|
|
6635
6655
|
topicBindingMatches: (topicId, cwd) => {
|
|
6636
6656
|
const bindings = scopeVerifier?.loadTopicBindings?.();
|
|
6637
6657
|
const binding = bindings?.[String(topicId)];
|
|
@@ -6639,16 +6659,7 @@ export async function startServer(options) {
|
|
|
6639
6659
|
return true; // unbound topic → default project
|
|
6640
6660
|
return path.resolve(cwd).startsWith(path.resolve(binding.projectDir));
|
|
6641
6661
|
},
|
|
6642
|
-
operatorStopSince:
|
|
6643
|
-
const since = Date.parse(sinceIso);
|
|
6644
|
-
const perTopic = operatorStopsByTopic.get(topicId) ?? 0;
|
|
6645
|
-
let flagAt = 0;
|
|
6646
|
-
try {
|
|
6647
|
-
flagAt = fs.statSync(path.join(_projectDir, '.instar', 'autonomous-emergency-stop')).mtimeMs;
|
|
6648
|
-
}
|
|
6649
|
-
catch { /* no flag */ }
|
|
6650
|
-
return Math.max(perTopic, globalOperatorStopAt, flagAt) > since;
|
|
6651
|
-
},
|
|
6662
|
+
operatorStopSince: sharedOperatorStopSince,
|
|
6652
6663
|
// Resume-idle-autonomous fix (spec: resume-idle-autonomous-on-reap.md):
|
|
6653
6664
|
// drain-time liveness re-check for an entry admitted via the
|
|
6654
6665
|
// age-limit-active-run path. Returns true (= run FINISHED) when the topic's
|
|
@@ -6827,6 +6838,242 @@ export async function startServer(options) {
|
|
|
6827
6838
|
}, 30_000);
|
|
6828
6839
|
if (typeof reconcileTimer.unref === 'function')
|
|
6829
6840
|
reconcileTimer.unref();
|
|
6841
|
+
// ── AutonomousLivenessReconciler (spec: autonomous-liveness-reconciler.md) ──
|
|
6842
|
+
// Level-triggered self-heal for a "dead but marked active" autonomous run.
|
|
6843
|
+
// Constructed here (after SessionManager + the ResumeQueueDrainer), inside
|
|
6844
|
+
// the queue-started block where rq + resolveTopicForTmux + the shared
|
|
6845
|
+
// closures are in scope. Gated by the dev-agent gate (enabled OMITTED in
|
|
6846
|
+
// ConfigDefaults). Reuses the SAME shared closures as the drainer.
|
|
6847
|
+
const livenessCfgRaw = config.monitoring?.autonomousLivenessReconciler ?? {};
|
|
6848
|
+
if (resolveDevAgentGate(livenessCfgRaw.enabled, config)) {
|
|
6849
|
+
const { AutonomousLivenessReconciler } = await import('../monitoring/AutonomousLivenessReconciler.js');
|
|
6850
|
+
const livenessAuditPath = path.join(_projectDir, 'logs', 'autonomous-liveness.jsonl');
|
|
6851
|
+
const auditLiveness = (entry) => {
|
|
6852
|
+
try {
|
|
6853
|
+
fs.mkdirSync(path.dirname(livenessAuditPath), { recursive: true });
|
|
6854
|
+
try {
|
|
6855
|
+
const st = fs.statSync(livenessAuditPath);
|
|
6856
|
+
if (st.size > 5 * 1024 * 1024)
|
|
6857
|
+
fs.renameSync(livenessAuditPath, `${livenessAuditPath}.1`);
|
|
6858
|
+
}
|
|
6859
|
+
catch { /* no file yet */ }
|
|
6860
|
+
fs.appendFileSync(livenessAuditPath, JSON.stringify(entry) + '\n');
|
|
6861
|
+
}
|
|
6862
|
+
catch { /* the audit sink never endangers the reconciler */ }
|
|
6863
|
+
};
|
|
6864
|
+
// Durable cap state (separate redie + spawn-failure counters).
|
|
6865
|
+
const livenessCapPath = path.join(_projectDir, 'state', 'autonomous-liveness-cap.json');
|
|
6866
|
+
// Process-local in-memory in-flight-spawn claim map (spec §"in-flight
|
|
6867
|
+
// claim primitive"): NOT a cross-machine lock — criterion 5 ensures only
|
|
6868
|
+
// the owning+lease-holding machine reconciles a topic, so this only
|
|
6869
|
+
// serializes THIS machine's ticks + its own queue/reconciler paths. A
|
|
6870
|
+
// process restart clears it (a claim never outlives its process).
|
|
6871
|
+
const livenessInflight = new Map();
|
|
6872
|
+
// Resolve the agent home for the cwd-escape check (the worktree root is a
|
|
6873
|
+
// legitimate non-project-root path INSIDE the agent home).
|
|
6874
|
+
const agentHomeReal = (() => {
|
|
6875
|
+
try {
|
|
6876
|
+
return fs.realpathSync(path.join(os.homedir(), '.instar', 'agents', config.projectName ?? ''));
|
|
6877
|
+
}
|
|
6878
|
+
catch {
|
|
6879
|
+
// @silent-fallback-ok — no resolvable agent home → resolveCwd falls
|
|
6880
|
+
// back to the project-dir prefix check only (still fail-closed on a
|
|
6881
|
+
// cwd that escapes the project). Never widens the cwd allowlist.
|
|
6882
|
+
return null;
|
|
6883
|
+
}
|
|
6884
|
+
})();
|
|
6885
|
+
const reconciler = new AutonomousLivenessReconciler({
|
|
6886
|
+
now: () => Date.now(),
|
|
6887
|
+
listActiveRuns: () => {
|
|
6888
|
+
// Source of truth = the per-topic run-state files; UNTRUSTED — read
|
|
6889
|
+
// ONLY active+remaining+topicId+started_at (generation key).
|
|
6890
|
+
const out = [];
|
|
6891
|
+
for (const job of listAutonomousJobs(config.stateDir)) {
|
|
6892
|
+
if (job.topic == null)
|
|
6893
|
+
continue;
|
|
6894
|
+
const topicId = Number(job.topic);
|
|
6895
|
+
if (!Number.isFinite(topicId))
|
|
6896
|
+
continue;
|
|
6897
|
+
const remaining = autonomousRunRemainingForTopic(config.stateDir, topicId);
|
|
6898
|
+
if (!remaining)
|
|
6899
|
+
continue; // not active or window over → not a candidate
|
|
6900
|
+
const startedAtMs = job.startedAt ? new Date(job.startedAt).getTime() : NaN;
|
|
6901
|
+
out.push({
|
|
6902
|
+
topicId,
|
|
6903
|
+
remainingSeconds: remaining.remainingSeconds,
|
|
6904
|
+
paused: job.paused,
|
|
6905
|
+
movedTo: null,
|
|
6906
|
+
moveSuspended: false,
|
|
6907
|
+
startedAtMs: Number.isFinite(startedAtMs) ? startedAtMs : null,
|
|
6908
|
+
});
|
|
6909
|
+
}
|
|
6910
|
+
return out;
|
|
6911
|
+
},
|
|
6912
|
+
liveTopicSnapshot: () => {
|
|
6913
|
+
const set = new Set();
|
|
6914
|
+
for (const s of sessionManager.listRunningSessions()) {
|
|
6915
|
+
const t = resolveTopicForTmux(s.tmuxSession);
|
|
6916
|
+
if (t != null)
|
|
6917
|
+
set.add(t);
|
|
6918
|
+
}
|
|
6919
|
+
return set;
|
|
6920
|
+
},
|
|
6921
|
+
queuePaused: () => rq.isPaused(),
|
|
6922
|
+
topicInResumeQueue: (topicId) => rq.list().some((e) => e.topicId === topicId && (e.status === 'queued' || e.status === 'starting')),
|
|
6923
|
+
operatorStoppedSince: sharedOperatorStopSince,
|
|
6924
|
+
topicOwnerElsewhere: sharedTopicOwnerElsewhere,
|
|
6925
|
+
// DEFAULTS TO HELD on a single-machine agent (coordinator.holdsLease()
|
|
6926
|
+
// returns _role==='awake' ⇒ true when no lease coordinator is wired).
|
|
6927
|
+
holdsLease: () => coordinator.holdsLease(),
|
|
6928
|
+
currentGenerationMs: (topicId) => {
|
|
6929
|
+
// Re-read the topic's CURRENT started_at from disk: a newer run on a
|
|
6930
|
+
// reused topic id writes a newer started_at → obsolete prior run rejected.
|
|
6931
|
+
try {
|
|
6932
|
+
const job = listAutonomousJobs(config.stateDir).find((j) => String(j.topic) === String(topicId));
|
|
6933
|
+
if (!job?.startedAt)
|
|
6934
|
+
return null;
|
|
6935
|
+
const ms = new Date(job.startedAt).getTime();
|
|
6936
|
+
return Number.isFinite(ms) ? ms : null;
|
|
6937
|
+
}
|
|
6938
|
+
catch {
|
|
6939
|
+
// @silent-fallback-ok — unreadable generation → null = "no
|
|
6940
|
+
// competing registration"; the reconciler treats the run as
|
|
6941
|
+
// current (the safe side: a missed obsolete-rejection at worst
|
|
6942
|
+
// respawns a run the state file still says is active).
|
|
6943
|
+
return null;
|
|
6944
|
+
}
|
|
6945
|
+
},
|
|
6946
|
+
quotaOk: sharedCanSpawnSession,
|
|
6947
|
+
sessionCountOk: sharedSessionCountOk,
|
|
6948
|
+
migrationInFlight: sharedMigrationInFlight,
|
|
6949
|
+
pressureTier: sharedPressureTier,
|
|
6950
|
+
inflightSpawnStatus: (topicId) => livenessInflight.get(topicId) ?? { state: 'none' },
|
|
6951
|
+
resolveResumeUuid: (topicId) => _topicResumeMap?.get(topicId) ?? null,
|
|
6952
|
+
resolveCwd: (topicId) => {
|
|
6953
|
+
// Authoritative cwd from the topic-binding registry; realpath-resolved
|
|
6954
|
+
// and validated inside the agent home (worktree root is legitimate).
|
|
6955
|
+
try {
|
|
6956
|
+
const bindings = scopeVerifier?.loadTopicBindings?.();
|
|
6957
|
+
const raw = bindings?.[String(topicId)]?.projectDir ?? _projectDir;
|
|
6958
|
+
let real;
|
|
6959
|
+
try {
|
|
6960
|
+
real = fs.realpathSync(raw);
|
|
6961
|
+
}
|
|
6962
|
+
catch {
|
|
6963
|
+
// @silent-fallback-ok — an unresolvable path is REFUSED (null),
|
|
6964
|
+
// never spawned against; the reconciler raises attention.
|
|
6965
|
+
return null;
|
|
6966
|
+
}
|
|
6967
|
+
if (agentHomeReal && !real.startsWith(agentHomeReal) && !real.startsWith(fs.realpathSync(_projectDir))) {
|
|
6968
|
+
// realpath escapes both the agent home AND the project dir → refuse.
|
|
6969
|
+
return null;
|
|
6970
|
+
}
|
|
6971
|
+
return real;
|
|
6972
|
+
}
|
|
6973
|
+
catch {
|
|
6974
|
+
// @silent-fallback-ok — cwd resolution failed → REFUSE (null);
|
|
6975
|
+
// the reconciler raises attention and does NOT spawn against a guess.
|
|
6976
|
+
return null;
|
|
6977
|
+
}
|
|
6978
|
+
},
|
|
6979
|
+
bindingUnambiguous: (topicId) => {
|
|
6980
|
+
// Consistent across the resume map + the running-session list: a
|
|
6981
|
+
// topic resolving to >1 distinct live session is ambiguous.
|
|
6982
|
+
try {
|
|
6983
|
+
const live = sessionManager
|
|
6984
|
+
.listRunningSessions()
|
|
6985
|
+
.filter((s) => resolveTopicForTmux(s.tmuxSession) === topicId);
|
|
6986
|
+
return live.length <= 1; // 0 (orphaned, expected) or 1 → unambiguous
|
|
6987
|
+
}
|
|
6988
|
+
catch {
|
|
6989
|
+
// @silent-fallback-ok — can't resolve the binding → treat as
|
|
6990
|
+
// AMBIGUOUS (false), which routes to attention, never an
|
|
6991
|
+
// auto-respawn (the safe side per criterion 8).
|
|
6992
|
+
return false;
|
|
6993
|
+
}
|
|
6994
|
+
},
|
|
6995
|
+
respawn: async ({ topicId, cwd }) => {
|
|
6996
|
+
if (!telegram)
|
|
6997
|
+
throw new Error('telegram adapter not available');
|
|
6998
|
+
const topicName = `topic-${topicId}`;
|
|
6999
|
+
const newSession = await spawnSessionForTopic(sessionManager, telegram, topicName, topicId, 'Resuming your autonomous run — your previous session had no live process; picking the work back up.', topicMemory, undefined, undefined, undefined, { cwd });
|
|
7000
|
+
// Tag the respawned session midWork so a later reaper kill is offered
|
|
7001
|
+
// to the ResumeQueue (attacking the incident root cause) rather than
|
|
7002
|
+
// re-entering the reconciler's respawn budget.
|
|
7003
|
+
try {
|
|
7004
|
+
const sess = sessionManager.listRunningSessions().find((s) => s.tmuxSession === newSession);
|
|
7005
|
+
if (sess) {
|
|
7006
|
+
sess.endedMidWork = true;
|
|
7007
|
+
state.saveSession(sess);
|
|
7008
|
+
}
|
|
7009
|
+
}
|
|
7010
|
+
catch { /* @silent-fallback-ok: best-effort midWork tag; a failure must never undo the successful respawn (the run is alive, which is the goal). */ }
|
|
7011
|
+
},
|
|
7012
|
+
claimInflight: (topicId) => {
|
|
7013
|
+
if (livenessInflight.has(topicId))
|
|
7014
|
+
return false; // CAS: someone holds it
|
|
7015
|
+
livenessInflight.set(topicId, { state: 'spawning', sinceMs: Date.now() });
|
|
7016
|
+
return true;
|
|
7017
|
+
},
|
|
7018
|
+
releaseClaim: (topicId) => { livenessInflight.delete(topicId); },
|
|
7019
|
+
settleKill: async (topicId) => {
|
|
7020
|
+
// Terminal abort: clear midWork FIRST (so the ResumeQueue does not
|
|
7021
|
+
// revive an operator-stopped topic), then kill — never via revival.
|
|
7022
|
+
try {
|
|
7023
|
+
const sess = sessionManager.listRunningSessions().find((s) => resolveTopicForTmux(s.tmuxSession) === topicId);
|
|
7024
|
+
if (sess) {
|
|
7025
|
+
sess.endedMidWork = false;
|
|
7026
|
+
state.saveSession(sess);
|
|
7027
|
+
sessionManager.killSession(sess.id);
|
|
7028
|
+
}
|
|
7029
|
+
}
|
|
7030
|
+
catch { /* @silent-fallback-ok: settle-kill is best-effort; the operator-stop record already prevents future revival of this topic. */ }
|
|
7031
|
+
},
|
|
7032
|
+
notifyTopic: async (topicId, text) => { notify('SUMMARY', 'autonomous-liveness', text, topicId); },
|
|
7033
|
+
raiseAggregated: (kind, detail) => raiseResumeAggregated(`liveness:${kind}`, detail),
|
|
7034
|
+
audit: auditLiveness,
|
|
7035
|
+
queueResurrectionCount: (topicId) => rq.resurrectionCountForTopic(topicId),
|
|
7036
|
+
loadCapState: () => {
|
|
7037
|
+
try {
|
|
7038
|
+
const raw = fs.readFileSync(livenessCapPath, 'utf8');
|
|
7039
|
+
const parsed = JSON.parse(raw);
|
|
7040
|
+
return { redie: parsed.redie ?? {}, spawnFailure: parsed.spawnFailure ?? {} };
|
|
7041
|
+
}
|
|
7042
|
+
catch {
|
|
7043
|
+
// @silent-fallback-ok — no/corrupt durable cap → start with empty
|
|
7044
|
+
// in-memory counters (a fresh window). The cap is a brake, not a
|
|
7045
|
+
// safety gate; the in-flight claim + recheck prevent over-spawn.
|
|
7046
|
+
return null;
|
|
7047
|
+
}
|
|
7048
|
+
},
|
|
7049
|
+
saveCapState: (st) => {
|
|
7050
|
+
try {
|
|
7051
|
+
fs.mkdirSync(path.dirname(livenessCapPath), { recursive: true });
|
|
7052
|
+
fs.writeFileSync(livenessCapPath, JSON.stringify(st));
|
|
7053
|
+
}
|
|
7054
|
+
catch { /* cap persistence is best-effort */ }
|
|
7055
|
+
},
|
|
7056
|
+
}, {
|
|
7057
|
+
enabled: livenessCfgRaw.enabled,
|
|
7058
|
+
dryRun: livenessCfgRaw.dryRun ?? true, // dryRun-FIRST on dev
|
|
7059
|
+
tickIntervalSec: livenessCfgRaw.tickIntervalSec ?? 120,
|
|
7060
|
+
debounceTicks: livenessCfgRaw.debounceTicks ?? 2,
|
|
7061
|
+
debounceWindowSec: livenessCfgRaw.debounceWindowSec ?? 180,
|
|
7062
|
+
respawnTimeoutMs: livenessCfgRaw.respawnTimeoutMs ?? 45_000,
|
|
7063
|
+
respawnCapPerWindow: livenessCfgRaw.respawnCapPerWindow ?? 3,
|
|
7064
|
+
respawnCapWindowSec: livenessCfgRaw.respawnCapWindowSec ?? 21_600,
|
|
7065
|
+
spawnFailureRetryCeiling: livenessCfgRaw.spawnFailureRetryCeiling ?? 6,
|
|
7066
|
+
maxPressureBlockedTicks: livenessCfgRaw.maxPressureBlockedTicks ?? 10,
|
|
7067
|
+
maxPressureBlockedSec: livenessCfgRaw.maxPressureBlockedSec ?? 1_800,
|
|
7068
|
+
allowFreshFallback: livenessCfgRaw.allowFreshFallback ?? false,
|
|
7069
|
+
inflightSpawnTtlMs: livenessCfgRaw.inflightSpawnTtlMs,
|
|
7070
|
+
notifyUser: livenessCfgRaw.notifyUser ?? true,
|
|
7071
|
+
});
|
|
7072
|
+
reconciler.start();
|
|
7073
|
+
autonomousLivenessReconciler = reconciler;
|
|
7074
|
+
guardRegistry.register('monitoring.autonomousLivenessReconciler.enabled', () => reconciler.guardStatus());
|
|
7075
|
+
console.log(pc.green(` AutonomousLivenessReconciler started (${(livenessCfgRaw.dryRun ?? true) ? 'dry-run observe-only' : 'LIVE'})`));
|
|
7076
|
+
}
|
|
6830
7077
|
}
|
|
6831
7078
|
}
|
|
6832
7079
|
sessionManager.on('sessionReaped', (e) => {
|
|
@@ -6913,7 +7160,26 @@ export async function startServer(options) {
|
|
|
6913
7160
|
try {
|
|
6914
7161
|
if (resumeQueue && !resumeQueue.isDisabled()) {
|
|
6915
7162
|
const rawTopic = telegram?.getTopicForSession(e.session.tmuxSession);
|
|
6916
|
-
|
|
7163
|
+
let topicId = rawTopic == null ? null : Number.isFinite(Number(rawTopic)) ? Number(rawTopic) : null;
|
|
7164
|
+
// ── ROOT-CAUSE FIX (spec §"Root cause"): the session-name→topic map can
|
|
7165
|
+
// be stale/evicted at the kill instant, so getTopicForSession returns
|
|
7166
|
+
// null precisely when an age-limit recycle should be resumed — and the
|
|
7167
|
+
// entire active-autonomous-run injection branch below is skipped, the
|
|
7168
|
+
// run silently dies (the 2026-06-16 incident). FALL BACK to resolving
|
|
7169
|
+
// the topic from the session name (the SAME parse used at the
|
|
7170
|
+
// coherence-journal emit below) and confirm it against the run-state
|
|
7171
|
+
// file, so the autonomousRunRemainingForTopic check still runs.
|
|
7172
|
+
// Confined to the null case (the resolved-topic path is unchanged) and
|
|
7173
|
+
// inside the existing try/catch (a throw fails toward NO injection).
|
|
7174
|
+
if (topicId == null) {
|
|
7175
|
+
const m = /(?:^|[-_])(?:topic|telegram)[-_]?(\d+)(?:$|[-_])/.exec(e.session.name ?? e.session.tmuxSession ?? '');
|
|
7176
|
+
const parsed = m ? Number(m[1]) : NaN;
|
|
7177
|
+
// Only adopt the parsed topic when the run-state file confirms an active
|
|
7178
|
+
// run for it — never resurrect against a guessed-but-unconfirmed id.
|
|
7179
|
+
if (Number.isFinite(parsed) && autonomousRunRemainingForTopic(config.stateDir, parsed) != null) {
|
|
7180
|
+
topicId = parsed;
|
|
7181
|
+
}
|
|
7182
|
+
}
|
|
6917
7183
|
const jobDef = e.session.jobSlug
|
|
6918
7184
|
? scheduler?.getJobs().find((j) => j.slug === e.session.jobSlug)
|
|
6919
7185
|
: undefined;
|
|
@@ -16655,7 +16921,7 @@ export async function startServer(options) {
|
|
|
16655
16921
|
carrier: _topicProfileCarrier,
|
|
16656
16922
|
}
|
|
16657
16923
|
: null;
|
|
16658
|
-
const server = new AgentServer({ config, sessionManager, state, scheduler, telegram, relationships, feedback, feedbackAnomalyDetector, dispatches, updateChecker, autoUpdater, autoDispatcher, quotaTracker, quotaManager, publisher, viewer, tunnel, evolution, watchdog, topicMemory, triageNurse, projectMapper, cartographer: cartographer ?? undefined, coherenceGate: scopeVerifier, contextHierarchy, canonicalState, operationGate, sentinel, adaptiveTrust, memoryMonitor, orphanReaper, coherenceMonitor, commitmentTracker, subscriptionPool, quotaPoller, quotaAwareScheduler: _quotaAwareScheduler ?? undefined, proactiveSwapMonitor: _proactiveSwapMonitor ?? undefined, inUseAccountResolver, enrollmentWizard, credentialRepointing, semanticMemory, activitySentinel, rateLimitSentinel, releaseReadinessSentinel: releaseReadinessSentinel ?? undefined, greenPrAutoMerger: greenPrAutoMerger ?? undefined, guardLatchStore: guardLatchStore ?? undefined, messageRouter, summarySentinel, spawnManager, systemReviewer, capabilityMapper, selfKnowledgeTree, coverageAuditor, topicResumeMap: _topicResumeMap ?? undefined, topicProfile: _topicProfileCtx ?? undefined, sessionRefresh: _sessionRefresh ?? undefined, autonomyManager, trustElevationTracker, autonomousEvolution, coordinator: coordinator.enabled ? coordinator : undefined, localSigningKeyPem, leaseTransport, onLeasePullRequest: () => leaseCoordinatorRef?.currentLease() ?? null, liveTailReceiver, handoffWireTransport, onHandoffBegin, onHandoffInitiate: handoffInitiate, handoffInProgress: handoffSentinelInProgress, messageLedger, currentInboundByTopic, replyMarkerTransport, onReplyMarker: messageLedger ? (marker) => { const m = marker; messageLedger.applyRemoteReplyMarker(m.dedupeKey, { platform: m.platform, replyIdempotencyKey: m.replyIdempotencyKey, epoch: m.epoch, topic: m.topic ?? null }); } : undefined, whatsapp: whatsappAdapter, slack: slackAdapter, imessage: imessageAdapter, whatsappBusinessBackend, messageBridge, hookEventReceiver, worktreeMonitor, subagentTracker, instructionsVerifier, handshakeManager: threadlineHandshake, threadlineRouter, conversationStore, threadLog, threadMessageRecorder, warrantsReplyGate, collaborationSurfacer, threadResumeMap, topicLinkageHandler: topicLinkageHandler ?? undefined, threadlineRelayClient, threadlineReplyWaiters, listenerManager: listenerManager ?? undefined, a2aDeliveryTracker: a2aDeliveryTracker ?? undefined, responseReviewGate, messagingToneGate, outboundDedupGate, telemetryHeartbeat, pasteManager, featureRegistry, discoveryEvaluator, completionEvaluator, unifiedTrust, liveConfig, sharedStateLedger, ledgerSessionRegistry, worktreeManager, oidcEnrolledRepos: parallelDevConfig?.oidcEnrolledRepos, initiativeTracker, projectRoundRunner, projectDriftChecker, machineHeartbeat, machinePoolRegistry, getInboundQueue: () => _inboundQueue, meshRpcDispatcher, workingSetPullCoordinator, commitmentReplicaStore, preferenceReplicaStore, replicatedRecordEmitter, conflictStore, rollbackUnmerge, droppedOriginRegistry, preferencesUnionReader, forwardCommitmentMutate, sessionOwnershipRegistry, topicPinStore: _topicPinStore ?? undefined, streamTicketStore: _streamTicketStore ?? undefined, poolStreamAllowRemoteInput: config.dashboard?.poolStream?.allowRemoteInput ?? false, poolStreamConnector: _poolStreamConnector ?? undefined, secretSync: _secretSyncHandle ?? undefined, meshSelfId: _meshSelfId ?? undefined, resolveRouterUrl: _resolveRouterUrl ?? undefined, resolvePeerUrls: _resolvePeerUrls ?? undefined, guardRegistry, listPoolMachines: _listPoolMachines ?? undefined, poolLink: _poolLink ?? undefined, poolPollCache: _poolPollCache ?? undefined, sessionPoolE2EResultStore, proxyCoordinator, topicIntentStore, topicIntentArcCheck, usherSignalStore, intelligence: sharedIntelligence ?? undefined, telegramBridgeConfig, telegramBridge: telegramBridge ?? undefined, threadlineObservability, briefDeps, workingMemory, taskFlowRegistry, threadlineFlowBridge, sessionReaper, agentWorktreeReaper, orphanedWorkSentinel, mcpProcessReaper, geminiLoopRunner, sleepController, agentActivityState, reapLog, resumeQueue, resumeDrainer, prHandLease: prHandLease ?? undefined, operatorStopRecorder: recordOperatorStop, sleepWakeDetector, unjustifiedStopGate, stopGateDb, stopNotifier, liveTestGate, liveTestGateMode, liveTestRunnerCtx }); // Resolve the late-bound topic-operator getter (increment 2e): routing was
|
|
16924
|
+
const server = new AgentServer({ config, sessionManager, state, scheduler, telegram, relationships, feedback, feedbackAnomalyDetector, dispatches, updateChecker, autoUpdater, autoDispatcher, quotaTracker, quotaManager, publisher, viewer, tunnel, evolution, watchdog, topicMemory, triageNurse, projectMapper, cartographer: cartographer ?? undefined, coherenceGate: scopeVerifier, contextHierarchy, canonicalState, operationGate, sentinel, adaptiveTrust, memoryMonitor, orphanReaper, coherenceMonitor, commitmentTracker, subscriptionPool, quotaPoller, quotaAwareScheduler: _quotaAwareScheduler ?? undefined, proactiveSwapMonitor: _proactiveSwapMonitor ?? undefined, inUseAccountResolver, enrollmentWizard, credentialRepointing, semanticMemory, activitySentinel, rateLimitSentinel, releaseReadinessSentinel: releaseReadinessSentinel ?? undefined, greenPrAutoMerger: greenPrAutoMerger ?? undefined, guardLatchStore: guardLatchStore ?? undefined, messageRouter, summarySentinel, spawnManager, systemReviewer, capabilityMapper, selfKnowledgeTree, coverageAuditor, topicResumeMap: _topicResumeMap ?? undefined, topicProfile: _topicProfileCtx ?? undefined, sessionRefresh: _sessionRefresh ?? undefined, autonomyManager, trustElevationTracker, autonomousEvolution, coordinator: coordinator.enabled ? coordinator : undefined, localSigningKeyPem, leaseTransport, onLeasePullRequest: () => leaseCoordinatorRef?.currentLease() ?? null, liveTailReceiver, handoffWireTransport, onHandoffBegin, onHandoffInitiate: handoffInitiate, handoffInProgress: handoffSentinelInProgress, messageLedger, currentInboundByTopic, replyMarkerTransport, onReplyMarker: messageLedger ? (marker) => { const m = marker; messageLedger.applyRemoteReplyMarker(m.dedupeKey, { platform: m.platform, replyIdempotencyKey: m.replyIdempotencyKey, epoch: m.epoch, topic: m.topic ?? null }); } : undefined, whatsapp: whatsappAdapter, slack: slackAdapter, imessage: imessageAdapter, whatsappBusinessBackend, messageBridge, hookEventReceiver, worktreeMonitor, subagentTracker, instructionsVerifier, handshakeManager: threadlineHandshake, threadlineRouter, conversationStore, threadLog, threadMessageRecorder, warrantsReplyGate, collaborationSurfacer, threadResumeMap, topicLinkageHandler: topicLinkageHandler ?? undefined, threadlineRelayClient, threadlineReplyWaiters, listenerManager: listenerManager ?? undefined, a2aDeliveryTracker: a2aDeliveryTracker ?? undefined, responseReviewGate, messagingToneGate, outboundDedupGate, telemetryHeartbeat, pasteManager, featureRegistry, discoveryEvaluator, completionEvaluator, unifiedTrust, liveConfig, sharedStateLedger, ledgerSessionRegistry, worktreeManager, oidcEnrolledRepos: parallelDevConfig?.oidcEnrolledRepos, initiativeTracker, projectRoundRunner, projectDriftChecker, machineHeartbeat, machinePoolRegistry, getInboundQueue: () => _inboundQueue, meshRpcDispatcher, workingSetPullCoordinator, commitmentReplicaStore, preferenceReplicaStore, replicatedRecordEmitter, conflictStore, rollbackUnmerge, droppedOriginRegistry, preferencesUnionReader, forwardCommitmentMutate, sessionOwnershipRegistry, topicPinStore: _topicPinStore ?? undefined, streamTicketStore: _streamTicketStore ?? undefined, poolStreamAllowRemoteInput: config.dashboard?.poolStream?.allowRemoteInput ?? false, poolStreamConnector: _poolStreamConnector ?? undefined, secretSync: _secretSyncHandle ?? undefined, meshSelfId: _meshSelfId ?? undefined, resolveRouterUrl: _resolveRouterUrl ?? undefined, resolvePeerUrls: _resolvePeerUrls ?? undefined, guardRegistry, listPoolMachines: _listPoolMachines ?? undefined, poolLink: _poolLink ?? undefined, poolPollCache: _poolPollCache ?? undefined, sessionPoolE2EResultStore, proxyCoordinator, topicIntentStore, topicIntentArcCheck, usherSignalStore, intelligence: sharedIntelligence ?? undefined, telegramBridgeConfig, telegramBridge: telegramBridge ?? undefined, threadlineObservability, briefDeps, workingMemory, taskFlowRegistry, threadlineFlowBridge, sessionReaper, agentWorktreeReaper, orphanedWorkSentinel, mcpProcessReaper, geminiLoopRunner, sleepController, agentActivityState, reapLog, resumeQueue, resumeDrainer, autonomousLivenessReconciler, prHandLease: prHandLease ?? undefined, operatorStopRecorder: recordOperatorStop, sleepWakeDetector, unjustifiedStopGate, stopGateDb, stopNotifier, liveTestGate, liveTestGateMode, liveTestRunnerCtx }); // Resolve the late-bound topic-operator getter (increment 2e): routing was
|
|
16659
16925
|
// wired before the server existed; from here on inbound binds use the
|
|
16660
16926
|
// server's own store instance.
|
|
16661
16927
|
_agentServerRef = server;
|