instar 1.3.906 → 1.3.908
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +340 -14
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +51 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/MessagingToneGate.d.ts.map +1 -1
- package/dist/core/MessagingToneGate.js +21 -0
- package/dist/core/MessagingToneGate.js.map +1 -1
- package/dist/core/MutualSshRuntime.d.ts +15 -0
- package/dist/core/MutualSshRuntime.d.ts.map +1 -1
- package/dist/core/MutualSshRuntime.js +123 -10
- package/dist/core/MutualSshRuntime.js.map +1 -1
- package/dist/core/PeerAuthorizedKeys.d.ts +36 -0
- package/dist/core/PeerAuthorizedKeys.d.ts.map +1 -0
- package/dist/core/PeerAuthorizedKeys.js +208 -0
- package/dist/core/PeerAuthorizedKeys.js.map +1 -0
- package/dist/core/PostUpdateMigrator.d.ts +35 -0
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +111 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/ProactiveSwapMonitor.d.ts +14 -0
- package/dist/core/ProactiveSwapMonitor.d.ts.map +1 -1
- package/dist/core/ProactiveSwapMonitor.js +45 -11
- package/dist/core/ProactiveSwapMonitor.js.map +1 -1
- package/dist/core/QuotaAwareScheduler.d.ts +6 -1
- package/dist/core/QuotaAwareScheduler.d.ts.map +1 -1
- package/dist/core/QuotaAwareScheduler.js +21 -11
- package/dist/core/QuotaAwareScheduler.js.map +1 -1
- package/dist/core/SessionManager.d.ts +19 -0
- package/dist/core/SessionManager.d.ts.map +1 -1
- package/dist/core/SessionManager.js +33 -0
- package/dist/core/SessionManager.js.map +1 -1
- package/dist/core/SessionPoolFailoverRunner.d.ts +94 -0
- package/dist/core/SessionPoolFailoverRunner.d.ts.map +1 -0
- package/dist/core/SessionPoolFailoverRunner.js +89 -0
- package/dist/core/SessionPoolFailoverRunner.js.map +1 -0
- package/dist/core/SessionPoolRolloutDriver.d.ts +74 -0
- package/dist/core/SessionPoolRolloutDriver.d.ts.map +1 -0
- package/dist/core/SessionPoolRolloutDriver.js +78 -0
- package/dist/core/SessionPoolRolloutDriver.js.map +1 -0
- package/dist/core/SessionRefresh.d.ts +3 -0
- package/dist/core/SessionRefresh.d.ts.map +1 -1
- package/dist/core/SessionRefresh.js +19 -0
- package/dist/core/SessionRefresh.js.map +1 -1
- package/dist/core/SshBootstrapAdvert.d.ts +6 -0
- package/dist/core/SshBootstrapAdvert.d.ts.map +1 -1
- package/dist/core/SshBootstrapAdvert.js +21 -1
- package/dist/core/SshBootstrapAdvert.js.map +1 -1
- package/dist/core/StandingSshVerifier.d.ts +12 -0
- package/dist/core/StandingSshVerifier.d.ts.map +1 -0
- package/dist/core/StandingSshVerifier.js +53 -0
- package/dist/core/StandingSshVerifier.js.map +1 -0
- package/dist/core/SubscriptionPool.d.ts +3 -0
- package/dist/core/SubscriptionPool.d.ts.map +1 -1
- package/dist/core/SubscriptionPool.js +7 -0
- package/dist/core/SubscriptionPool.js.map +1 -1
- package/dist/core/SwapAntiThrash.d.ts +5 -0
- package/dist/core/SwapAntiThrash.d.ts.map +1 -1
- package/dist/core/SwapAntiThrash.js +11 -6
- package/dist/core/SwapAntiThrash.js.map +1 -1
- package/dist/core/SwapLedger.d.ts +4 -0
- package/dist/core/SwapLedger.d.ts.map +1 -1
- package/dist/core/SwapLedger.js.map +1 -1
- package/dist/core/deferral-floor.d.ts +50 -0
- package/dist/core/deferral-floor.d.ts.map +1 -0
- package/dist/core/deferral-floor.js +169 -0
- package/dist/core/deferral-floor.js.map +1 -0
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +30 -0
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/machineCoherenceManifest.d.ts.map +1 -1
- package/dist/core/machineCoherenceManifest.js +9 -0
- package/dist/core/machineCoherenceManifest.js.map +1 -1
- package/dist/core/sessionPoolFailoverCheck.d.ts +67 -0
- package/dist/core/sessionPoolFailoverCheck.d.ts.map +1 -0
- package/dist/core/sessionPoolFailoverCheck.js +57 -0
- package/dist/core/sessionPoolFailoverCheck.js.map +1 -0
- package/dist/core/sessionPoolFailoverRunnerConfig.d.ts +152 -0
- package/dist/core/sessionPoolFailoverRunnerConfig.d.ts.map +1 -0
- package/dist/core/sessionPoolFailoverRunnerConfig.js +181 -0
- package/dist/core/sessionPoolFailoverRunnerConfig.js.map +1 -0
- package/dist/core/types.d.ts +72 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/messaging/slack/SlackAdapter.d.ts +3 -0
- package/dist/messaging/slack/SlackAdapter.d.ts.map +1 -1
- package/dist/messaging/slack/SlackAdapter.js +18 -0
- package/dist/messaging/slack/SlackAdapter.js.map +1 -1
- package/dist/monitoring/MissingLoginSessionDetector.d.ts +159 -0
- package/dist/monitoring/MissingLoginSessionDetector.d.ts.map +1 -0
- package/dist/monitoring/MissingLoginSessionDetector.js +166 -0
- package/dist/monitoring/MissingLoginSessionDetector.js.map +1 -0
- package/dist/monitoring/SingleMachineFailoverGapDetector.d.ts +144 -0
- package/dist/monitoring/SingleMachineFailoverGapDetector.d.ts.map +1 -0
- package/dist/monitoring/SingleMachineFailoverGapDetector.js +152 -0
- package/dist/monitoring/SingleMachineFailoverGapDetector.js.map +1 -0
- package/dist/monitoring/guardManifest.d.ts.map +1 -1
- package/dist/monitoring/guardManifest.js +56 -0
- package/dist/monitoring/guardManifest.js.map +1 -1
- package/dist/monitoring/missingLoginSessionWiring.d.ts +96 -0
- package/dist/monitoring/missingLoginSessionWiring.d.ts.map +1 -0
- package/dist/monitoring/missingLoginSessionWiring.js +97 -0
- package/dist/monitoring/missingLoginSessionWiring.js.map +1 -0
- package/dist/monitoring/singleMachineFailoverGapWiring.d.ts +56 -0
- package/dist/monitoring/singleMachineFailoverGapWiring.d.ts.map +1 -0
- package/dist/monitoring/singleMachineFailoverGapWiring.js +48 -0
- package/dist/monitoring/singleMachineFailoverGapWiring.js.map +1 -0
- package/dist/server/AgentServer.d.ts +5 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +3 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/CapabilityIndex.d.ts.map +1 -1
- package/dist/server/CapabilityIndex.js +31 -1
- package/dist/server/CapabilityIndex.js.map +1 -1
- package/dist/server/routes.d.ts +5 -0
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +38 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +65 -65
- package/upgrades/1.3.907.md +30 -0
- package/upgrades/1.3.908.md +33 -0
- package/upgrades/side-effects/login-loss-account-swap.md +87 -0
- package/upgrades/side-effects/proactive-default-account-swap.md +124 -0
- package/upgrades/side-effects/reliable-peer-machine-execution.md +115 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/commands/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA2EH,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAU3D,OAAO,EAAE,eAAe,EAAiC,MAAM,iCAAiC,CAAC;AA0BjG,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AA8H7D,OAAO,KAAK,EAAW,oBAAoB,EAAe,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjG,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAgFtD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAC1C,OAAO,CAUT;AAyID,UAAU,YAAY;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;2DACuD;IACvD,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/commands/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA2EH,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAU3D,OAAO,EAAE,eAAe,EAAiC,MAAM,iCAAiC,CAAC;AA0BjG,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAGvD,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AA8H7D,OAAO,KAAK,EAAW,oBAAoB,EAAe,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjG,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAgFtD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAC1C,OAAO,CAUT;AAyID,UAAU,YAAY;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;2DACuD;IACvD,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AA4qDD;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,oBAAoB,GAAG,IAAI,GACxC;IACD,YAAY,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAC1C,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IAC3C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAqBA;AAED,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,eAAe,EACzB,cAAc,EAAE,cAAc,EAC9B,YAAY,CAAC,EAAE,YAAY,EAC3B,WAAW,CAAC,EAAE,WAAW,EACzB,WAAW,CAAC,EAAE,WAAW,EACzB,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,EAGvE,UAAU,CAAC,EAAE,MAAM,OAAO,8BAA8B,EAAE,WAAW,GAAG,IAAI,EAK5E,qBAAqB,CAAC,EAAE,MAAM,OAAO,gCAAgC,EAAE,kBAAkB,GAAG,IAAI,EAKhG,mBAAmB,CAAC,EAAE,MAAM,MAAM,GAAG,IAAI,GAAG,SAAS,EAQrD,oBAAoB,CAAC,EAAE,MAAM;IAC3B,YAAY,EAAE,OAAO,kBAAkB,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACrE,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IAC3C,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,GACP,IAAI,CAwtBN;AAgmBD,wBAAsB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAgupBtE;AAED,wBAAsB,UAAU,CAAC,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAsDzE;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAuD5E"}
|
package/dist/commands/server.js
CHANGED
|
@@ -510,6 +510,19 @@ let _refreshPoolHeartbeat = () => { };
|
|
|
510
510
|
// §6) so `GET /pool/machine-coherence` can read the sentinel constructed deep in
|
|
511
511
|
// the mesh peer-presence wiring. Null when the guard is dark (route 503s).
|
|
512
512
|
let _machineCoherenceSentinel = null;
|
|
513
|
+
// Module-level handle for the single-machine failover-gap detector (increment 2)
|
|
514
|
+
// so `GET /pool/failover-gap` can read the detector constructed deep in the mesh
|
|
515
|
+
// peer-presence wiring. Null when the guard is dark (route 503s).
|
|
516
|
+
let _singleMachineFailoverGap = null;
|
|
517
|
+
// Module-level handle for the missing-login-session detector (increment 2) so
|
|
518
|
+
// `GET /pool/missing-login` can read the detector constructed deep in the mesh
|
|
519
|
+
// peer-presence wiring. Null when the guard is dark (route 503s).
|
|
520
|
+
let _missingLoginSession = null;
|
|
521
|
+
// Module-level handle for the SessionPoolFailoverRunner driver (§Rollout, Track H)
|
|
522
|
+
// so `GET /session-pool/failover-runner` can read its status(). Null when the
|
|
523
|
+
// runner is dark (dev-gated) — the route 503s. Constructed inside the rollout-gate
|
|
524
|
+
// boot block next to the E2E result store + StageAdvancer.
|
|
525
|
+
let _sessionPoolFailoverRunnerDriver = null;
|
|
513
526
|
// The store the unconditional boot sweep opened when the queue will run this
|
|
514
527
|
// boot — adopted by the engine construction (one open handle, single-writer).
|
|
515
528
|
let _sweptInboundStore = null;
|
|
@@ -12012,7 +12025,7 @@ export async function startServer(options) {
|
|
|
12012
12025
|
// Subscription & Auth Standard). Always instantiated; ships DARK because an
|
|
12013
12026
|
// empty pool is a pure no-op (no background loop, no cost) until an operator
|
|
12014
12027
|
// enrolls an account. Stores each account's login LOCATION, never tokens.
|
|
12015
|
-
const { SubscriptionPool } = await import('../core/SubscriptionPool.js');
|
|
12028
|
+
const { SubscriptionPool, requiresOwnerRelogin } = await import('../core/SubscriptionPool.js');
|
|
12016
12029
|
const subscriptionPool = new SubscriptionPool({ stateDir: config.stateDir });
|
|
12017
12030
|
if (subscriptionPool.size() > 0) {
|
|
12018
12031
|
console.log(pc.green(` Subscription pool: ${subscriptionPool.size()} account(s) registered`));
|
|
@@ -16794,8 +16807,8 @@ export async function startServer(options) {
|
|
|
16794
16807
|
}
|
|
16795
16808
|
// ── QuotaAwareScheduler (Subscription & Auth Standard P1.3) ──
|
|
16796
16809
|
// Telegram-specific (createAttentionItem + subscription-pool wiring).
|
|
16797
|
-
if (telegram) {
|
|
16798
|
-
const telegramRef = telegram; //
|
|
16810
|
+
if (telegram || _slackAdapter) {
|
|
16811
|
+
const telegramRef = telegram; // optional notification surface only
|
|
16799
16812
|
// Selects the optimal account + enforces the continuity guarantee: on
|
|
16800
16813
|
// quota pressure it resumes the session under another account (via the
|
|
16801
16814
|
// SessionRefresh account-swap path), never letting it die. Auto-trigger
|
|
@@ -16816,7 +16829,19 @@ export async function startServer(options) {
|
|
|
16816
16829
|
const k = antiThrashKnobsGetter();
|
|
16817
16830
|
return { thresholdPct: k.thresholdPct, targetHeadroomPct: k.targetHeadroomPct, minImprovementPct: k.minImprovementPct };
|
|
16818
16831
|
},
|
|
16819
|
-
|
|
16832
|
+
resolveEffectiveAccountId: async (sessionName, sourceWasUntagged, sourceTrigger) => {
|
|
16833
|
+
const tagged = state.listSessions({ status: 'running' })
|
|
16834
|
+
.find((s) => s.tmuxSession === sessionName)?.subscriptionAccountId ?? null;
|
|
16835
|
+
if (!sourceWasUntagged || tagged)
|
|
16836
|
+
return tagged;
|
|
16837
|
+
if (sourceTrigger === 'login-loss') {
|
|
16838
|
+
const realHome = sessionManager.configHomeForSession(sessionName);
|
|
16839
|
+
if (!realHome)
|
|
16840
|
+
return null;
|
|
16841
|
+
return subscriptionPool.list().find((a) => a.configHome === realHome)?.id ?? null;
|
|
16842
|
+
}
|
|
16843
|
+
return (await inUseAccountResolver.resolve(subscriptionPool.list())).activeAccountId;
|
|
16844
|
+
},
|
|
16820
16845
|
onReactiveExecuted: (a) => _swapAntiThrashEngine?.recordReactiveExecuted(a),
|
|
16821
16846
|
onReactiveFailed: (a) => _swapAntiThrashEngine?.recordExecFailure({
|
|
16822
16847
|
session: a.session,
|
|
@@ -16845,7 +16870,7 @@ export async function startServer(options) {
|
|
|
16845
16870
|
// anti-thrash observation layer can classify failures (§3.1/§3.6).
|
|
16846
16871
|
return r.ok ? { ok: true } : { ok: false, code: r.code };
|
|
16847
16872
|
},
|
|
16848
|
-
onNoAlternate: (sessionName, exhaustedAccountId) => {
|
|
16873
|
+
onNoAlternate: telegramRef ? (sessionName, exhaustedAccountId) => {
|
|
16849
16874
|
void telegramRef.createAttentionItem({
|
|
16850
16875
|
id: `subpool-no-alternate-${exhaustedAccountId}`,
|
|
16851
16876
|
title: 'Subscription pool — no alternate account to swap to',
|
|
@@ -16854,7 +16879,7 @@ export async function startServer(options) {
|
|
|
16854
16879
|
priority: 'HIGH',
|
|
16855
16880
|
sourceContext: 'subscription-pool:no-alternate',
|
|
16856
16881
|
}).catch(() => { });
|
|
16857
|
-
},
|
|
16882
|
+
} : undefined,
|
|
16858
16883
|
logger: { log: (m) => console.log(m), warn: (m) => console.warn(m) },
|
|
16859
16884
|
});
|
|
16860
16885
|
// DARK auto-trigger: only when explicitly enabled does a rate-limit
|
|
@@ -16912,11 +16937,19 @@ export async function startServer(options) {
|
|
|
16912
16937
|
// Only claude-code sessions ride the claude-account pool (legacy
|
|
16913
16938
|
// records with no framework default to claude-code).
|
|
16914
16939
|
.filter((s) => s.framework === undefined || s.framework === 'claude-code')
|
|
16915
|
-
.map((s) =>
|
|
16916
|
-
|
|
16917
|
-
|
|
16918
|
-
|
|
16919
|
-
|
|
16940
|
+
.map((s) => {
|
|
16941
|
+
const realHome = sessionManager.configHomeForSession(s.tmuxSession);
|
|
16942
|
+
const loginLossAccountId = realHome
|
|
16943
|
+
? subscriptionPool.list().find((a) => a.configHome === realHome && requiresOwnerRelogin(a))?.id ?? null
|
|
16944
|
+
: null;
|
|
16945
|
+
return {
|
|
16946
|
+
sessionName: s.tmuxSession,
|
|
16947
|
+
accountId: s.subscriptionAccountId ?? null,
|
|
16948
|
+
loginLossAccountId,
|
|
16949
|
+
startedAt: s.startedAt,
|
|
16950
|
+
refreshable: _sessionRefresh?.canRefreshSession(s.tmuxSession) ?? false,
|
|
16951
|
+
};
|
|
16952
|
+
}),
|
|
16920
16953
|
resolveDefaultAccountId: async () => (await inUseAccountResolver.resolve(subscriptionPool.list())).activeAccountId,
|
|
16921
16954
|
swap: (a) => _quotaAwareScheduler
|
|
16922
16955
|
? _quotaAwareScheduler.onQuotaPressure(a)
|
|
@@ -16934,6 +16967,10 @@ export async function startServer(options) {
|
|
|
16934
16967
|
antiThrash: _swapAntiThrashEngine
|
|
16935
16968
|
? { engine: _swapAntiThrashEngine, getKnobs: antiThrashKnobsGetter }
|
|
16936
16969
|
: undefined,
|
|
16970
|
+
loginLoss: {
|
|
16971
|
+
enabled: resolveDevAgentGate(proactiveCfg.loginLoss?.enabled, config),
|
|
16972
|
+
dryRun: proactiveCfg.loginLoss?.dryRun !== false,
|
|
16973
|
+
},
|
|
16937
16974
|
// §4 — the in-flight work gate (proactive arm: defer, ceiling-drop).
|
|
16938
16975
|
// The monitor owns the deferral lifecycle; the gate is stateless.
|
|
16939
16976
|
workGate: _swapWorkGate
|
|
@@ -20377,6 +20414,18 @@ export async function startServer(options) {
|
|
|
20377
20414
|
// Mutual SSH phase 1: the signed HTTP mesh is the bootstrap carrier;
|
|
20378
20415
|
// the resulting proof itself traverses the restricted SSH subsystem.
|
|
20379
20416
|
// Hard-dark on fleet, dry-run first on the development agent.
|
|
20417
|
+
const peerExecutionEnabled = resolveDevAgentGate(config.multiMachine?.peerExecution?.enabled, config);
|
|
20418
|
+
if (!peerExecutionEnabled) {
|
|
20419
|
+
try {
|
|
20420
|
+
const { PeerAuthorizedKeys } = await import('../core/PeerAuthorizedKeys.js');
|
|
20421
|
+
new PeerAuthorizedKeys(os.homedir(), false).revokeUnknown(config.projectName, new Set());
|
|
20422
|
+
}
|
|
20423
|
+
catch (error) {
|
|
20424
|
+
// @silent-fallback-ok — disabled-feature cleanup failure is loud;
|
|
20425
|
+
// no runtime starts and no readiness success can mask stale access.
|
|
20426
|
+
console.warn(`[peer-execution] disabled-grant cleanup blocked: ${error instanceof Error ? error.message : String(error)}`);
|
|
20427
|
+
}
|
|
20428
|
+
}
|
|
20380
20429
|
if (resolveDevAgentGate(config.multiMachine?.mutualSsh?.enabled, config)) {
|
|
20381
20430
|
try {
|
|
20382
20431
|
const sshMod = await import('../core/MutualSshRuntime.js');
|
|
@@ -20399,6 +20448,18 @@ export async function startServer(options) {
|
|
|
20399
20448
|
?? privateHosts.find(host => /^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(host))
|
|
20400
20449
|
?? '127.0.0.1';
|
|
20401
20450
|
const bindPort = portMod.allocatePort(`${config.projectName}-mutual-ssh`, 42000, 49000);
|
|
20451
|
+
const standingHostKey = (() => {
|
|
20452
|
+
if (!peerExecutionEnabled)
|
|
20453
|
+
return null;
|
|
20454
|
+
try {
|
|
20455
|
+
return fs.readFileSync('/etc/ssh/ssh_host_ed25519_key.pub', 'utf8').trim();
|
|
20456
|
+
}
|
|
20457
|
+
catch {
|
|
20458
|
+
// @silent-fallback-ok — absence is a named standing endpoint
|
|
20459
|
+
// readiness blocker; never substitute an unpinned host key.
|
|
20460
|
+
return null;
|
|
20461
|
+
}
|
|
20462
|
+
})();
|
|
20402
20463
|
const peers = () => meshIdMgr.getActiveMachines()
|
|
20403
20464
|
.filter(row => row.machineId !== meshSelfId)
|
|
20404
20465
|
.map(row => ({
|
|
@@ -20414,6 +20475,19 @@ export async function startServer(options) {
|
|
|
20414
20475
|
selfMachineFingerprint: machineFingerprint(selfPem),
|
|
20415
20476
|
observerBootId: `${meshSelfId}:${process.pid}:${Date.now()}`,
|
|
20416
20477
|
bindHost, bindPort, dryRun: config.multiMachine?.mutualSsh?.dryRun !== false,
|
|
20478
|
+
peerExecution: peerExecutionEnabled ? {
|
|
20479
|
+
// The SSH daemon resolves AuthorizedKeysFile beneath the
|
|
20480
|
+
// account home; never write a system/global sshd path.
|
|
20481
|
+
agentHome: os.homedir(),
|
|
20482
|
+
dryRun: config.multiMachine?.peerExecution?.dryRun !== false,
|
|
20483
|
+
requiredForReadiness: config.multiMachine?.peerExecution?.requiredForReadiness !== false,
|
|
20484
|
+
} : undefined,
|
|
20485
|
+
localStandingSsh: peerExecutionEnabled && standingHostKey && bindHost !== '127.0.0.1' ? {
|
|
20486
|
+
host: bindHost,
|
|
20487
|
+
port: config.multiMachine?.peerExecution?.port ?? 22,
|
|
20488
|
+
username: os.userInfo().username,
|
|
20489
|
+
hostPublicKey: standingHostKey,
|
|
20490
|
+
} : undefined,
|
|
20417
20491
|
requiredForReadiness: config.multiMachine?.mutualSsh?.requiredForEmployeeRole === true,
|
|
20418
20492
|
freshnessMs: config.multiMachine?.mutualSsh?.freshnessMs,
|
|
20419
20493
|
cadenceMs: config.multiMachine?.mutualSsh?.cadenceMs,
|
|
@@ -20452,6 +20526,11 @@ export async function startServer(options) {
|
|
|
20452
20526
|
enabled: true, dryRun: config.multiMachine?.mutualSsh?.dryRun !== false,
|
|
20453
20527
|
status: mutualSshRuntime?.status() ?? null,
|
|
20454
20528
|
}));
|
|
20529
|
+
guardRegistry.register('multiMachine.peerExecution.enabled', () => ({
|
|
20530
|
+
enabled: resolveDevAgentGate(config.multiMachine?.peerExecution?.enabled, config),
|
|
20531
|
+
dryRun: config.multiMachine?.peerExecution?.dryRun !== false,
|
|
20532
|
+
status: mutualSshRuntime?.status() ?? null,
|
|
20533
|
+
}));
|
|
20455
20534
|
}
|
|
20456
20535
|
catch (err) {
|
|
20457
20536
|
// @silent-fallback-ok — the feature fails closed and surfaces its
|
|
@@ -21985,6 +22064,111 @@ export async function startServer(options) {
|
|
|
21985
22064
|
// — the monitoring sentinel must never crash server boot.
|
|
21986
22065
|
console.error(' MachineCoherenceGuard wiring failed (non-fatal):', e instanceof Error ? e.message : String(e));
|
|
21987
22066
|
}
|
|
22067
|
+
// ── Single-Machine Failover-Gap Detector (increment 2 — pure SIGNAL) ──
|
|
22068
|
+
// Surfaces the "no failover target for active autonomous work" gap BEFORE
|
|
22069
|
+
// it bites (the 2026-07-22 Codey overnight loss): single-machine (no online
|
|
22070
|
+
// mesh peer) WHILE active autonomous runs → ONE deduped HIGH attention item.
|
|
22071
|
+
// Rides THIS 30s peerPresenceTick; SIGNAL-ONLY (never blocks/provisions/kills).
|
|
22072
|
+
// Dev-gated dark on the fleet (`enabled` OMITTED → resolveDevAgentGate), dry-
|
|
22073
|
+
// run FIRST even on dev (computes + counts would-raise, raises nothing).
|
|
22074
|
+
// Constructed ONLY when the gate resolves enabled — a dark guard is never
|
|
22075
|
+
// constructed (route 503s, no tick); single-machine-with-no-work is a strict
|
|
22076
|
+
// no-op inside tick(). Own try/catch: a wiring failure is logged non-fatal.
|
|
22077
|
+
try {
|
|
22078
|
+
const smfgMod = await import('../monitoring/singleMachineFailoverGapWiring.js');
|
|
22079
|
+
const smfgDetMod = await import('../monitoring/SingleMachineFailoverGapDetector.js');
|
|
22080
|
+
const { getByPath: smfgGetByPath } = await import('../core/machineCoherenceManifest.js');
|
|
22081
|
+
const { activeAutonomousJobs: smfgActiveJobs } = await import('../core/AutonomousSessions.js');
|
|
22082
|
+
const smfgCfg = smfgDetMod.resolveSingleMachineFailoverGapConfig(smfgGetByPath(config, 'monitoring.singleMachineFailoverGap'), (explicit) => resolveDevAgentGate(explicit, config));
|
|
22083
|
+
if (smfgCfg.enabled) {
|
|
22084
|
+
const smfgDetector = smfgMod.makeSingleMachineFailoverGapDetector({
|
|
22085
|
+
enabled: () => smfgCfg.enabled,
|
|
22086
|
+
dryRun: () => smfgCfg.dryRun,
|
|
22087
|
+
getCapacities: () => machinePoolRegistry?.getCapacities() ?? [],
|
|
22088
|
+
selfMachineId: () => meshSelfId ?? null,
|
|
22089
|
+
// "configured" = a multiMachine block exists at all (peer-offline vs
|
|
22090
|
+
// not-configured only changes the message text, never the gap verdict).
|
|
22091
|
+
multiMachineEnabled: () => !!config.multiMachine,
|
|
22092
|
+
getActiveAutonomousRunCount: () => config.stateDir ? smfgActiveJobs(config.stateDir).length : 0,
|
|
22093
|
+
createAttentionItem: (item) => {
|
|
22094
|
+
// fire-and-forget into the real attention queue; a Telegram fault
|
|
22095
|
+
// must never crash the shared presence tick (fail toward silence).
|
|
22096
|
+
void telegram?.createAttentionItem(item)?.catch?.(() => { });
|
|
22097
|
+
},
|
|
22098
|
+
});
|
|
22099
|
+
// expectRuntime:true on the GUARD_MANIFEST entry — register the
|
|
22100
|
+
// synchronous runtime getter so /guards grades the guard's posture
|
|
22101
|
+
// (enabled/dryRun + real lastTickAt liveness from status()).
|
|
22102
|
+
guardRegistry.register('monitoring.singleMachineFailoverGap.enabled', () => {
|
|
22103
|
+
const s = smfgDetector.status();
|
|
22104
|
+
return { enabled: s.enabled, dryRun: s.dryRun, lastTickAt: s.lastTickAt ? Date.parse(s.lastTickAt) : 0 };
|
|
22105
|
+
});
|
|
22106
|
+
_singleMachineFailoverGap = smfgDetector; // expose to GET /pool/failover-gap
|
|
22107
|
+
console.log(pc.dim(` SingleMachineFailoverGap: enabled dryRun=${smfgCfg.dryRun} (no-failover-target detector — pure signal)`));
|
|
22108
|
+
}
|
|
22109
|
+
}
|
|
22110
|
+
catch (e) {
|
|
22111
|
+
// @silent-fallback-ok — optional dev-gated, pure-signal feature: a wiring
|
|
22112
|
+
// failure is logged (console.error, NOT silent) and is non-fatal BY DESIGN.
|
|
22113
|
+
console.error(' SingleMachineFailoverGap wiring failed (non-fatal):', e instanceof Error ? e.message : String(e));
|
|
22114
|
+
}
|
|
22115
|
+
// ── Missing-Login-Session Detector (increment 2 — pure SIGNAL) ──
|
|
22116
|
+
// Surfaces the "a live session is running on an account whose local login
|
|
22117
|
+
// has gone missing" gap BEFORE the session walls silently (the 2026-07-22
|
|
22118
|
+
// justin-gmail silent auth-death): a running session bound to an account
|
|
22119
|
+
// whose subscription-pool identity drift is `owner-relogin-required` /
|
|
22120
|
+
// `missing-local-login` → ONE deduped HIGH attention item naming the account.
|
|
22121
|
+
// Rides THIS 30s peerPresenceTick; SIGNAL-ONLY (never swaps/relogins/kills).
|
|
22122
|
+
// Dev-gated dark on the fleet (`enabled` OMITTED → resolveDevAgentGate), dry-
|
|
22123
|
+
// run FIRST even on dev (computes + counts would-raise, raises nothing).
|
|
22124
|
+
// Constructed ONLY when the gate resolves enabled — a dark guard is never
|
|
22125
|
+
// constructed (route 503s, no tick); no-drift-under-a-session is a strict
|
|
22126
|
+
// no-op inside tick(). Own try/catch: a wiring failure is logged non-fatal.
|
|
22127
|
+
try {
|
|
22128
|
+
const mlsMod = await import('../monitoring/missingLoginSessionWiring.js');
|
|
22129
|
+
const mlsDetMod = await import('../monitoring/MissingLoginSessionDetector.js');
|
|
22130
|
+
const { getByPath: mlsGetByPath } = await import('../core/machineCoherenceManifest.js');
|
|
22131
|
+
const mlsCfg = mlsDetMod.resolveMissingLoginSessionConfig(mlsGetByPath(config, 'monitoring.missingLoginSession'), (explicit) => resolveDevAgentGate(explicit, config));
|
|
22132
|
+
if (mlsCfg.enabled) {
|
|
22133
|
+
const mlsDetector = mlsMod.makeMissingLoginSessionDetector({
|
|
22134
|
+
enabled: () => mlsCfg.enabled,
|
|
22135
|
+
dryRun: () => mlsCfg.dryRun,
|
|
22136
|
+
// Raw subscription-pool accounts (identity-drift flags → missing-login);
|
|
22137
|
+
// a pool of zero accounts (single-account agent) is a strict no-op.
|
|
22138
|
+
getPoolAccounts: () => subscriptionPool.list(),
|
|
22139
|
+
// Running sessions + each one's REAL login slot (its live
|
|
22140
|
+
// CLAUDE_CONFIG_DIR, resolved from tmux). This is the config home
|
|
22141
|
+
// the session is ACTUALLY on — NOT its recorded subscriptionAccountId,
|
|
22142
|
+
// which diverges from the real slot under identity drift (the gap this
|
|
22143
|
+
// guard exists for). A session whose config home is unresolvable
|
|
22144
|
+
// contributes configHome:undefined and is skipped by the factory (never
|
|
22145
|
+
// a guessed slot). configHomeForSession fails toward silence internally.
|
|
22146
|
+
getRunningSessions: () => sessionManager.listRunningSessions().map((s) => ({
|
|
22147
|
+
sessionName: s.tmuxSession,
|
|
22148
|
+
configHome: sessionManager.configHomeForSession(s.tmuxSession),
|
|
22149
|
+
})),
|
|
22150
|
+
createAttentionItem: (item) => {
|
|
22151
|
+
// fire-and-forget into the real attention queue; a Telegram fault
|
|
22152
|
+
// must never crash the shared presence tick (fail toward silence).
|
|
22153
|
+
void telegram?.createAttentionItem(item)?.catch?.(() => { });
|
|
22154
|
+
},
|
|
22155
|
+
});
|
|
22156
|
+
// expectRuntime:true on the GUARD_MANIFEST entry — register the
|
|
22157
|
+
// synchronous runtime getter so /guards grades the guard's posture
|
|
22158
|
+
// (enabled/dryRun + real lastTickAt liveness from status()).
|
|
22159
|
+
guardRegistry.register('monitoring.missingLoginSession.enabled', () => {
|
|
22160
|
+
const s = mlsDetector.status();
|
|
22161
|
+
return { enabled: s.enabled, dryRun: s.dryRun, lastTickAt: s.lastTickAt ? Date.parse(s.lastTickAt) : 0 };
|
|
22162
|
+
});
|
|
22163
|
+
_missingLoginSession = mlsDetector; // expose to GET /pool/missing-login
|
|
22164
|
+
console.log(pc.dim(` MissingLoginSession: enabled dryRun=${mlsCfg.dryRun} (missing-login-under-a-session detector — pure signal)`));
|
|
22165
|
+
}
|
|
22166
|
+
}
|
|
22167
|
+
catch (e) {
|
|
22168
|
+
// @silent-fallback-ok — optional dev-gated, pure-signal feature: a wiring
|
|
22169
|
+
// failure is logged (console.error, NOT silent) and is non-fatal BY DESIGN.
|
|
22170
|
+
console.error(' MissingLoginSession wiring failed (non-fatal):', e instanceof Error ? e.message : String(e));
|
|
22171
|
+
}
|
|
21988
22172
|
// Drain + execute the machine-coherence episode effects (§4) the last
|
|
21989
22173
|
// reconcile produced (raise/append/resolve). Async + best-effort — a
|
|
21990
22174
|
// Telegram fault never crashes the shared presence tick (fail toward
|
|
@@ -22073,6 +22257,20 @@ export async function startServer(options) {
|
|
|
22073
22257
|
catch { /* @silent-fallback-ok: §3.3 the evaluator tick fails toward silence by design — a sentinel fault must never crash the shared 30s presence tick; the next reconcile re-derives from durable episode state */ }
|
|
22074
22258
|
executeMachineCoherenceEffects(machineCoherenceSentinel, telegram);
|
|
22075
22259
|
}
|
|
22260
|
+
// Single-machine failover-gap detector rider (increment 2): one pure-
|
|
22261
|
+
// signal evaluation per tick. Fails toward silence — a detector fault
|
|
22262
|
+
// must never crash the shared 30s presence tick.
|
|
22263
|
+
try {
|
|
22264
|
+
_singleMachineFailoverGap?.tick();
|
|
22265
|
+
}
|
|
22266
|
+
catch { /* @silent-fallback-ok: fails toward silence */ }
|
|
22267
|
+
// Missing-login-session detector rider (increment 2): one pure-signal
|
|
22268
|
+
// evaluation per tick. Fails toward silence — a detector fault must
|
|
22269
|
+
// never crash the shared 30s presence tick.
|
|
22270
|
+
try {
|
|
22271
|
+
_missingLoginSession?.tick();
|
|
22272
|
+
}
|
|
22273
|
+
catch { /* @silent-fallback-ok: fails toward silence */ }
|
|
22076
22274
|
};
|
|
22077
22275
|
const peerPresenceTimer = setInterval(peerPresenceTick, 30_000);
|
|
22078
22276
|
if (typeof peerPresenceTimer.unref === 'function')
|
|
@@ -22400,7 +22598,7 @@ export async function startServer(options) {
|
|
|
22400
22598
|
return s.length === exp.length && crypto.timingSafeEqual(Buffer.from(s), Buffer.from(exp));
|
|
22401
22599
|
}
|
|
22402
22600
|
catch {
|
|
22403
|
-
return false;
|
|
22601
|
+
return false; /* @silent-fallback-ok: HMAC verify fails closed — a malformed/short input is treated as an invalid signature, never accepted */
|
|
22404
22602
|
}
|
|
22405
22603
|
},
|
|
22406
22604
|
});
|
|
@@ -22428,15 +22626,143 @@ export async function startServer(options) {
|
|
|
22428
22626
|
writeStageConfig: (s) => liveConfig.set(guardMod.STAGE_CONFIG_PATH, s, { stageWriteToken: guardMod.STAGE_WRITE_TOKEN }),
|
|
22429
22627
|
audit: (event, detail) => console.log(pc.dim(` [stage-advancer] ${event} ${JSON.stringify(detail)}`)),
|
|
22430
22628
|
});
|
|
22629
|
+
// ── SessionPoolFailoverRunner (§Rollout, Track H) — the in-agent PRODUCER of
|
|
22630
|
+
// a real failover-E2E green so a DEPLOYED dev agent can promote its own
|
|
22631
|
+
// sessionPool stage instead of sitting at 'shadow' forever. Dev-gated dark
|
|
22632
|
+
// (enabled OMITTED → resolveDevAgentGate) + dryRun-first: while dryRun holds
|
|
22633
|
+
// the recorded verdict lands in a SIDE store the StageAdvancer never reads,
|
|
22634
|
+
// so nothing promotes until a deliberate dryRun:false. Constructed here so it
|
|
22635
|
+
// shares the E2E result store + boot commit SHA + the slow reconcile tick.
|
|
22636
|
+
const fromMod = await import('../core/sessionPoolFailoverRunnerConfig.js');
|
|
22637
|
+
const failoverRunnerCfg = fromMod.resolveSessionPoolFailoverRunnerConfig(config.multiMachine?.sessionPool?.failoverRunner, (explicit) => resolveDevAgentGate(explicit, config));
|
|
22638
|
+
if (failoverRunnerCfg.enabled) {
|
|
22639
|
+
// The dry-run SIDE store: a separate signed file the promotion path
|
|
22640
|
+
// (StageAdvancer/driver) NEVER reads, so a would-record green captured
|
|
22641
|
+
// during the soak can never wrongly promote the stage.
|
|
22642
|
+
const failoverDryRunStore = new e2eMod.SessionPoolE2EResultStore({
|
|
22643
|
+
filePath: path.join(config.stateDir, 'session-pool-failover-runner-dryrun.json'),
|
|
22644
|
+
sign: hmac,
|
|
22645
|
+
verifySig: (c, s) => {
|
|
22646
|
+
try {
|
|
22647
|
+
const exp = hmac(c);
|
|
22648
|
+
return s.length === exp.length && crypto.timingSafeEqual(Buffer.from(s), Buffer.from(exp));
|
|
22649
|
+
}
|
|
22650
|
+
catch {
|
|
22651
|
+
return false; /* @silent-fallback-ok: HMAC verify fails closed — a malformed/short input is treated as an invalid signature, never accepted */
|
|
22652
|
+
}
|
|
22653
|
+
},
|
|
22654
|
+
});
|
|
22655
|
+
// The REAL bounded subprocess runner. Resolves the instar SOURCE root (the
|
|
22656
|
+
// agent's own checkout) — a deployed agent with no source/vitest yields
|
|
22657
|
+
// ranToCompletion:false → the check throws → the runner records NOTHING
|
|
22658
|
+
// (honest degrade, never a fabricated verdict). The spawn is fully bounded
|
|
22659
|
+
// (kill on timeout) and its output is streamed to a durable evidence log.
|
|
22660
|
+
const cp = await import('node:child_process');
|
|
22661
|
+
const failoverLogPath = path.join(config.stateDir, 'logs', 'session-pool-failover-runner.log');
|
|
22662
|
+
const runProcess = (args) => new Promise((resolve) => {
|
|
22663
|
+
// Candidate source roots, most-specific first: an explicit env override,
|
|
22664
|
+
// then this build's package root (dist/../..), then cwd. The first that
|
|
22665
|
+
// carries BOTH node_modules/.bin/vitest AND the test file wins.
|
|
22666
|
+
const candidates = [
|
|
22667
|
+
process.env.INSTAR_SOURCE_ROOT,
|
|
22668
|
+
path.resolve(__dirname, '..', '..'),
|
|
22669
|
+
process.cwd(),
|
|
22670
|
+
].filter((c) => typeof c === 'string' && c.length > 0);
|
|
22671
|
+
let sourceRoot = null;
|
|
22672
|
+
for (const c of candidates) {
|
|
22673
|
+
try {
|
|
22674
|
+
if (fs.existsSync(path.join(c, 'node_modules', '.bin', 'vitest')) && fs.existsSync(path.join(c, args.testPath))) {
|
|
22675
|
+
sourceRoot = c;
|
|
22676
|
+
break;
|
|
22677
|
+
}
|
|
22678
|
+
}
|
|
22679
|
+
catch { /* @silent-fallback-ok — probe next candidate */ }
|
|
22680
|
+
}
|
|
22681
|
+
if (!sourceRoot) {
|
|
22682
|
+
// No source/vitest resolvable → NOT a failover verdict; the check
|
|
22683
|
+
// contract makes this THROW so the runner records nothing.
|
|
22684
|
+
resolve({ ranToCompletion: false, exitCode: null, evidenceRef: `no-source: vitest+${args.testPath} not resolvable (checkout absent)` });
|
|
22685
|
+
return;
|
|
22686
|
+
}
|
|
22687
|
+
const vitestBin = path.join(sourceRoot, 'node_modules', '.bin', 'vitest');
|
|
22688
|
+
let logFd = null;
|
|
22689
|
+
try {
|
|
22690
|
+
fs.mkdirSync(path.dirname(failoverLogPath), { recursive: true });
|
|
22691
|
+
logFd = fs.openSync(failoverLogPath, 'a');
|
|
22692
|
+
}
|
|
22693
|
+
catch {
|
|
22694
|
+
logFd = null; /* @silent-fallback-ok — evidence log optional */
|
|
22695
|
+
}
|
|
22696
|
+
const evidenceRef = `${failoverLogPath}@${new Date().toISOString()}`;
|
|
22697
|
+
let settled = false;
|
|
22698
|
+
const done = (r) => {
|
|
22699
|
+
if (settled)
|
|
22700
|
+
return;
|
|
22701
|
+
settled = true;
|
|
22702
|
+
if (logFd !== null) {
|
|
22703
|
+
try {
|
|
22704
|
+
fs.closeSync(logFd);
|
|
22705
|
+
}
|
|
22706
|
+
catch { /* @silent-fallback-ok */ }
|
|
22707
|
+
}
|
|
22708
|
+
resolve(r);
|
|
22709
|
+
};
|
|
22710
|
+
try {
|
|
22711
|
+
const child = cp.spawn(vitestBin, ['run', args.testPath, '--reporter=dot'], {
|
|
22712
|
+
cwd: sourceRoot,
|
|
22713
|
+
stdio: ['ignore', logFd ?? 'ignore', logFd ?? 'ignore'],
|
|
22714
|
+
env: { ...process.env },
|
|
22715
|
+
});
|
|
22716
|
+
const killTimer = setTimeout(() => {
|
|
22717
|
+
try {
|
|
22718
|
+
child.kill('SIGKILL');
|
|
22719
|
+
}
|
|
22720
|
+
catch { /* @silent-fallback-ok */ }
|
|
22721
|
+
done({ ranToCompletion: false, exitCode: null, evidenceRef: `${evidenceRef} (timeout ${args.timeoutMs}ms)` });
|
|
22722
|
+
}, args.timeoutMs);
|
|
22723
|
+
if (killTimer.unref)
|
|
22724
|
+
killTimer.unref();
|
|
22725
|
+
child.on('error', () => { clearTimeout(killTimer); done({ ranToCompletion: false, exitCode: null, evidenceRef: `${evidenceRef} (spawn-error)` }); });
|
|
22726
|
+
child.on('close', (code) => { clearTimeout(killTimer); done({ ranToCompletion: true, exitCode: code, evidenceRef }); });
|
|
22727
|
+
}
|
|
22728
|
+
catch {
|
|
22729
|
+
// @silent-fallback-ok — a spawn throw is an infra failure, NOT a
|
|
22730
|
+
// failover verdict; ranToCompletion:false → the check throws → nothing recorded.
|
|
22731
|
+
done({ ranToCompletion: false, exitCode: null, evidenceRef: `${evidenceRef} (spawn-threw)` });
|
|
22732
|
+
}
|
|
22733
|
+
});
|
|
22734
|
+
_sessionPoolFailoverRunnerDriver = fromMod.buildSessionPoolFailoverRunnerDriver({
|
|
22735
|
+
config: failoverRunnerCfg,
|
|
22736
|
+
resultStore: sessionPoolE2EResultStore,
|
|
22737
|
+
dryRunResultStore: failoverDryRunStore,
|
|
22738
|
+
runProcess,
|
|
22739
|
+
currentCommitSha: () => bootCommitSha || 'unknown',
|
|
22740
|
+
// The shadow→live-transfer failover is proven at the 'shadow' stage index
|
|
22741
|
+
// (0), which StageAdvancer.advanceTo('live-transfer') reads.
|
|
22742
|
+
provenStage: () => 0,
|
|
22743
|
+
audit: (event, detail) => console.log(pc.dim(` [failover-runner] ${event} ${JSON.stringify(detail)}`)),
|
|
22744
|
+
});
|
|
22745
|
+
if (_sessionPoolFailoverRunnerDriver) {
|
|
22746
|
+
console.log(pc.green(` SessionPoolFailoverRunner enabled (${failoverRunnerCfg.dryRun ? 'dry-run — records to SIDE store' : 'LIVE — records to promotion store'}; cadence ${failoverRunnerCfg.tickIntervalMs}ms)`));
|
|
22747
|
+
}
|
|
22748
|
+
}
|
|
22431
22749
|
// Revert-ONLY reconcile tick (§Rollout): reconcile() can solely DEMOTE a live
|
|
22432
22750
|
// stage when the CURRENT commit's Tier-3 E2E goes red — it never advances.
|
|
22433
22751
|
// Promotion (advanceTo) stays operator-triggered via the rollout route/job.
|
|
22434
|
-
// Cheap + inert while stage is 'dark' (reconcile() no-ops at the floor).
|
|
22752
|
+
// Cheap + inert while stage is 'dark' (reconcile() no-ops at the floor). The
|
|
22753
|
+
// failover runner's maybeTick() rides the SAME slow tick — it self-throttles
|
|
22754
|
+
// to its own (far slower) tickIntervalMs, so it is NOT a hot loop here.
|
|
22435
22755
|
const stageReconcileTimer = setInterval(() => {
|
|
22436
22756
|
try {
|
|
22437
22757
|
stageAdvancer.reconcile();
|
|
22438
22758
|
}
|
|
22439
22759
|
catch { /* @silent-fallback-ok — retried next tick */ }
|
|
22760
|
+
// Fire-and-forget: the driver owns the in-flight guard + cadence gate; a
|
|
22761
|
+
// heavy E2E subprocess must never block the timer. Fail toward silence.
|
|
22762
|
+
try {
|
|
22763
|
+
void _sessionPoolFailoverRunnerDriver?.maybeTick();
|
|
22764
|
+
}
|
|
22765
|
+
catch { /* @silent-fallback-ok — retried next tick */ }
|
|
22440
22766
|
}, 60_000);
|
|
22441
22767
|
if (stageReconcileTimer.unref)
|
|
22442
22768
|
stageReconcileTimer.unref();
|
|
@@ -22799,7 +23125,7 @@ export async function startServer(options) {
|
|
|
22799
23125
|
const server = new AgentServer({ config, sessionManager, llmQueue: sharedLlmQueue, state, scheduler, telegram, relationships, feedback, feedbackAnomalyDetector, dispatches, updateChecker, autoUpdater, autoDispatcher, quotaTracker, quotaManager, publisher, viewer, tunnel, evolution, watchdog, topicMemory, triageNurse, projectMapper, cartographer: cartographer ?? undefined, coherenceGate: scopeVerifier, contextHierarchy, canonicalState, operationGate, sentinel, adaptiveTrust, memoryMonitor, orphanReaper, coherenceMonitor, commitmentTracker, subscriptionPool, accountFollowMePeerViews: async () => { const nickById = new Map((_listPoolMachines?.() ?? []).map((m) => [m.machineId, m.nickname ?? m.machineId])); let peers = (_resolvePeerUrls?.() ?? []).map((p) => ({ machineId: p.machineId, nickname: nickById.get(p.machineId) ?? p.machineId, url: p.url })); if (peers.length === 0) {
|
|
22800
23126
|
peers = (_listPoolMachines?.() ?? []).filter((m) => m.machineId !== _meshSelfId && !!m.lastKnownUrl).map((m) => ({ machineId: m.machineId, nickname: m.nickname ?? m.machineId, url: m.lastKnownUrl }));
|
|
22801
23127
|
} if (peers.length === 0)
|
|
22802
|
-
return []; const { fetchPeerSubscriptionViews } = await import('../core/fetchPeerSubscriptionViews.js'); return fetchPeerSubscriptionViews({ peers: () => peers, fetchImpl: fetch, authToken: config.authToken ?? '' }); }, quotaPoller, quotaAwareScheduler: _quotaAwareScheduler ?? undefined, proactiveSwapMonitor: _proactiveSwapMonitor ?? undefined, inUseAccountResolver, enrollmentWizard, accountFollowMeRevocation, credentialRepointing, semanticMemory, activitySentinel, rateLimitSentinel, releaseReadinessSentinel: releaseReadinessSentinel ?? undefined, greenPrAutoMerger: greenPrAutoMerger ?? undefined, guardLatchStore: guardLatchStore ?? undefined, messageRouter, summarySentinel, spawnManager, systemReviewer, capabilityMapper, selfKnowledgeTree, coverageAuditor, topicResumeMap: _topicResumeMap ?? undefined, topicProfile: _topicProfileCtx ?? undefined, sessionRefresh: _sessionRefresh ?? undefined, autonomyManager, trustElevationTracker, autonomousEvolution, coordinator: coordinator.enabled ? coordinator : undefined, meshBindActive: coordinator.managers.identityManager.hasIdentity() && config.multiMachine?.meshTransport?.enabled !== false, localSigningKeyPem, leaseTransport, peerEndpointRecorder, getSelfMeshEndpoints, onLeasePullRequest: () => leaseCoordinatorRef?.currentLease() ?? null, liveTailReceiver, handoffWireTransport, onHandoffBegin, onHandoffInitiate: handoffInitiate, handoffInProgress: handoffSentinelInProgress, messageLedger, currentInboundByTopic, replyMarkerTransport, onReplyMarker: messageLedger ? (marker) => { const m = marker; messageLedger.applyRemoteReplyMarker(m.dedupeKey, { platform: m.platform, replyIdempotencyKey: m.replyIdempotencyKey, epoch: m.epoch, topic: m.topic ?? null }); } : undefined, whatsapp: whatsappAdapter, slack: slackAdapter, imessage: imessageAdapter, conversationRegistry, conversationBindAuth, conversationFollowThrough, whatsappBusinessBackend, messageBridge, hookEventReceiver, worktreeMonitor, subagentTracker, instructionsVerifier, handshakeManager: threadlineHandshake, threadlineRouter, conversationStore, threadLog, threadMessageRecorder, warrantsReplyGate, collaborationSurfacer, threadResumeMap, topicLinkageHandler: topicLinkageHandler ?? undefined, threadlineRelayClient, threadlineReplyWaiters, listenerManager: listenerManager ?? undefined, a2aDeliveryTracker: a2aDeliveryTracker ?? undefined, responseReviewGate, reviewCanaryBattery, messagingToneGate, outboundDedupGate, telemetryHeartbeat, pasteManager, featureRegistry, discoveryEvaluator, completionEvaluator, unifiedTrust, liveConfig, sharedStateLedger, ledgerSessionRegistry, worktreeManager, oidcEnrolledRepos: parallelDevConfig?.oidcEnrolledRepos, initiativeTracker, projectRoundRunner, projectDriftChecker, machineHeartbeat, machinePoolRegistry, ropeHealthMonitor, writeAdmission: writeAdmission ?? undefined, getInboundQueue: () => _inboundQueue, getMachineCoherence: () => _machineCoherenceSentinel, meshRpcDispatcher, deliverA2aToMachine: _deliverA2aToMachine ?? undefined, workingSetPullCoordinator, workingSetArtifactManager, orchestratorPoller, commitmentReplicaStore, preferenceReplicaStore, replicatedRecordEmitter, conflictStore, rollbackUnmerge, droppedOriginRegistry, preferencesUnionReader, forwardCommitmentMutate, sessionOwnershipRegistry, sendDrain: _sendDrain ?? undefined, topicPinStore: _topicPinStore ?? undefined, topicPinSkewQuarantine: _topicPinSkewQuarantine ?? undefined, topicPinFoldView: _topicPinFoldView ?? undefined, ownershipReconciler: _ownershipReconciler ?? undefined, staleOwnerEngine: _staleOwnerEngine ?? undefined, duplicateReconciler: _duplicateReconciler ?? undefined, ownerDarkLadder: _ownerDarkLadder ?? undefined, spawnAdmission: _spawnAdmission ?? undefined, judgmentProvenance: _judgmentProvenance ?? undefined, leaseHandback: _leaseHandbackCtx ?? undefined, streamTicketStore: _streamTicketStore ?? undefined, poolStreamAllowRemoteInput: config.dashboard?.poolStream?.allowRemoteInput ?? false, poolStreamConnector: _poolStreamConnector ?? undefined, secretSync: _secretSyncHandle ?? undefined, meshSelfId: _meshSelfId ?? undefined, resolveRouterUrl: _resolveRouterUrl ?? undefined, resolvePeerUrls: _resolvePeerUrls ?? undefined, guardRegistry, listPoolMachines: _listPoolMachines ?? undefined, deliverMandateToMachine: _deliverMandateToMachine ?? undefined, poolLink: _poolLink ?? undefined, poolPollCache: _poolPollCache ?? undefined, sessionPoolE2EResultStore, proxyCoordinator, topicIntentStore, topicIntentArcCheck, usherSignalStore, intelligence: sharedIntelligence ?? undefined, telegramBridgeConfig, telegramBridge: telegramBridge ?? undefined, threadlineObservability, briefDeps, workingMemory, taskFlowRegistry, threadlineFlowBridge, sessionReaper, agentWorktreeReaper, externalHogSentinel, orphanedWorkSentinel, mcpProcessReaper, geminiLoopRunner, sleepController, agentActivityState, reapLog, resumeQueue, resumeDrainer, autonomousLivenessReconciler, enforcedTerminationStatus: () => enforcedTerminationWatchdog?.guardStatus() ?? null, prHandLease: prHandLease ?? undefined, operatorStopRecorder: recordOperatorStop, sleepWakeDetector, unjustifiedStopGate, stopGateDb, stopNotifier, liveTestGate, liveTestGateMode, liveTestRunnerCtx }); // Resolve the late-bound topic-operator getter (increment 2e): routing was
|
|
23128
|
+
return []; const { fetchPeerSubscriptionViews } = await import('../core/fetchPeerSubscriptionViews.js'); return fetchPeerSubscriptionViews({ peers: () => peers, fetchImpl: fetch, authToken: config.authToken ?? '' }); }, quotaPoller, quotaAwareScheduler: _quotaAwareScheduler ?? undefined, proactiveSwapMonitor: _proactiveSwapMonitor ?? undefined, inUseAccountResolver, enrollmentWizard, accountFollowMeRevocation, credentialRepointing, semanticMemory, activitySentinel, rateLimitSentinel, releaseReadinessSentinel: releaseReadinessSentinel ?? undefined, greenPrAutoMerger: greenPrAutoMerger ?? undefined, guardLatchStore: guardLatchStore ?? undefined, messageRouter, summarySentinel, spawnManager, systemReviewer, capabilityMapper, selfKnowledgeTree, coverageAuditor, topicResumeMap: _topicResumeMap ?? undefined, topicProfile: _topicProfileCtx ?? undefined, sessionRefresh: _sessionRefresh ?? undefined, autonomyManager, trustElevationTracker, autonomousEvolution, coordinator: coordinator.enabled ? coordinator : undefined, meshBindActive: coordinator.managers.identityManager.hasIdentity() && config.multiMachine?.meshTransport?.enabled !== false, localSigningKeyPem, leaseTransport, peerEndpointRecorder, getSelfMeshEndpoints, onLeasePullRequest: () => leaseCoordinatorRef?.currentLease() ?? null, liveTailReceiver, handoffWireTransport, onHandoffBegin, onHandoffInitiate: handoffInitiate, handoffInProgress: handoffSentinelInProgress, messageLedger, currentInboundByTopic, replyMarkerTransport, onReplyMarker: messageLedger ? (marker) => { const m = marker; messageLedger.applyRemoteReplyMarker(m.dedupeKey, { platform: m.platform, replyIdempotencyKey: m.replyIdempotencyKey, epoch: m.epoch, topic: m.topic ?? null }); } : undefined, whatsapp: whatsappAdapter, slack: slackAdapter, imessage: imessageAdapter, conversationRegistry, conversationBindAuth, conversationFollowThrough, whatsappBusinessBackend, messageBridge, hookEventReceiver, worktreeMonitor, subagentTracker, instructionsVerifier, handshakeManager: threadlineHandshake, threadlineRouter, conversationStore, threadLog, threadMessageRecorder, warrantsReplyGate, collaborationSurfacer, threadResumeMap, topicLinkageHandler: topicLinkageHandler ?? undefined, threadlineRelayClient, threadlineReplyWaiters, listenerManager: listenerManager ?? undefined, a2aDeliveryTracker: a2aDeliveryTracker ?? undefined, responseReviewGate, reviewCanaryBattery, messagingToneGate, outboundDedupGate, telemetryHeartbeat, pasteManager, featureRegistry, discoveryEvaluator, completionEvaluator, unifiedTrust, liveConfig, sharedStateLedger, ledgerSessionRegistry, worktreeManager, oidcEnrolledRepos: parallelDevConfig?.oidcEnrolledRepos, initiativeTracker, projectRoundRunner, projectDriftChecker, machineHeartbeat, machinePoolRegistry, ropeHealthMonitor, writeAdmission: writeAdmission ?? undefined, getInboundQueue: () => _inboundQueue, getMachineCoherence: () => _machineCoherenceSentinel, getSingleMachineFailoverGap: () => _singleMachineFailoverGap, getMissingLoginSession: () => _missingLoginSession, getSessionPoolFailoverRunner: () => _sessionPoolFailoverRunnerDriver?.status() ?? null, meshRpcDispatcher, deliverA2aToMachine: _deliverA2aToMachine ?? undefined, workingSetPullCoordinator, workingSetArtifactManager, orchestratorPoller, commitmentReplicaStore, preferenceReplicaStore, replicatedRecordEmitter, conflictStore, rollbackUnmerge, droppedOriginRegistry, preferencesUnionReader, forwardCommitmentMutate, sessionOwnershipRegistry, sendDrain: _sendDrain ?? undefined, topicPinStore: _topicPinStore ?? undefined, topicPinSkewQuarantine: _topicPinSkewQuarantine ?? undefined, topicPinFoldView: _topicPinFoldView ?? undefined, ownershipReconciler: _ownershipReconciler ?? undefined, staleOwnerEngine: _staleOwnerEngine ?? undefined, duplicateReconciler: _duplicateReconciler ?? undefined, ownerDarkLadder: _ownerDarkLadder ?? undefined, spawnAdmission: _spawnAdmission ?? undefined, judgmentProvenance: _judgmentProvenance ?? undefined, leaseHandback: _leaseHandbackCtx ?? undefined, streamTicketStore: _streamTicketStore ?? undefined, poolStreamAllowRemoteInput: config.dashboard?.poolStream?.allowRemoteInput ?? false, poolStreamConnector: _poolStreamConnector ?? undefined, secretSync: _secretSyncHandle ?? undefined, meshSelfId: _meshSelfId ?? undefined, resolveRouterUrl: _resolveRouterUrl ?? undefined, resolvePeerUrls: _resolvePeerUrls ?? undefined, guardRegistry, listPoolMachines: _listPoolMachines ?? undefined, deliverMandateToMachine: _deliverMandateToMachine ?? undefined, poolLink: _poolLink ?? undefined, poolPollCache: _poolPollCache ?? undefined, sessionPoolE2EResultStore, proxyCoordinator, topicIntentStore, topicIntentArcCheck, usherSignalStore, intelligence: sharedIntelligence ?? undefined, telegramBridgeConfig, telegramBridge: telegramBridge ?? undefined, threadlineObservability, briefDeps, workingMemory, taskFlowRegistry, threadlineFlowBridge, sessionReaper, agentWorktreeReaper, externalHogSentinel, orphanedWorkSentinel, mcpProcessReaper, geminiLoopRunner, sleepController, agentActivityState, reapLog, resumeQueue, resumeDrainer, autonomousLivenessReconciler, enforcedTerminationStatus: () => enforcedTerminationWatchdog?.guardStatus() ?? null, prHandLease: prHandLease ?? undefined, operatorStopRecorder: recordOperatorStop, sleepWakeDetector, unjustifiedStopGate, stopGateDb, stopNotifier, liveTestGate, liveTestGateMode, liveTestRunnerCtx }); // Resolve the late-bound topic-operator getter (increment 2e): routing was
|
|
22803
23129
|
if (_stateSyncStoresResolved?.classReview?.enabled && replicatedPeerStreamReader) {
|
|
22804
23130
|
const { CLASS_REVIEW_STORE_KEY, classReviewFromOriginRecord } = await import('../core/ClassReviewReplicatedStore.js');
|
|
22805
23131
|
const reader = replicatedPeerStreamReader;
|