@roj-ai/sdk 0.1.33 → 0.1.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap.d.ts +39 -0
- package/dist/bootstrap.d.ts.map +1 -1
- package/dist/bootstrap.js +5 -0
- package/dist/bootstrap.js.map +1 -1
- package/dist/core/agents/agent.d.ts +11 -0
- package/dist/core/agents/agent.d.ts.map +1 -1
- package/dist/core/agents/agent.js +19 -2
- package/dist/core/agents/agent.js.map +1 -1
- package/dist/core/preset/index.d.ts +1 -0
- package/dist/core/preset/index.d.ts.map +1 -1
- package/dist/core/preset/index.js +2 -0
- package/dist/core/preset/index.js.map +1 -1
- package/dist/core/preset/overrides.d.ts +14 -0
- package/dist/core/preset/overrides.d.ts.map +1 -0
- package/dist/core/preset/overrides.js +20 -0
- package/dist/core/preset/overrides.js.map +1 -0
- package/dist/core/sessions/session-manager.d.ts +5 -0
- package/dist/core/sessions/session-manager.d.ts.map +1 -1
- package/dist/core/sessions/session-manager.js +20 -6
- package/dist/core/sessions/session-manager.js.map +1 -1
- package/dist/core/sessions/session-overrides.integration.test.d.ts +2 -0
- package/dist/core/sessions/session-overrides.integration.test.d.ts.map +1 -0
- package/dist/core/sessions/session-overrides.integration.test.js +154 -0
- package/dist/core/sessions/session-overrides.integration.test.js.map +1 -0
- package/dist/core/sessions/session.d.ts +10 -1
- package/dist/core/sessions/session.d.ts.map +1 -1
- package/dist/core/sessions/session.js +15 -0
- package/dist/core/sessions/session.js.map +1 -1
- package/dist/core/sessions/state.d.ts +57 -0
- package/dist/core/sessions/state.d.ts.map +1 -1
- package/dist/core/sessions/state.js +48 -0
- package/dist/core/sessions/state.js.map +1 -1
- package/dist/core/system.d.ts +2 -0
- package/dist/core/system.d.ts.map +1 -1
- package/dist/core/system.js +2 -1
- package/dist/core/system.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/plugins/services/pid-registry.d.ts +72 -0
- package/dist/plugins/services/pid-registry.d.ts.map +1 -0
- package/dist/plugins/services/pid-registry.js +171 -0
- package/dist/plugins/services/pid-registry.js.map +1 -0
- package/dist/plugins/services/pid-registry.test.d.ts +2 -0
- package/dist/plugins/services/pid-registry.test.d.ts.map +1 -0
- package/dist/plugins/services/pid-registry.test.js +161 -0
- package/dist/plugins/services/pid-registry.test.js.map +1 -0
- package/dist/plugins/services/plugin.d.ts +6 -0
- package/dist/plugins/services/plugin.d.ts.map +1 -1
- package/dist/plugins/services/plugin.js +55 -15
- package/dist/plugins/services/plugin.js.map +1 -1
- package/dist/plugins/services/prompt.d.ts.map +1 -1
- package/dist/plugins/services/prompt.js +8 -0
- package/dist/plugins/services/prompt.js.map +1 -1
- package/dist/plugins/services/schema.d.ts +33 -0
- package/dist/plugins/services/schema.d.ts.map +1 -1
- package/dist/plugins/services/service.d.ts +24 -0
- package/dist/plugins/services/service.d.ts.map +1 -1
- package/dist/plugins/services/service.js +96 -1
- package/dist/plugins/services/service.js.map +1 -1
- package/dist/plugins/services/services.integration.test.js +254 -0
- package/dist/plugins/services/services.integration.test.js.map +1 -1
- package/dist/plugins/session-lifecycle/plugin.d.ts +36 -0
- package/dist/plugins/session-lifecycle/plugin.d.ts.map +1 -1
- package/dist/plugins/session-lifecycle/plugin.js +53 -1
- package/dist/plugins/session-lifecycle/plugin.js.map +1 -1
- package/dist/testing/test-harness.d.ts +12 -2
- package/dist/testing/test-harness.d.ts.map +1 -1
- package/dist/testing/test-harness.js +13 -2
- package/dist/testing/test-harness.js.map +1 -1
- package/dist/transport/http/middleware/error-handler.d.ts +1 -1
- package/package.json +2 -2
- package/src/bootstrap.ts +7 -0
- package/src/core/agents/agent.ts +19 -2
- package/src/core/preset/index.ts +3 -0
- package/src/core/preset/overrides.ts +24 -0
- package/src/core/sessions/session-manager.ts +27 -4
- package/src/core/sessions/session-overrides.integration.test.ts +210 -0
- package/src/core/sessions/session.ts +19 -1
- package/src/core/sessions/state.ts +77 -0
- package/src/core/system.ts +4 -0
- package/src/index.ts +8 -1
- package/src/plugins/services/pid-registry.test.ts +180 -0
- package/src/plugins/services/pid-registry.ts +191 -0
- package/src/plugins/services/plugin.ts +58 -15
- package/src/plugins/services/prompt.ts +8 -0
- package/src/plugins/services/schema.ts +33 -0
- package/src/plugins/services/service.ts +125 -1
- package/src/plugins/services/services.integration.test.ts +270 -0
- package/src/plugins/session-lifecycle/plugin.ts +59 -1
- package/src/testing/test-harness.ts +22 -3
|
@@ -13,6 +13,7 @@ import { Err, Ok } from '~/lib/utils/result.js'
|
|
|
13
13
|
import type { FileSystem } from '~/platform/fs.js'
|
|
14
14
|
import type { ChildProcess, ProcessRunner } from '~/platform/process.js'
|
|
15
15
|
import type { PortPool } from '~/plugins/services/port-pool.js'
|
|
16
|
+
import type { ServicePidRegistry } from '~/plugins/services/pid-registry.js'
|
|
16
17
|
import type { ServiceConfig, ServiceStartArgs, ServiceStatus } from '~/plugins/services/schema.js'
|
|
17
18
|
import type { ToolError } from '../../core/tools/executor.js'
|
|
18
19
|
import type { Logger } from '../../lib/logger/logger.js'
|
|
@@ -69,6 +70,17 @@ export interface ServiceStatusChangeDetails {
|
|
|
69
70
|
pidStartTime?: number
|
|
70
71
|
cwd?: string
|
|
71
72
|
command?: string
|
|
73
|
+
/** Set on a `failed` change that already has a revival queued — see ServiceEntry.restartAt. */
|
|
74
|
+
restartAt?: number
|
|
75
|
+
restartAttempt?: number
|
|
76
|
+
restartMaxRetries?: number
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** A revival the policy has queued, as reported on the `failed` status change. */
|
|
80
|
+
interface ScheduledRestart {
|
|
81
|
+
restartAt: number
|
|
82
|
+
restartAttempt: number
|
|
83
|
+
restartMaxRetries: number
|
|
72
84
|
}
|
|
73
85
|
|
|
74
86
|
/** Matches a port bind-conflict across the Node and Bun runtimes. */
|
|
@@ -82,6 +94,12 @@ const PORT_CONFLICT_PATTERN = /EADDRINUSE|address already in use/i
|
|
|
82
94
|
*/
|
|
83
95
|
const MAX_PORT_CONFLICT_RETRIES = 3
|
|
84
96
|
|
|
97
|
+
/** `restartPolicy` defaults — see ServiceConfig.restartPolicy for the rationale. */
|
|
98
|
+
const DEFAULT_MAX_RESTART_RETRIES = 3
|
|
99
|
+
const DEFAULT_RESTART_DELAY_MS = 1000
|
|
100
|
+
const DEFAULT_MAX_RESTART_DELAY_MS = 30_000
|
|
101
|
+
const DEFAULT_RESTART_HEALTHY_AFTER_MS = 60_000
|
|
102
|
+
|
|
85
103
|
// ============================================================================
|
|
86
104
|
// ServiceExecutor
|
|
87
105
|
// ============================================================================
|
|
@@ -89,6 +107,8 @@ const MAX_PORT_CONFLICT_RETRIES = 3
|
|
|
89
107
|
export interface ServiceExecutorDeps {
|
|
90
108
|
fs: FileSystem
|
|
91
109
|
process: ProcessRunner
|
|
110
|
+
/** Durable pid record, swept at agent boot. Optional so embedders without a data dir still work. */
|
|
111
|
+
pidRegistry?: ServicePidRegistry
|
|
92
112
|
}
|
|
93
113
|
|
|
94
114
|
export class ServiceExecutor {
|
|
@@ -99,10 +119,15 @@ export class ServiceExecutor {
|
|
|
99
119
|
private readonly startInFlight = new Map<string, Promise<Result<void, ToolError>>>()
|
|
100
120
|
/** Per-type counter bounding automatic EADDRINUSE port re-allocation retries. */
|
|
101
121
|
private readonly portConflictRetries = new Map<string, number>()
|
|
122
|
+
/** Per-type counter bounding `restartPolicy` revivals after an unexpected exit. */
|
|
123
|
+
private readonly restartRetries = new Map<string, number>()
|
|
124
|
+
/** Pending `restartPolicy` timers, cancelled when someone stops or starts the service by hand. */
|
|
125
|
+
private readonly restartTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
|
102
126
|
private readonly logger: Logger
|
|
103
127
|
private readonly portPool: PortPool
|
|
104
128
|
private readonly fs: FileSystem
|
|
105
129
|
private readonly processRunner: ProcessRunner
|
|
130
|
+
private readonly pidRegistry?: ServicePidRegistry
|
|
106
131
|
|
|
107
132
|
/** Optional callback invoked on every service status change */
|
|
108
133
|
onStatusChanged?: (
|
|
@@ -117,6 +142,7 @@ export class ServiceExecutor {
|
|
|
117
142
|
this.portPool = portPool
|
|
118
143
|
this.fs = deps.fs
|
|
119
144
|
this.processRunner = deps.process
|
|
145
|
+
this.pidRegistry = deps.pidRegistry
|
|
120
146
|
}
|
|
121
147
|
|
|
122
148
|
private notifyStatusChanged(
|
|
@@ -249,6 +275,9 @@ export class ServiceExecutor {
|
|
|
249
275
|
workspaceDir?: string,
|
|
250
276
|
preferredPort?: number,
|
|
251
277
|
): Promise<Result<void, ToolError>> {
|
|
278
|
+
// This start supersedes any revival the policy had queued.
|
|
279
|
+
this.cancelPendingRestart(config.type)
|
|
280
|
+
|
|
252
281
|
const existing = this.services.get(config.type)
|
|
253
282
|
if (existing && (existing.status === 'starting' || existing.status === 'ready')) {
|
|
254
283
|
return Ok(undefined)
|
|
@@ -354,6 +383,11 @@ export class ServiceExecutor {
|
|
|
354
383
|
// "our process" from "an unrelated process that grabbed this PID after ours died"
|
|
355
384
|
const pidStartTime = await getProcessStartTime(this.fs, child.pid)
|
|
356
385
|
|
|
386
|
+
// Durably record the process so a future agent can reap it if we die before the
|
|
387
|
+
// `close` handler gets to forget it. The session projection cannot serve that
|
|
388
|
+
// purpose — see ServicePidRegistry.
|
|
389
|
+
await this.pidRegistry?.record({ sessionId: String(sessionId), serviceType: config.type, pid: child.pid, pidStartTime, command })
|
|
390
|
+
|
|
357
391
|
// Emit starting event with PID, port, resolved cwd/command, and start time.
|
|
358
392
|
this.notifyStatusChanged(sessionId, config.type, 'starting', { port, pid: child.pid, pidStartTime, cwd, command })
|
|
359
393
|
|
|
@@ -512,6 +546,8 @@ export class ServiceExecutor {
|
|
|
512
546
|
// Handle unexpected exit
|
|
513
547
|
child.on('close', (code) => {
|
|
514
548
|
clearReadinessTimers()
|
|
549
|
+
// The process is gone, so its durable record has nothing left to reap.
|
|
550
|
+
void this.pidRegistry?.forget(String(sessionId), config.type)
|
|
515
551
|
// Flush remaining partial lines
|
|
516
552
|
if (stdoutPartial) {
|
|
517
553
|
processLine(stdoutPartial)
|
|
@@ -555,17 +591,21 @@ export class ServiceExecutor {
|
|
|
555
591
|
// Unexpected exit
|
|
556
592
|
current.status = 'failed'
|
|
557
593
|
const errorMsg = `Process exited unexpectedly with code ${code}`
|
|
594
|
+
this.portConflictRetries.delete(config.type)
|
|
595
|
+
// Queue the revival first so the `failed` change can carry it — a
|
|
596
|
+
// consumer that sees a bare `failed` is entitled to treat it as terminal.
|
|
597
|
+
const revival = this.scheduleRestart(config, sessionId, workspaceDir, Date.now() - startTime, code)
|
|
558
598
|
this.notifyStatusChanged(sessionId, config.type, 'failed', {
|
|
559
599
|
port: current.port,
|
|
560
600
|
cwd: current.cwd,
|
|
561
601
|
command: current.command,
|
|
562
602
|
error: errorMsg,
|
|
603
|
+
...revival,
|
|
563
604
|
})
|
|
564
605
|
this.logger.warn('Service process exited unexpectedly', {
|
|
565
606
|
serviceType: config.type,
|
|
566
607
|
code,
|
|
567
608
|
})
|
|
568
|
-
this.portConflictRetries.delete(config.type)
|
|
569
609
|
}
|
|
570
610
|
})
|
|
571
611
|
|
|
@@ -587,16 +627,96 @@ export class ServiceExecutor {
|
|
|
587
627
|
return Ok(undefined)
|
|
588
628
|
}
|
|
589
629
|
|
|
630
|
+
/**
|
|
631
|
+
* Bring a crashed service back if its `restartPolicy` allows it. The port
|
|
632
|
+
* allocation is kept on purpose — the preview URL has to survive the bounce.
|
|
633
|
+
*
|
|
634
|
+
* Returns the queued revival so the caller can put it on the `failed` status
|
|
635
|
+
* change, or undefined when the service is left failed for good.
|
|
636
|
+
*/
|
|
637
|
+
private scheduleRestart(
|
|
638
|
+
config: ServiceConfig,
|
|
639
|
+
sessionId: SessionId,
|
|
640
|
+
workspaceDir: string | undefined,
|
|
641
|
+
uptimeMs: number,
|
|
642
|
+
exitCode: number | null,
|
|
643
|
+
): ScheduledRestart | undefined {
|
|
644
|
+
const policy = config.restartPolicy
|
|
645
|
+
if (!policy) return undefined
|
|
646
|
+
|
|
647
|
+
// A service that ran for a while and then died is a new problem, not the
|
|
648
|
+
// continuation of a boot loop — give it a full budget again.
|
|
649
|
+
if (uptimeMs >= (policy.healthyAfterMs ?? DEFAULT_RESTART_HEALTHY_AFTER_MS)) {
|
|
650
|
+
this.restartRetries.delete(config.type)
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
const attempt = this.restartRetries.get(config.type) ?? 0
|
|
654
|
+
const maxRetries = policy.maxRetries ?? DEFAULT_MAX_RESTART_RETRIES
|
|
655
|
+
if (attempt >= maxRetries) {
|
|
656
|
+
this.logger.warn('Service left failed — automatic restart budget spent', {
|
|
657
|
+
serviceType: config.type,
|
|
658
|
+
attempts: attempt,
|
|
659
|
+
exitCode,
|
|
660
|
+
})
|
|
661
|
+
this.restartRetries.delete(config.type)
|
|
662
|
+
return undefined
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
const initialDelayMs = policy.initialDelayMs ?? DEFAULT_RESTART_DELAY_MS
|
|
666
|
+
const delayMs = Math.min(initialDelayMs * 2 ** attempt, policy.maxDelayMs ?? DEFAULT_MAX_RESTART_DELAY_MS)
|
|
667
|
+
this.restartRetries.set(config.type, attempt + 1)
|
|
668
|
+
this.logger.info('Restarting service after unexpected exit', {
|
|
669
|
+
serviceType: config.type,
|
|
670
|
+
attempt: attempt + 1,
|
|
671
|
+
maxRetries,
|
|
672
|
+
delayMs,
|
|
673
|
+
exitCode,
|
|
674
|
+
})
|
|
675
|
+
|
|
676
|
+
const timer = setTimeout(() => {
|
|
677
|
+
this.restartTimers.delete(config.type)
|
|
678
|
+
void this.start(config, sessionId, workspaceDir)
|
|
679
|
+
}, delayMs)
|
|
680
|
+
// A pending revival must not keep the process alive on its own.
|
|
681
|
+
timer.unref?.()
|
|
682
|
+
this.restartTimers.set(config.type, timer)
|
|
683
|
+
|
|
684
|
+
return { restartAt: Date.now() + delayMs, restartAttempt: attempt + 1, restartMaxRetries: maxRetries }
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/** Drop a pending automatic restart — an explicit start or stop wins over it. Returns true if one was queued. */
|
|
688
|
+
private cancelPendingRestart(serviceType: string): boolean {
|
|
689
|
+
const timer = this.restartTimers.get(serviceType)
|
|
690
|
+
if (!timer) return false
|
|
691
|
+
clearTimeout(timer)
|
|
692
|
+
this.restartTimers.delete(serviceType)
|
|
693
|
+
return true
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/** Whether a `restartPolicy` revival is queued — the service is down but coming back. */
|
|
697
|
+
hasScheduledRestart(serviceType: string): boolean {
|
|
698
|
+
return this.restartTimers.has(serviceType)
|
|
699
|
+
}
|
|
700
|
+
|
|
590
701
|
/**
|
|
591
702
|
* Stop a running service gracefully.
|
|
592
703
|
* Port is NOT released — kept for session-level stability across restarts.
|
|
593
704
|
*/
|
|
594
705
|
async stop(serviceType: string, sessionId: SessionId): Promise<Result<void, ToolError>> {
|
|
706
|
+
const hadPendingRestart = this.cancelPendingRestart(serviceType)
|
|
707
|
+
this.restartRetries.delete(serviceType)
|
|
595
708
|
const entry = this.services.get(serviceType)
|
|
596
709
|
if (!entry) {
|
|
597
710
|
return Err({ message: `Service '${serviceType}' not found`, recoverable: false })
|
|
598
711
|
}
|
|
599
712
|
if (entry.status !== 'starting' && entry.status !== 'ready' && entry.status !== 'paused') {
|
|
713
|
+
// A failed service with a revival queued is really "about to restart",
|
|
714
|
+
// and calling that off is a legitimate stop rather than an error.
|
|
715
|
+
if (hadPendingRestart) {
|
|
716
|
+
entry.status = 'stopped'
|
|
717
|
+
this.notifyStatusChanged(sessionId, serviceType, 'stopped')
|
|
718
|
+
return Ok(undefined)
|
|
719
|
+
}
|
|
600
720
|
return Err({ message: `Service '${serviceType}' is ${entry.status}, cannot stop`, recoverable: false })
|
|
601
721
|
}
|
|
602
722
|
|
|
@@ -751,6 +871,10 @@ export class ServiceExecutor {
|
|
|
751
871
|
* Called on session close.
|
|
752
872
|
*/
|
|
753
873
|
async shutdown(): Promise<void> {
|
|
874
|
+
// Nothing queued may spawn after the executor is gone.
|
|
875
|
+
for (const serviceType of [...this.restartTimers.keys()]) this.cancelPendingRestart(serviceType)
|
|
876
|
+
this.restartRetries.clear()
|
|
877
|
+
|
|
754
878
|
const promises: Promise<void>[] = []
|
|
755
879
|
|
|
756
880
|
for (const [serviceType, entry] of this.services) {
|
|
@@ -15,6 +15,7 @@ import type { ServiceAgentConfig, ServicePluginConfig } from './plugin.js'
|
|
|
15
15
|
import { PortPool } from './port-pool.js'
|
|
16
16
|
import { buildServiceStatusMessage } from './prompt.js'
|
|
17
17
|
import type { ServiceCommandArgs, ServiceConfig, ServiceCwdArgs, ServiceEntry, ServiceStatus } from './schema.js'
|
|
18
|
+
import type { ServiceStatusChangeDetails } from './service.js'
|
|
18
19
|
import { ServiceExecutor } from './service.js'
|
|
19
20
|
|
|
20
21
|
// ============================================================================
|
|
@@ -823,6 +824,275 @@ describe('services plugin', () => {
|
|
|
823
824
|
})
|
|
824
825
|
})
|
|
825
826
|
|
|
827
|
+
// =========================================================================
|
|
828
|
+
// restartPolicy — reviving a service that exited on its own
|
|
829
|
+
// =========================================================================
|
|
830
|
+
|
|
831
|
+
describe('restartPolicy', () => {
|
|
832
|
+
const waitUntil = async (predicate: () => boolean, timeoutMs = 8000): Promise<void> => {
|
|
833
|
+
const deadline = Date.now() + timeoutMs
|
|
834
|
+
while (Date.now() < deadline) {
|
|
835
|
+
if (predicate()) return
|
|
836
|
+
await new Promise((r) => setTimeout(r, 20))
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
const waitUntilAsync = async (predicate: () => Promise<boolean>, timeoutMs = 8000): Promise<void> => {
|
|
841
|
+
const deadline = Date.now() + timeoutMs
|
|
842
|
+
while (Date.now() < deadline) {
|
|
843
|
+
if (await predicate()) return
|
|
844
|
+
await new Promise((r) => setTimeout(r, 20))
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
const marker = (name: string) => join(tmpdir(), `roj-restart-${name}-${Date.now()}-${Math.floor(Math.random() * 1e6)}`)
|
|
849
|
+
/** Records every spawn, then crashes or stays up depending on the run count. */
|
|
850
|
+
const crashCommand = (path: string, crashRuns: number) =>
|
|
851
|
+
`echo x >> ${path}; runs=$(wc -l < ${path} | tr -d ' '); if [ "$runs" -le ${crashRuns} ]; then exit 1; fi; echo "listening READY"; sleep 30`
|
|
852
|
+
const spawnCount = async (path: string): Promise<number> => {
|
|
853
|
+
try {
|
|
854
|
+
return (await readFile(path, 'utf-8')).split('\n').filter((l) => l.length > 0).length
|
|
855
|
+
} catch {
|
|
856
|
+
return 0
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
it('revives a crashed service on the port it already holds', async () => {
|
|
861
|
+
const platform = createNodePlatform()
|
|
862
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
863
|
+
const path = marker('revive')
|
|
864
|
+
const config: ServiceConfig = {
|
|
865
|
+
type: 'revive-svc',
|
|
866
|
+
description: 'crashes once, then stays up',
|
|
867
|
+
command: crashCommand(path, 1),
|
|
868
|
+
readyPattern: 'READY',
|
|
869
|
+
startupTimeoutMs: 5000,
|
|
870
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 20 },
|
|
871
|
+
}
|
|
872
|
+
const ports: number[] = []
|
|
873
|
+
executor.onStatusChanged = (_s, _t, status, details) => {
|
|
874
|
+
if (status === 'starting' && details.port !== undefined) ports.push(details.port)
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
try {
|
|
878
|
+
await executor.start(config, SessionId('s-revive'))
|
|
879
|
+
await waitUntil(() => executor.getStatus('revive-svc') === 'ready')
|
|
880
|
+
expect(executor.getStatus('revive-svc')).toBe('ready')
|
|
881
|
+
expect(await spawnCount(path)).toBe(2)
|
|
882
|
+
// The preview URL must survive the bounce.
|
|
883
|
+
expect(new Set(ports).size).toBe(1)
|
|
884
|
+
} finally {
|
|
885
|
+
await executor.shutdown()
|
|
886
|
+
await rm(path, { force: true })
|
|
887
|
+
}
|
|
888
|
+
})
|
|
889
|
+
|
|
890
|
+
it('leaves the service failed once the retry budget is spent', async () => {
|
|
891
|
+
const platform = createNodePlatform()
|
|
892
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
893
|
+
const path = marker('budget')
|
|
894
|
+
const config: ServiceConfig = {
|
|
895
|
+
type: 'budget-svc',
|
|
896
|
+
description: 'always crashes',
|
|
897
|
+
command: crashCommand(path, 99),
|
|
898
|
+
readyPattern: 'READY',
|
|
899
|
+
startupTimeoutMs: 5000,
|
|
900
|
+
restartPolicy: { maxRetries: 2, initialDelayMs: 20 },
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
try {
|
|
904
|
+
await executor.start(config, SessionId('s-budget'))
|
|
905
|
+
// One initial spawn plus two retries, then it stays down.
|
|
906
|
+
await waitUntilAsync(async () => (await spawnCount(path)) >= 3 && executor.getStatus('budget-svc') === 'failed')
|
|
907
|
+
await new Promise((r) => setTimeout(r, 300))
|
|
908
|
+
expect(await spawnCount(path)).toBe(3)
|
|
909
|
+
expect(executor.getStatus('budget-svc')).toBe('failed')
|
|
910
|
+
} finally {
|
|
911
|
+
await executor.shutdown()
|
|
912
|
+
await rm(path, { force: true })
|
|
913
|
+
}
|
|
914
|
+
})
|
|
915
|
+
|
|
916
|
+
it('without a policy a crash stays a crash', async () => {
|
|
917
|
+
const platform = createNodePlatform()
|
|
918
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
919
|
+
const path = marker('nopolicy')
|
|
920
|
+
const config: ServiceConfig = {
|
|
921
|
+
type: 'nopolicy-svc',
|
|
922
|
+
description: 'always crashes, no restartPolicy',
|
|
923
|
+
command: crashCommand(path, 99),
|
|
924
|
+
readyPattern: 'READY',
|
|
925
|
+
startupTimeoutMs: 5000,
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
try {
|
|
929
|
+
await executor.start(config, SessionId('s-nopolicy'))
|
|
930
|
+
await waitUntil(() => executor.getStatus('nopolicy-svc') === 'failed')
|
|
931
|
+
await new Promise((r) => setTimeout(r, 300))
|
|
932
|
+
expect(await spawnCount(path)).toBe(1)
|
|
933
|
+
} finally {
|
|
934
|
+
await executor.shutdown()
|
|
935
|
+
await rm(path, { force: true })
|
|
936
|
+
}
|
|
937
|
+
})
|
|
938
|
+
|
|
939
|
+
it('a queued revival rides along on the failed status change', async () => {
|
|
940
|
+
const platform = createNodePlatform()
|
|
941
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
942
|
+
const path = marker('event')
|
|
943
|
+
const config: ServiceConfig = {
|
|
944
|
+
type: 'event-svc',
|
|
945
|
+
description: 'crashes once, then stays up',
|
|
946
|
+
command: crashCommand(path, 1),
|
|
947
|
+
readyPattern: 'READY',
|
|
948
|
+
startupTimeoutMs: 5000,
|
|
949
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 20 },
|
|
950
|
+
}
|
|
951
|
+
const changes: Array<{ status: ServiceStatus; details: ServiceStatusChangeDetails }> = []
|
|
952
|
+
executor.onStatusChanged = (_s, _t, status, details) => {
|
|
953
|
+
changes.push({ status, details })
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
try {
|
|
957
|
+
await executor.start(config, SessionId('s-event'))
|
|
958
|
+
await waitUntil(() => executor.getStatus('event-svc') === 'ready')
|
|
959
|
+
|
|
960
|
+
const failedIndex = changes.findIndex((c) => c.status === 'failed')
|
|
961
|
+
expect(failedIndex).toBeGreaterThanOrEqual(0)
|
|
962
|
+
const failed = changes[failedIndex]!
|
|
963
|
+
expect(failed.details.restartAttempt).toBe(1)
|
|
964
|
+
expect(failed.details.restartMaxRetries).toBe(3)
|
|
965
|
+
expect(failed.details.restartAt).toBeGreaterThan(0)
|
|
966
|
+
|
|
967
|
+
// The revival's own start must not carry the intent it just honoured.
|
|
968
|
+
const revivalStart = changes.slice(failedIndex + 1).find((c) => c.status === 'starting')
|
|
969
|
+
expect(revivalStart).toBeDefined()
|
|
970
|
+
expect(revivalStart!.details.restartAt).toBeUndefined()
|
|
971
|
+
} finally {
|
|
972
|
+
await executor.shutdown()
|
|
973
|
+
await rm(path, { force: true })
|
|
974
|
+
}
|
|
975
|
+
})
|
|
976
|
+
|
|
977
|
+
it('the failure that spends the budget carries no revival', async () => {
|
|
978
|
+
const platform = createNodePlatform()
|
|
979
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
980
|
+
const path = marker('terminal')
|
|
981
|
+
const config: ServiceConfig = {
|
|
982
|
+
type: 'terminal-svc',
|
|
983
|
+
description: 'always crashes',
|
|
984
|
+
command: crashCommand(path, 99),
|
|
985
|
+
readyPattern: 'READY',
|
|
986
|
+
startupTimeoutMs: 5000,
|
|
987
|
+
restartPolicy: { maxRetries: 1, initialDelayMs: 20 },
|
|
988
|
+
}
|
|
989
|
+
const failures: ServiceStatusChangeDetails[] = []
|
|
990
|
+
executor.onStatusChanged = (_s, _t, status, details) => {
|
|
991
|
+
if (status === 'failed') failures.push(details)
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
try {
|
|
995
|
+
await executor.start(config, SessionId('s-terminal'))
|
|
996
|
+
await waitUntilAsync(async () => (await spawnCount(path)) >= 2 && failures.length >= 2)
|
|
997
|
+
expect(failures[0]?.restartAttempt).toBe(1)
|
|
998
|
+
expect(failures[1]?.restartAt).toBeUndefined()
|
|
999
|
+
expect(failures[1]?.restartAttempt).toBeUndefined()
|
|
1000
|
+
} finally {
|
|
1001
|
+
await executor.shutdown()
|
|
1002
|
+
await rm(path, { force: true })
|
|
1003
|
+
}
|
|
1004
|
+
})
|
|
1005
|
+
|
|
1006
|
+
it('session state shows the queued revival and drops it once called off', async () => {
|
|
1007
|
+
const portPool = new PortPool()
|
|
1008
|
+
const config: ServiceConfig = {
|
|
1009
|
+
type: 'reviving',
|
|
1010
|
+
description: 'crashes early, revival queued behind a long delay',
|
|
1011
|
+
// The sleep keeps the exit from beating the executor's close listener.
|
|
1012
|
+
command: 'sleep 0.2; exit 1',
|
|
1013
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 3000 },
|
|
1014
|
+
}
|
|
1015
|
+
const harness = createServicesHarness({
|
|
1016
|
+
presets: [createServicesPreset([config], ['reviving'], portPool)],
|
|
1017
|
+
})
|
|
1018
|
+
const session = await harness.createSession('test')
|
|
1019
|
+
|
|
1020
|
+
await session.callPluginMethod('services.start', { serviceType: 'reviving' })
|
|
1021
|
+
await waitForServiceStateStatus(session, 'reviving', 'failed')
|
|
1022
|
+
|
|
1023
|
+
const entry = selectPluginState<Map<string, ServiceEntry>>(session.state, 'services')?.get('reviving')
|
|
1024
|
+
expect(entry?.status).toBe('failed')
|
|
1025
|
+
expect(entry?.restartAt).toBeGreaterThan(Date.now())
|
|
1026
|
+
expect(entry?.restartAttempt).toBe(1)
|
|
1027
|
+
expect(entry?.restartMaxRetries).toBe(3)
|
|
1028
|
+
|
|
1029
|
+
const stopped = await session.callPluginMethod('services.stop', { serviceType: 'reviving' })
|
|
1030
|
+
expect(stopped.ok).toBe(true)
|
|
1031
|
+
await waitForServiceStateStatus(session, 'reviving', 'stopped')
|
|
1032
|
+
|
|
1033
|
+
const after = selectPluginState<Map<string, ServiceEntry>>(session.state, 'services')?.get('reviving')
|
|
1034
|
+
expect(after?.status).toBe('stopped')
|
|
1035
|
+
expect(after?.restartAt).toBeUndefined()
|
|
1036
|
+
expect(after?.restartAttempt).toBeUndefined()
|
|
1037
|
+
})
|
|
1038
|
+
|
|
1039
|
+
it('closing the session calls off a queued revival', async () => {
|
|
1040
|
+
const portPool = new PortPool()
|
|
1041
|
+
const path = marker('close')
|
|
1042
|
+
const config: ServiceConfig = {
|
|
1043
|
+
type: 'close-svc',
|
|
1044
|
+
description: 'crashes early, revival queued behind a delay',
|
|
1045
|
+
// The sleep keeps the exit from beating the executor's close listener.
|
|
1046
|
+
command: `echo x >> ${path}; sleep 0.2; exit 1`,
|
|
1047
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 800 },
|
|
1048
|
+
}
|
|
1049
|
+
const harness = createServicesHarness({
|
|
1050
|
+
presets: [createServicesPreset([config], ['close-svc'], portPool)],
|
|
1051
|
+
})
|
|
1052
|
+
const session = await harness.createSession('test')
|
|
1053
|
+
|
|
1054
|
+
try {
|
|
1055
|
+
await session.callPluginMethod('services.start', { serviceType: 'close-svc' })
|
|
1056
|
+
await waitForServiceStateStatus(session, 'close-svc', 'failed')
|
|
1057
|
+
expect(await spawnCount(path)).toBe(1)
|
|
1058
|
+
|
|
1059
|
+
await session.close()
|
|
1060
|
+
// Long enough for the revival to have fired had nothing called it off.
|
|
1061
|
+
await new Promise((r) => setTimeout(r, 1500))
|
|
1062
|
+
expect(await spawnCount(path)).toBe(1)
|
|
1063
|
+
} finally {
|
|
1064
|
+
await rm(path, { force: true })
|
|
1065
|
+
}
|
|
1066
|
+
})
|
|
1067
|
+
|
|
1068
|
+
it('an explicit stop calls off a queued revival', async () => {
|
|
1069
|
+
const platform = createNodePlatform()
|
|
1070
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
1071
|
+
const path = marker('stop')
|
|
1072
|
+
const config: ServiceConfig = {
|
|
1073
|
+
type: 'stop-svc',
|
|
1074
|
+
description: 'always crashes, revival queued behind a long delay',
|
|
1075
|
+
command: crashCommand(path, 99),
|
|
1076
|
+
readyPattern: 'READY',
|
|
1077
|
+
startupTimeoutMs: 5000,
|
|
1078
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 1500 },
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
try {
|
|
1082
|
+
await executor.start(config, SessionId('s-stop'))
|
|
1083
|
+
await waitUntil(() => executor.getStatus('stop-svc') === 'failed')
|
|
1084
|
+
const stopped = await executor.stop('stop-svc', SessionId('s-stop'))
|
|
1085
|
+
expect(stopped.ok).toBe(true)
|
|
1086
|
+
expect(executor.getStatus('stop-svc')).toBe('stopped')
|
|
1087
|
+
await new Promise((r) => setTimeout(r, 2000))
|
|
1088
|
+
expect(await spawnCount(path)).toBe(1)
|
|
1089
|
+
} finally {
|
|
1090
|
+
await executor.shutdown()
|
|
1091
|
+
await rm(path, { force: true })
|
|
1092
|
+
}
|
|
1093
|
+
})
|
|
1094
|
+
})
|
|
1095
|
+
|
|
826
1096
|
// =========================================================================
|
|
827
1097
|
// agentVisible: false — infrastructure services hidden from agents
|
|
828
1098
|
// =========================================================================
|
|
@@ -11,8 +11,9 @@ import { agentIdSchema } from '~/core/agents'
|
|
|
11
11
|
import { COMMUNICATOR_ROLE, ORCHESTRATOR_ROLE } from '~/core/agents/agent-roles.js'
|
|
12
12
|
import { type DomainError, PresetErrors, SessionErrors, ValidationErrors } from '~/core/errors.js'
|
|
13
13
|
import { definePlugin } from '~/core/plugins/index.js'
|
|
14
|
+
import { knownDefinitionNames, unknownOverrideTargets } from '~/core/preset/overrides.js'
|
|
14
15
|
import { SessionId, sessionIdSchema } from '~/core/sessions/schema.js'
|
|
15
|
-
import { getEntryAgentId, sessionEvents } from '~/core/sessions/state.js'
|
|
16
|
+
import { agentOverridesSchema, getEntryAgentId, sessionEvents, sessionOverridesPatchSchema } from '~/core/sessions/state.js'
|
|
16
17
|
import { Err, Ok } from '~/lib/utils/result.js'
|
|
17
18
|
|
|
18
19
|
// ============================================================================
|
|
@@ -115,6 +116,7 @@ export const sessionLifecyclePlugin = definePlugin('sessions')
|
|
|
115
116
|
presetId: z4.string().min(1),
|
|
116
117
|
workspaceDir: z4.string().optional(),
|
|
117
118
|
sessionId: z4.string().optional(),
|
|
119
|
+
overrides: sessionOverridesPatchSchema.optional(),
|
|
118
120
|
}),
|
|
119
121
|
output: z4.object({
|
|
120
122
|
sessionId: sessionIdSchema,
|
|
@@ -126,11 +128,13 @@ export const sessionLifecyclePlugin = definePlugin('sessions')
|
|
|
126
128
|
presetId: input.presetId,
|
|
127
129
|
workspaceDir: input.workspaceDir ?? null,
|
|
128
130
|
sessionId: input.sessionId ?? null,
|
|
131
|
+
overrides: input.overrides ?? null,
|
|
129
132
|
})
|
|
130
133
|
|
|
131
134
|
const sessionResult = await sm.createSession(input.presetId, {
|
|
132
135
|
workspaceDir: input.workspaceDir,
|
|
133
136
|
sessionId: input.sessionId,
|
|
137
|
+
overrides: input.overrides,
|
|
134
138
|
})
|
|
135
139
|
if (!sessionResult.ok) return sessionResult
|
|
136
140
|
|
|
@@ -183,6 +187,12 @@ export const sessionLifecyclePlugin = definePlugin('sessions')
|
|
|
183
187
|
closedAt: z4.number().optional(),
|
|
184
188
|
agentCount: z4.number(),
|
|
185
189
|
entryAgentId: agentIdSchema.nullable(),
|
|
190
|
+
// Reported back so a caller can tell "the override did not apply" from
|
|
191
|
+
// "this build predates overrides and dropped the field on input".
|
|
192
|
+
overrides: z4.object({
|
|
193
|
+
defaults: agentOverridesSchema.optional(),
|
|
194
|
+
agents: z4.record(z4.string(), agentOverridesSchema),
|
|
195
|
+
}),
|
|
186
196
|
}),
|
|
187
197
|
handler: async (ctx) => {
|
|
188
198
|
const state = ctx.sessionState
|
|
@@ -194,6 +204,54 @@ export const sessionLifecyclePlugin = definePlugin('sessions')
|
|
|
194
204
|
closedAt: state.closedAt,
|
|
195
205
|
agentCount: state.agents.size,
|
|
196
206
|
entryAgentId: getEntryAgentId(state),
|
|
207
|
+
overrides: {
|
|
208
|
+
defaults: state.overrides.defaults,
|
|
209
|
+
agents: Object.fromEntries(state.overrides.agents),
|
|
210
|
+
},
|
|
211
|
+
})
|
|
212
|
+
},
|
|
213
|
+
})
|
|
214
|
+
// A manager method, not a session method, purely so it can see the preset and
|
|
215
|
+
// reject unknown agent names — session method contexts carry no preset.
|
|
216
|
+
// `presets.getAgents` is a manager method for the same reason.
|
|
217
|
+
.managerMethod('setOverrides', {
|
|
218
|
+
input: z4.object({
|
|
219
|
+
sessionId: sessionIdSchema,
|
|
220
|
+
overrides: sessionOverridesPatchSchema,
|
|
221
|
+
}),
|
|
222
|
+
output: z4.object({
|
|
223
|
+
agents: z4.record(z4.string(), agentOverridesSchema),
|
|
224
|
+
defaults: agentOverridesSchema.optional(),
|
|
225
|
+
}),
|
|
226
|
+
handler: async (ctx, input) => {
|
|
227
|
+
const sm = ctx.sessionManager
|
|
228
|
+
const sessionId = SessionId(input.sessionId)
|
|
229
|
+
|
|
230
|
+
const sessionResult = await sm.getSession(sessionId)
|
|
231
|
+
if (!sessionResult.ok) return sessionResult
|
|
232
|
+
const session = sessionResult.value
|
|
233
|
+
|
|
234
|
+
const preset = ctx.presets.get(session.state.presetId)
|
|
235
|
+
if (!preset) return Err(PresetErrors.notFound(session.state.presetId))
|
|
236
|
+
|
|
237
|
+
const unknown = unknownOverrideTargets(preset, input.overrides)
|
|
238
|
+
if (unknown.length > 0) {
|
|
239
|
+
return Err(ValidationErrors.invalid(
|
|
240
|
+
`Unknown agent definitions in overrides: ${unknown.join(', ')}. Preset '${preset.id}' defines: ${knownDefinitionNames(preset).join(', ')}`,
|
|
241
|
+
))
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
await session.setOverrides(input.overrides)
|
|
245
|
+
|
|
246
|
+
// Agents already mid-turn finish on the model they started with: the
|
|
247
|
+
// override is read when the next inference request is built, not applied
|
|
248
|
+
// retroactively to one in flight.
|
|
249
|
+
ctx.logger.info('Session overrides set', { sessionId, overrides: input.overrides })
|
|
250
|
+
|
|
251
|
+
const state = session.state
|
|
252
|
+
return Ok({
|
|
253
|
+
agents: Object.fromEntries(state.overrides.agents),
|
|
254
|
+
defaults: state.overrides.defaults,
|
|
197
255
|
})
|
|
198
256
|
},
|
|
199
257
|
})
|
|
@@ -13,7 +13,7 @@ import { selectPluginState } from '~/core/sessions/reducer.js'
|
|
|
13
13
|
import type { SessionId } from '~/core/sessions/schema.js'
|
|
14
14
|
import { SessionManager } from '~/core/sessions/session-manager.js'
|
|
15
15
|
import type { Session } from '~/core/sessions/session.js'
|
|
16
|
-
import type { SessionState } from '~/core/sessions/state.js'
|
|
16
|
+
import type { SessionOverridesPatch, SessionState } from '~/core/sessions/state.js'
|
|
17
17
|
import { ToolExecutor } from '~/core/tools/executor.js'
|
|
18
18
|
import { silentLogger } from '~/lib/logger/logger.js'
|
|
19
19
|
import { createNodePlatform } from './node-platform.js'
|
|
@@ -145,14 +145,25 @@ export class TestHarness {
|
|
|
145
145
|
/**
|
|
146
146
|
* Create a session and return a TestSession wrapper.
|
|
147
147
|
*/
|
|
148
|
-
async createSession(
|
|
149
|
-
|
|
148
|
+
async createSession(
|
|
149
|
+
presetId: string,
|
|
150
|
+
options?: { workspaceDir?: string; overrides?: SessionOverridesPatch },
|
|
151
|
+
): Promise<TestSession> {
|
|
152
|
+
const result = await this.sessionManager.createSession(presetId, options)
|
|
150
153
|
if (!result.ok) {
|
|
151
154
|
throw new Error(`Failed to create session: ${result.error.type} — ${result.error.message}`)
|
|
152
155
|
}
|
|
153
156
|
return new TestSession(result.value, this)
|
|
154
157
|
}
|
|
155
158
|
|
|
159
|
+
/** Create a session expecting failure, returning the error instead of throwing. */
|
|
160
|
+
async createSessionResult(
|
|
161
|
+
presetId: string,
|
|
162
|
+
options?: { workspaceDir?: string; overrides?: SessionOverridesPatch },
|
|
163
|
+
): Promise<Result<Session, DomainError>> {
|
|
164
|
+
return this.sessionManager.createSession(presetId, options)
|
|
165
|
+
}
|
|
166
|
+
|
|
156
167
|
/**
|
|
157
168
|
* Open an existing session from the event store (replays events, runs onSessionReady hooks).
|
|
158
169
|
* Used to simulate server restart by pairing with another harness over the same eventStore.
|
|
@@ -218,6 +229,14 @@ export class TestSession {
|
|
|
218
229
|
await waitForAllAgentsIdle(this.session, opts)
|
|
219
230
|
}
|
|
220
231
|
|
|
232
|
+
/** Patch session overrides (see Session.setOverrides). */
|
|
233
|
+
async setOverrides(patch: SessionOverridesPatch): Promise<void> {
|
|
234
|
+
const result = await this.session.setOverrides(patch)
|
|
235
|
+
if (!result.ok) {
|
|
236
|
+
throw new Error(`setOverrides failed: ${result.error.type} — ${result.error.message}`)
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
221
240
|
/**
|
|
222
241
|
* Send a message and wait for all agents to become idle.
|
|
223
242
|
*/
|