@roj-ai/sdk 0.1.34 → 0.1.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap.d.ts +36 -0
- package/dist/bootstrap.d.ts.map +1 -1
- package/dist/core/agents/agent.d.ts +11 -0
- package/dist/core/agents/agent.d.ts.map +1 -1
- package/dist/core/agents/agent.js +26 -5
- package/dist/core/agents/agent.js.map +1 -1
- package/dist/core/agents/state.d.ts +24 -0
- package/dist/core/agents/state.d.ts.map +1 -1
- package/dist/core/agents/state.js.map +1 -1
- package/dist/core/context/state.d.ts.map +1 -1
- package/dist/core/context/state.js +4 -0
- package/dist/core/context/state.js.map +1 -1
- package/dist/core/llm/anthropic.d.ts.map +1 -1
- package/dist/core/llm/anthropic.js +49 -8
- package/dist/core/llm/anthropic.js.map +1 -1
- package/dist/core/llm/model-shortlist-live.test.d.ts +35 -0
- package/dist/core/llm/model-shortlist-live.test.d.ts.map +1 -0
- package/dist/core/llm/model-shortlist-live.test.js +452 -0
- package/dist/core/llm/model-shortlist-live.test.js.map +1 -0
- package/dist/core/llm/openrouter-mapping.test.js +18 -0
- package/dist/core/llm/openrouter-mapping.test.js.map +1 -1
- package/dist/core/llm/openrouter.d.ts.map +1 -1
- package/dist/core/llm/openrouter.js +20 -11
- package/dist/core/llm/openrouter.js.map +1 -1
- package/dist/core/llm/provider.d.ts +34 -3
- package/dist/core/llm/provider.d.ts.map +1 -1
- package/dist/core/llm/provider.js.map +1 -1
- package/dist/core/llm/reasoning-details.test.d.ts +2 -0
- package/dist/core/llm/reasoning-details.test.d.ts.map +1 -0
- package/dist/core/llm/reasoning-details.test.js +144 -0
- package/dist/core/llm/reasoning-details.test.js.map +1 -0
- package/dist/core/llm/state.d.ts +2 -0
- package/dist/core/llm/state.d.ts.map +1 -1
- package/dist/core/llm/state.js +11 -0
- package/dist/core/llm/state.js.map +1 -1
- package/dist/core/llm/thinking-blocks.test.d.ts +2 -0
- package/dist/core/llm/thinking-blocks.test.d.ts.map +1 -0
- package/dist/core/llm/thinking-blocks.test.js +291 -0
- package/dist/core/llm/thinking-blocks.test.js.map +1 -0
- package/dist/core/preset/index.d.ts +1 -0
- package/dist/core/preset/index.d.ts.map +1 -1
- package/dist/core/preset/index.js +2 -0
- package/dist/core/preset/index.js.map +1 -1
- package/dist/core/preset/overrides.d.ts +14 -0
- package/dist/core/preset/overrides.d.ts.map +1 -0
- package/dist/core/preset/overrides.js +20 -0
- package/dist/core/preset/overrides.js.map +1 -0
- package/dist/core/reasoning-round-trip.integration.test.d.ts +2 -0
- package/dist/core/reasoning-round-trip.integration.test.d.ts.map +1 -0
- package/dist/core/reasoning-round-trip.integration.test.js +150 -0
- package/dist/core/reasoning-round-trip.integration.test.js.map +1 -0
- package/dist/core/sessions/session-manager.d.ts +2 -0
- package/dist/core/sessions/session-manager.d.ts.map +1 -1
- package/dist/core/sessions/session-manager.js +17 -5
- package/dist/core/sessions/session-manager.js.map +1 -1
- package/dist/core/sessions/session-overrides.integration.test.d.ts +2 -0
- package/dist/core/sessions/session-overrides.integration.test.d.ts.map +1 -0
- package/dist/core/sessions/session-overrides.integration.test.js +154 -0
- package/dist/core/sessions/session-overrides.integration.test.js.map +1 -0
- package/dist/core/sessions/session.d.ts +10 -1
- package/dist/core/sessions/session.d.ts.map +1 -1
- package/dist/core/sessions/session.js +15 -0
- package/dist/core/sessions/session.js.map +1 -1
- package/dist/core/sessions/state.d.ts +57 -0
- package/dist/core/sessions/state.d.ts.map +1 -1
- package/dist/core/sessions/state.js +53 -0
- package/dist/core/sessions/state.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/plugins/services/plugin.d.ts +3 -0
- package/dist/plugins/services/plugin.d.ts.map +1 -1
- package/dist/plugins/services/plugin.js +50 -14
- package/dist/plugins/services/plugin.js.map +1 -1
- package/dist/plugins/services/prompt.d.ts.map +1 -1
- package/dist/plugins/services/prompt.js +8 -0
- package/dist/plugins/services/prompt.js.map +1 -1
- package/dist/plugins/services/schema.d.ts +33 -0
- package/dist/plugins/services/schema.d.ts.map +1 -1
- package/dist/plugins/services/service.d.ts +20 -0
- package/dist/plugins/services/service.d.ts.map +1 -1
- package/dist/plugins/services/service.js +88 -1
- package/dist/plugins/services/service.js.map +1 -1
- package/dist/plugins/services/services.integration.test.js +254 -0
- package/dist/plugins/services/services.integration.test.js.map +1 -1
- package/dist/plugins/session-lifecycle/plugin.d.ts +36 -0
- package/dist/plugins/session-lifecycle/plugin.d.ts.map +1 -1
- package/dist/plugins/session-lifecycle/plugin.js +53 -1
- package/dist/plugins/session-lifecycle/plugin.js.map +1 -1
- package/dist/testing/test-harness.d.ts +12 -2
- package/dist/testing/test-harness.d.ts.map +1 -1
- package/dist/testing/test-harness.js +13 -2
- package/dist/testing/test-harness.js.map +1 -1
- package/package.json +3 -3
- package/src/core/agents/agent.ts +28 -5
- package/src/core/agents/state.ts +26 -0
- package/src/core/context/state.ts +4 -0
- package/src/core/llm/anthropic.ts +71 -10
- package/src/core/llm/model-shortlist-live.test.ts +577 -0
- package/src/core/llm/openrouter-mapping.test.ts +24 -0
- package/src/core/llm/openrouter.ts +28 -12
- package/src/core/llm/provider.ts +44 -3
- package/src/core/llm/reasoning-details.test.ts +177 -0
- package/src/core/llm/state.ts +12 -0
- package/src/core/llm/thinking-blocks.test.ts +345 -0
- package/src/core/preset/index.ts +3 -0
- package/src/core/preset/overrides.ts +24 -0
- package/src/core/reasoning-round-trip.integration.test.ts +188 -0
- package/src/core/sessions/session-manager.ts +22 -3
- package/src/core/sessions/session-overrides.integration.test.ts +210 -0
- package/src/core/sessions/session.ts +19 -1
- package/src/core/sessions/state.ts +82 -0
- package/src/index.ts +9 -1
- package/src/plugins/services/plugin.ts +50 -14
- package/src/plugins/services/prompt.ts +8 -0
- package/src/plugins/services/schema.ts +33 -0
- package/src/plugins/services/service.ts +113 -1
- package/src/plugins/services/services.integration.test.ts +270 -0
- package/src/plugins/session-lifecycle/plugin.ts +59 -1
- package/src/testing/test-harness.ts +22 -3
|
@@ -70,6 +70,17 @@ export interface ServiceStatusChangeDetails {
|
|
|
70
70
|
pidStartTime?: number
|
|
71
71
|
cwd?: string
|
|
72
72
|
command?: string
|
|
73
|
+
/** Set on a `failed` change that already has a revival queued — see ServiceEntry.restartAt. */
|
|
74
|
+
restartAt?: number
|
|
75
|
+
restartAttempt?: number
|
|
76
|
+
restartMaxRetries?: number
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** A revival the policy has queued, as reported on the `failed` status change. */
|
|
80
|
+
interface ScheduledRestart {
|
|
81
|
+
restartAt: number
|
|
82
|
+
restartAttempt: number
|
|
83
|
+
restartMaxRetries: number
|
|
73
84
|
}
|
|
74
85
|
|
|
75
86
|
/** Matches a port bind-conflict across the Node and Bun runtimes. */
|
|
@@ -83,6 +94,12 @@ const PORT_CONFLICT_PATTERN = /EADDRINUSE|address already in use/i
|
|
|
83
94
|
*/
|
|
84
95
|
const MAX_PORT_CONFLICT_RETRIES = 3
|
|
85
96
|
|
|
97
|
+
/** `restartPolicy` defaults — see ServiceConfig.restartPolicy for the rationale. */
|
|
98
|
+
const DEFAULT_MAX_RESTART_RETRIES = 3
|
|
99
|
+
const DEFAULT_RESTART_DELAY_MS = 1000
|
|
100
|
+
const DEFAULT_MAX_RESTART_DELAY_MS = 30_000
|
|
101
|
+
const DEFAULT_RESTART_HEALTHY_AFTER_MS = 60_000
|
|
102
|
+
|
|
86
103
|
// ============================================================================
|
|
87
104
|
// ServiceExecutor
|
|
88
105
|
// ============================================================================
|
|
@@ -102,6 +119,10 @@ export class ServiceExecutor {
|
|
|
102
119
|
private readonly startInFlight = new Map<string, Promise<Result<void, ToolError>>>()
|
|
103
120
|
/** Per-type counter bounding automatic EADDRINUSE port re-allocation retries. */
|
|
104
121
|
private readonly portConflictRetries = new Map<string, number>()
|
|
122
|
+
/** Per-type counter bounding `restartPolicy` revivals after an unexpected exit. */
|
|
123
|
+
private readonly restartRetries = new Map<string, number>()
|
|
124
|
+
/** Pending `restartPolicy` timers, cancelled when someone stops or starts the service by hand. */
|
|
125
|
+
private readonly restartTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
|
105
126
|
private readonly logger: Logger
|
|
106
127
|
private readonly portPool: PortPool
|
|
107
128
|
private readonly fs: FileSystem
|
|
@@ -254,6 +275,9 @@ export class ServiceExecutor {
|
|
|
254
275
|
workspaceDir?: string,
|
|
255
276
|
preferredPort?: number,
|
|
256
277
|
): Promise<Result<void, ToolError>> {
|
|
278
|
+
// This start supersedes any revival the policy had queued.
|
|
279
|
+
this.cancelPendingRestart(config.type)
|
|
280
|
+
|
|
257
281
|
const existing = this.services.get(config.type)
|
|
258
282
|
if (existing && (existing.status === 'starting' || existing.status === 'ready')) {
|
|
259
283
|
return Ok(undefined)
|
|
@@ -567,17 +591,21 @@ export class ServiceExecutor {
|
|
|
567
591
|
// Unexpected exit
|
|
568
592
|
current.status = 'failed'
|
|
569
593
|
const errorMsg = `Process exited unexpectedly with code ${code}`
|
|
594
|
+
this.portConflictRetries.delete(config.type)
|
|
595
|
+
// Queue the revival first so the `failed` change can carry it — a
|
|
596
|
+
// consumer that sees a bare `failed` is entitled to treat it as terminal.
|
|
597
|
+
const revival = this.scheduleRestart(config, sessionId, workspaceDir, Date.now() - startTime, code)
|
|
570
598
|
this.notifyStatusChanged(sessionId, config.type, 'failed', {
|
|
571
599
|
port: current.port,
|
|
572
600
|
cwd: current.cwd,
|
|
573
601
|
command: current.command,
|
|
574
602
|
error: errorMsg,
|
|
603
|
+
...revival,
|
|
575
604
|
})
|
|
576
605
|
this.logger.warn('Service process exited unexpectedly', {
|
|
577
606
|
serviceType: config.type,
|
|
578
607
|
code,
|
|
579
608
|
})
|
|
580
|
-
this.portConflictRetries.delete(config.type)
|
|
581
609
|
}
|
|
582
610
|
})
|
|
583
611
|
|
|
@@ -599,16 +627,96 @@ export class ServiceExecutor {
|
|
|
599
627
|
return Ok(undefined)
|
|
600
628
|
}
|
|
601
629
|
|
|
630
|
+
/**
|
|
631
|
+
* Bring a crashed service back if its `restartPolicy` allows it. The port
|
|
632
|
+
* allocation is kept on purpose — the preview URL has to survive the bounce.
|
|
633
|
+
*
|
|
634
|
+
* Returns the queued revival so the caller can put it on the `failed` status
|
|
635
|
+
* change, or undefined when the service is left failed for good.
|
|
636
|
+
*/
|
|
637
|
+
private scheduleRestart(
|
|
638
|
+
config: ServiceConfig,
|
|
639
|
+
sessionId: SessionId,
|
|
640
|
+
workspaceDir: string | undefined,
|
|
641
|
+
uptimeMs: number,
|
|
642
|
+
exitCode: number | null,
|
|
643
|
+
): ScheduledRestart | undefined {
|
|
644
|
+
const policy = config.restartPolicy
|
|
645
|
+
if (!policy) return undefined
|
|
646
|
+
|
|
647
|
+
// A service that ran for a while and then died is a new problem, not the
|
|
648
|
+
// continuation of a boot loop — give it a full budget again.
|
|
649
|
+
if (uptimeMs >= (policy.healthyAfterMs ?? DEFAULT_RESTART_HEALTHY_AFTER_MS)) {
|
|
650
|
+
this.restartRetries.delete(config.type)
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
const attempt = this.restartRetries.get(config.type) ?? 0
|
|
654
|
+
const maxRetries = policy.maxRetries ?? DEFAULT_MAX_RESTART_RETRIES
|
|
655
|
+
if (attempt >= maxRetries) {
|
|
656
|
+
this.logger.warn('Service left failed — automatic restart budget spent', {
|
|
657
|
+
serviceType: config.type,
|
|
658
|
+
attempts: attempt,
|
|
659
|
+
exitCode,
|
|
660
|
+
})
|
|
661
|
+
this.restartRetries.delete(config.type)
|
|
662
|
+
return undefined
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
const initialDelayMs = policy.initialDelayMs ?? DEFAULT_RESTART_DELAY_MS
|
|
666
|
+
const delayMs = Math.min(initialDelayMs * 2 ** attempt, policy.maxDelayMs ?? DEFAULT_MAX_RESTART_DELAY_MS)
|
|
667
|
+
this.restartRetries.set(config.type, attempt + 1)
|
|
668
|
+
this.logger.info('Restarting service after unexpected exit', {
|
|
669
|
+
serviceType: config.type,
|
|
670
|
+
attempt: attempt + 1,
|
|
671
|
+
maxRetries,
|
|
672
|
+
delayMs,
|
|
673
|
+
exitCode,
|
|
674
|
+
})
|
|
675
|
+
|
|
676
|
+
const timer = setTimeout(() => {
|
|
677
|
+
this.restartTimers.delete(config.type)
|
|
678
|
+
void this.start(config, sessionId, workspaceDir)
|
|
679
|
+
}, delayMs)
|
|
680
|
+
// A pending revival must not keep the process alive on its own.
|
|
681
|
+
timer.unref?.()
|
|
682
|
+
this.restartTimers.set(config.type, timer)
|
|
683
|
+
|
|
684
|
+
return { restartAt: Date.now() + delayMs, restartAttempt: attempt + 1, restartMaxRetries: maxRetries }
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/** Drop a pending automatic restart — an explicit start or stop wins over it. Returns true if one was queued. */
|
|
688
|
+
private cancelPendingRestart(serviceType: string): boolean {
|
|
689
|
+
const timer = this.restartTimers.get(serviceType)
|
|
690
|
+
if (!timer) return false
|
|
691
|
+
clearTimeout(timer)
|
|
692
|
+
this.restartTimers.delete(serviceType)
|
|
693
|
+
return true
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/** Whether a `restartPolicy` revival is queued — the service is down but coming back. */
|
|
697
|
+
hasScheduledRestart(serviceType: string): boolean {
|
|
698
|
+
return this.restartTimers.has(serviceType)
|
|
699
|
+
}
|
|
700
|
+
|
|
602
701
|
/**
|
|
603
702
|
* Stop a running service gracefully.
|
|
604
703
|
* Port is NOT released — kept for session-level stability across restarts.
|
|
605
704
|
*/
|
|
606
705
|
async stop(serviceType: string, sessionId: SessionId): Promise<Result<void, ToolError>> {
|
|
706
|
+
const hadPendingRestart = this.cancelPendingRestart(serviceType)
|
|
707
|
+
this.restartRetries.delete(serviceType)
|
|
607
708
|
const entry = this.services.get(serviceType)
|
|
608
709
|
if (!entry) {
|
|
609
710
|
return Err({ message: `Service '${serviceType}' not found`, recoverable: false })
|
|
610
711
|
}
|
|
611
712
|
if (entry.status !== 'starting' && entry.status !== 'ready' && entry.status !== 'paused') {
|
|
713
|
+
// A failed service with a revival queued is really "about to restart",
|
|
714
|
+
// and calling that off is a legitimate stop rather than an error.
|
|
715
|
+
if (hadPendingRestart) {
|
|
716
|
+
entry.status = 'stopped'
|
|
717
|
+
this.notifyStatusChanged(sessionId, serviceType, 'stopped')
|
|
718
|
+
return Ok(undefined)
|
|
719
|
+
}
|
|
612
720
|
return Err({ message: `Service '${serviceType}' is ${entry.status}, cannot stop`, recoverable: false })
|
|
613
721
|
}
|
|
614
722
|
|
|
@@ -763,6 +871,10 @@ export class ServiceExecutor {
|
|
|
763
871
|
* Called on session close.
|
|
764
872
|
*/
|
|
765
873
|
async shutdown(): Promise<void> {
|
|
874
|
+
// Nothing queued may spawn after the executor is gone.
|
|
875
|
+
for (const serviceType of [...this.restartTimers.keys()]) this.cancelPendingRestart(serviceType)
|
|
876
|
+
this.restartRetries.clear()
|
|
877
|
+
|
|
766
878
|
const promises: Promise<void>[] = []
|
|
767
879
|
|
|
768
880
|
for (const [serviceType, entry] of this.services) {
|
|
@@ -15,6 +15,7 @@ import type { ServiceAgentConfig, ServicePluginConfig } from './plugin.js'
|
|
|
15
15
|
import { PortPool } from './port-pool.js'
|
|
16
16
|
import { buildServiceStatusMessage } from './prompt.js'
|
|
17
17
|
import type { ServiceCommandArgs, ServiceConfig, ServiceCwdArgs, ServiceEntry, ServiceStatus } from './schema.js'
|
|
18
|
+
import type { ServiceStatusChangeDetails } from './service.js'
|
|
18
19
|
import { ServiceExecutor } from './service.js'
|
|
19
20
|
|
|
20
21
|
// ============================================================================
|
|
@@ -823,6 +824,275 @@ describe('services plugin', () => {
|
|
|
823
824
|
})
|
|
824
825
|
})
|
|
825
826
|
|
|
827
|
+
// =========================================================================
|
|
828
|
+
// restartPolicy — reviving a service that exited on its own
|
|
829
|
+
// =========================================================================
|
|
830
|
+
|
|
831
|
+
describe('restartPolicy', () => {
|
|
832
|
+
const waitUntil = async (predicate: () => boolean, timeoutMs = 8000): Promise<void> => {
|
|
833
|
+
const deadline = Date.now() + timeoutMs
|
|
834
|
+
while (Date.now() < deadline) {
|
|
835
|
+
if (predicate()) return
|
|
836
|
+
await new Promise((r) => setTimeout(r, 20))
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
const waitUntilAsync = async (predicate: () => Promise<boolean>, timeoutMs = 8000): Promise<void> => {
|
|
841
|
+
const deadline = Date.now() + timeoutMs
|
|
842
|
+
while (Date.now() < deadline) {
|
|
843
|
+
if (await predicate()) return
|
|
844
|
+
await new Promise((r) => setTimeout(r, 20))
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
const marker = (name: string) => join(tmpdir(), `roj-restart-${name}-${Date.now()}-${Math.floor(Math.random() * 1e6)}`)
|
|
849
|
+
/** Records every spawn, then crashes or stays up depending on the run count. */
|
|
850
|
+
const crashCommand = (path: string, crashRuns: number) =>
|
|
851
|
+
`echo x >> ${path}; runs=$(wc -l < ${path} | tr -d ' '); if [ "$runs" -le ${crashRuns} ]; then exit 1; fi; echo "listening READY"; sleep 30`
|
|
852
|
+
const spawnCount = async (path: string): Promise<number> => {
|
|
853
|
+
try {
|
|
854
|
+
return (await readFile(path, 'utf-8')).split('\n').filter((l) => l.length > 0).length
|
|
855
|
+
} catch {
|
|
856
|
+
return 0
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
it('revives a crashed service on the port it already holds', async () => {
|
|
861
|
+
const platform = createNodePlatform()
|
|
862
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
863
|
+
const path = marker('revive')
|
|
864
|
+
const config: ServiceConfig = {
|
|
865
|
+
type: 'revive-svc',
|
|
866
|
+
description: 'crashes once, then stays up',
|
|
867
|
+
command: crashCommand(path, 1),
|
|
868
|
+
readyPattern: 'READY',
|
|
869
|
+
startupTimeoutMs: 5000,
|
|
870
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 20 },
|
|
871
|
+
}
|
|
872
|
+
const ports: number[] = []
|
|
873
|
+
executor.onStatusChanged = (_s, _t, status, details) => {
|
|
874
|
+
if (status === 'starting' && details.port !== undefined) ports.push(details.port)
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
try {
|
|
878
|
+
await executor.start(config, SessionId('s-revive'))
|
|
879
|
+
await waitUntil(() => executor.getStatus('revive-svc') === 'ready')
|
|
880
|
+
expect(executor.getStatus('revive-svc')).toBe('ready')
|
|
881
|
+
expect(await spawnCount(path)).toBe(2)
|
|
882
|
+
// The preview URL must survive the bounce.
|
|
883
|
+
expect(new Set(ports).size).toBe(1)
|
|
884
|
+
} finally {
|
|
885
|
+
await executor.shutdown()
|
|
886
|
+
await rm(path, { force: true })
|
|
887
|
+
}
|
|
888
|
+
})
|
|
889
|
+
|
|
890
|
+
it('leaves the service failed once the retry budget is spent', async () => {
|
|
891
|
+
const platform = createNodePlatform()
|
|
892
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
893
|
+
const path = marker('budget')
|
|
894
|
+
const config: ServiceConfig = {
|
|
895
|
+
type: 'budget-svc',
|
|
896
|
+
description: 'always crashes',
|
|
897
|
+
command: crashCommand(path, 99),
|
|
898
|
+
readyPattern: 'READY',
|
|
899
|
+
startupTimeoutMs: 5000,
|
|
900
|
+
restartPolicy: { maxRetries: 2, initialDelayMs: 20 },
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
try {
|
|
904
|
+
await executor.start(config, SessionId('s-budget'))
|
|
905
|
+
// One initial spawn plus two retries, then it stays down.
|
|
906
|
+
await waitUntilAsync(async () => (await spawnCount(path)) >= 3 && executor.getStatus('budget-svc') === 'failed')
|
|
907
|
+
await new Promise((r) => setTimeout(r, 300))
|
|
908
|
+
expect(await spawnCount(path)).toBe(3)
|
|
909
|
+
expect(executor.getStatus('budget-svc')).toBe('failed')
|
|
910
|
+
} finally {
|
|
911
|
+
await executor.shutdown()
|
|
912
|
+
await rm(path, { force: true })
|
|
913
|
+
}
|
|
914
|
+
})
|
|
915
|
+
|
|
916
|
+
it('without a policy a crash stays a crash', async () => {
|
|
917
|
+
const platform = createNodePlatform()
|
|
918
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
919
|
+
const path = marker('nopolicy')
|
|
920
|
+
const config: ServiceConfig = {
|
|
921
|
+
type: 'nopolicy-svc',
|
|
922
|
+
description: 'always crashes, no restartPolicy',
|
|
923
|
+
command: crashCommand(path, 99),
|
|
924
|
+
readyPattern: 'READY',
|
|
925
|
+
startupTimeoutMs: 5000,
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
try {
|
|
929
|
+
await executor.start(config, SessionId('s-nopolicy'))
|
|
930
|
+
await waitUntil(() => executor.getStatus('nopolicy-svc') === 'failed')
|
|
931
|
+
await new Promise((r) => setTimeout(r, 300))
|
|
932
|
+
expect(await spawnCount(path)).toBe(1)
|
|
933
|
+
} finally {
|
|
934
|
+
await executor.shutdown()
|
|
935
|
+
await rm(path, { force: true })
|
|
936
|
+
}
|
|
937
|
+
})
|
|
938
|
+
|
|
939
|
+
it('a queued revival rides along on the failed status change', async () => {
|
|
940
|
+
const platform = createNodePlatform()
|
|
941
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
942
|
+
const path = marker('event')
|
|
943
|
+
const config: ServiceConfig = {
|
|
944
|
+
type: 'event-svc',
|
|
945
|
+
description: 'crashes once, then stays up',
|
|
946
|
+
command: crashCommand(path, 1),
|
|
947
|
+
readyPattern: 'READY',
|
|
948
|
+
startupTimeoutMs: 5000,
|
|
949
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 20 },
|
|
950
|
+
}
|
|
951
|
+
const changes: Array<{ status: ServiceStatus; details: ServiceStatusChangeDetails }> = []
|
|
952
|
+
executor.onStatusChanged = (_s, _t, status, details) => {
|
|
953
|
+
changes.push({ status, details })
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
try {
|
|
957
|
+
await executor.start(config, SessionId('s-event'))
|
|
958
|
+
await waitUntil(() => executor.getStatus('event-svc') === 'ready')
|
|
959
|
+
|
|
960
|
+
const failedIndex = changes.findIndex((c) => c.status === 'failed')
|
|
961
|
+
expect(failedIndex).toBeGreaterThanOrEqual(0)
|
|
962
|
+
const failed = changes[failedIndex]!
|
|
963
|
+
expect(failed.details.restartAttempt).toBe(1)
|
|
964
|
+
expect(failed.details.restartMaxRetries).toBe(3)
|
|
965
|
+
expect(failed.details.restartAt).toBeGreaterThan(0)
|
|
966
|
+
|
|
967
|
+
// The revival's own start must not carry the intent it just honoured.
|
|
968
|
+
const revivalStart = changes.slice(failedIndex + 1).find((c) => c.status === 'starting')
|
|
969
|
+
expect(revivalStart).toBeDefined()
|
|
970
|
+
expect(revivalStart!.details.restartAt).toBeUndefined()
|
|
971
|
+
} finally {
|
|
972
|
+
await executor.shutdown()
|
|
973
|
+
await rm(path, { force: true })
|
|
974
|
+
}
|
|
975
|
+
})
|
|
976
|
+
|
|
977
|
+
it('the failure that spends the budget carries no revival', async () => {
|
|
978
|
+
const platform = createNodePlatform()
|
|
979
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
980
|
+
const path = marker('terminal')
|
|
981
|
+
const config: ServiceConfig = {
|
|
982
|
+
type: 'terminal-svc',
|
|
983
|
+
description: 'always crashes',
|
|
984
|
+
command: crashCommand(path, 99),
|
|
985
|
+
readyPattern: 'READY',
|
|
986
|
+
startupTimeoutMs: 5000,
|
|
987
|
+
restartPolicy: { maxRetries: 1, initialDelayMs: 20 },
|
|
988
|
+
}
|
|
989
|
+
const failures: ServiceStatusChangeDetails[] = []
|
|
990
|
+
executor.onStatusChanged = (_s, _t, status, details) => {
|
|
991
|
+
if (status === 'failed') failures.push(details)
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
try {
|
|
995
|
+
await executor.start(config, SessionId('s-terminal'))
|
|
996
|
+
await waitUntilAsync(async () => (await spawnCount(path)) >= 2 && failures.length >= 2)
|
|
997
|
+
expect(failures[0]?.restartAttempt).toBe(1)
|
|
998
|
+
expect(failures[1]?.restartAt).toBeUndefined()
|
|
999
|
+
expect(failures[1]?.restartAttempt).toBeUndefined()
|
|
1000
|
+
} finally {
|
|
1001
|
+
await executor.shutdown()
|
|
1002
|
+
await rm(path, { force: true })
|
|
1003
|
+
}
|
|
1004
|
+
})
|
|
1005
|
+
|
|
1006
|
+
it('session state shows the queued revival and drops it once called off', async () => {
|
|
1007
|
+
const portPool = new PortPool()
|
|
1008
|
+
const config: ServiceConfig = {
|
|
1009
|
+
type: 'reviving',
|
|
1010
|
+
description: 'crashes early, revival queued behind a long delay',
|
|
1011
|
+
// The sleep keeps the exit from beating the executor's close listener.
|
|
1012
|
+
command: 'sleep 0.2; exit 1',
|
|
1013
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 3000 },
|
|
1014
|
+
}
|
|
1015
|
+
const harness = createServicesHarness({
|
|
1016
|
+
presets: [createServicesPreset([config], ['reviving'], portPool)],
|
|
1017
|
+
})
|
|
1018
|
+
const session = await harness.createSession('test')
|
|
1019
|
+
|
|
1020
|
+
await session.callPluginMethod('services.start', { serviceType: 'reviving' })
|
|
1021
|
+
await waitForServiceStateStatus(session, 'reviving', 'failed')
|
|
1022
|
+
|
|
1023
|
+
const entry = selectPluginState<Map<string, ServiceEntry>>(session.state, 'services')?.get('reviving')
|
|
1024
|
+
expect(entry?.status).toBe('failed')
|
|
1025
|
+
expect(entry?.restartAt).toBeGreaterThan(Date.now())
|
|
1026
|
+
expect(entry?.restartAttempt).toBe(1)
|
|
1027
|
+
expect(entry?.restartMaxRetries).toBe(3)
|
|
1028
|
+
|
|
1029
|
+
const stopped = await session.callPluginMethod('services.stop', { serviceType: 'reviving' })
|
|
1030
|
+
expect(stopped.ok).toBe(true)
|
|
1031
|
+
await waitForServiceStateStatus(session, 'reviving', 'stopped')
|
|
1032
|
+
|
|
1033
|
+
const after = selectPluginState<Map<string, ServiceEntry>>(session.state, 'services')?.get('reviving')
|
|
1034
|
+
expect(after?.status).toBe('stopped')
|
|
1035
|
+
expect(after?.restartAt).toBeUndefined()
|
|
1036
|
+
expect(after?.restartAttempt).toBeUndefined()
|
|
1037
|
+
})
|
|
1038
|
+
|
|
1039
|
+
it('closing the session calls off a queued revival', async () => {
|
|
1040
|
+
const portPool = new PortPool()
|
|
1041
|
+
const path = marker('close')
|
|
1042
|
+
const config: ServiceConfig = {
|
|
1043
|
+
type: 'close-svc',
|
|
1044
|
+
description: 'crashes early, revival queued behind a delay',
|
|
1045
|
+
// The sleep keeps the exit from beating the executor's close listener.
|
|
1046
|
+
command: `echo x >> ${path}; sleep 0.2; exit 1`,
|
|
1047
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 800 },
|
|
1048
|
+
}
|
|
1049
|
+
const harness = createServicesHarness({
|
|
1050
|
+
presets: [createServicesPreset([config], ['close-svc'], portPool)],
|
|
1051
|
+
})
|
|
1052
|
+
const session = await harness.createSession('test')
|
|
1053
|
+
|
|
1054
|
+
try {
|
|
1055
|
+
await session.callPluginMethod('services.start', { serviceType: 'close-svc' })
|
|
1056
|
+
await waitForServiceStateStatus(session, 'close-svc', 'failed')
|
|
1057
|
+
expect(await spawnCount(path)).toBe(1)
|
|
1058
|
+
|
|
1059
|
+
await session.close()
|
|
1060
|
+
// Long enough for the revival to have fired had nothing called it off.
|
|
1061
|
+
await new Promise((r) => setTimeout(r, 1500))
|
|
1062
|
+
expect(await spawnCount(path)).toBe(1)
|
|
1063
|
+
} finally {
|
|
1064
|
+
await rm(path, { force: true })
|
|
1065
|
+
}
|
|
1066
|
+
})
|
|
1067
|
+
|
|
1068
|
+
it('an explicit stop calls off a queued revival', async () => {
|
|
1069
|
+
const platform = createNodePlatform()
|
|
1070
|
+
const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process })
|
|
1071
|
+
const path = marker('stop')
|
|
1072
|
+
const config: ServiceConfig = {
|
|
1073
|
+
type: 'stop-svc',
|
|
1074
|
+
description: 'always crashes, revival queued behind a long delay',
|
|
1075
|
+
command: crashCommand(path, 99),
|
|
1076
|
+
readyPattern: 'READY',
|
|
1077
|
+
startupTimeoutMs: 5000,
|
|
1078
|
+
restartPolicy: { maxRetries: 3, initialDelayMs: 1500 },
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
try {
|
|
1082
|
+
await executor.start(config, SessionId('s-stop'))
|
|
1083
|
+
await waitUntil(() => executor.getStatus('stop-svc') === 'failed')
|
|
1084
|
+
const stopped = await executor.stop('stop-svc', SessionId('s-stop'))
|
|
1085
|
+
expect(stopped.ok).toBe(true)
|
|
1086
|
+
expect(executor.getStatus('stop-svc')).toBe('stopped')
|
|
1087
|
+
await new Promise((r) => setTimeout(r, 2000))
|
|
1088
|
+
expect(await spawnCount(path)).toBe(1)
|
|
1089
|
+
} finally {
|
|
1090
|
+
await executor.shutdown()
|
|
1091
|
+
await rm(path, { force: true })
|
|
1092
|
+
}
|
|
1093
|
+
})
|
|
1094
|
+
})
|
|
1095
|
+
|
|
826
1096
|
// =========================================================================
|
|
827
1097
|
// agentVisible: false — infrastructure services hidden from agents
|
|
828
1098
|
// =========================================================================
|
|
@@ -11,8 +11,9 @@ import { agentIdSchema } from '~/core/agents'
|
|
|
11
11
|
import { COMMUNICATOR_ROLE, ORCHESTRATOR_ROLE } from '~/core/agents/agent-roles.js'
|
|
12
12
|
import { type DomainError, PresetErrors, SessionErrors, ValidationErrors } from '~/core/errors.js'
|
|
13
13
|
import { definePlugin } from '~/core/plugins/index.js'
|
|
14
|
+
import { knownDefinitionNames, unknownOverrideTargets } from '~/core/preset/overrides.js'
|
|
14
15
|
import { SessionId, sessionIdSchema } from '~/core/sessions/schema.js'
|
|
15
|
-
import { getEntryAgentId, sessionEvents } from '~/core/sessions/state.js'
|
|
16
|
+
import { agentOverridesSchema, getEntryAgentId, sessionEvents, sessionOverridesPatchSchema } from '~/core/sessions/state.js'
|
|
16
17
|
import { Err, Ok } from '~/lib/utils/result.js'
|
|
17
18
|
|
|
18
19
|
// ============================================================================
|
|
@@ -115,6 +116,7 @@ export const sessionLifecyclePlugin = definePlugin('sessions')
|
|
|
115
116
|
presetId: z4.string().min(1),
|
|
116
117
|
workspaceDir: z4.string().optional(),
|
|
117
118
|
sessionId: z4.string().optional(),
|
|
119
|
+
overrides: sessionOverridesPatchSchema.optional(),
|
|
118
120
|
}),
|
|
119
121
|
output: z4.object({
|
|
120
122
|
sessionId: sessionIdSchema,
|
|
@@ -126,11 +128,13 @@ export const sessionLifecyclePlugin = definePlugin('sessions')
|
|
|
126
128
|
presetId: input.presetId,
|
|
127
129
|
workspaceDir: input.workspaceDir ?? null,
|
|
128
130
|
sessionId: input.sessionId ?? null,
|
|
131
|
+
overrides: input.overrides ?? null,
|
|
129
132
|
})
|
|
130
133
|
|
|
131
134
|
const sessionResult = await sm.createSession(input.presetId, {
|
|
132
135
|
workspaceDir: input.workspaceDir,
|
|
133
136
|
sessionId: input.sessionId,
|
|
137
|
+
overrides: input.overrides,
|
|
134
138
|
})
|
|
135
139
|
if (!sessionResult.ok) return sessionResult
|
|
136
140
|
|
|
@@ -183,6 +187,12 @@ export const sessionLifecyclePlugin = definePlugin('sessions')
|
|
|
183
187
|
closedAt: z4.number().optional(),
|
|
184
188
|
agentCount: z4.number(),
|
|
185
189
|
entryAgentId: agentIdSchema.nullable(),
|
|
190
|
+
// Reported back so a caller can tell "the override did not apply" from
|
|
191
|
+
// "this build predates overrides and dropped the field on input".
|
|
192
|
+
overrides: z4.object({
|
|
193
|
+
defaults: agentOverridesSchema.optional(),
|
|
194
|
+
agents: z4.record(z4.string(), agentOverridesSchema),
|
|
195
|
+
}),
|
|
186
196
|
}),
|
|
187
197
|
handler: async (ctx) => {
|
|
188
198
|
const state = ctx.sessionState
|
|
@@ -194,6 +204,54 @@ export const sessionLifecyclePlugin = definePlugin('sessions')
|
|
|
194
204
|
closedAt: state.closedAt,
|
|
195
205
|
agentCount: state.agents.size,
|
|
196
206
|
entryAgentId: getEntryAgentId(state),
|
|
207
|
+
overrides: {
|
|
208
|
+
defaults: state.overrides.defaults,
|
|
209
|
+
agents: Object.fromEntries(state.overrides.agents),
|
|
210
|
+
},
|
|
211
|
+
})
|
|
212
|
+
},
|
|
213
|
+
})
|
|
214
|
+
// A manager method, not a session method, purely so it can see the preset and
|
|
215
|
+
// reject unknown agent names — session method contexts carry no preset.
|
|
216
|
+
// `presets.getAgents` is a manager method for the same reason.
|
|
217
|
+
.managerMethod('setOverrides', {
|
|
218
|
+
input: z4.object({
|
|
219
|
+
sessionId: sessionIdSchema,
|
|
220
|
+
overrides: sessionOverridesPatchSchema,
|
|
221
|
+
}),
|
|
222
|
+
output: z4.object({
|
|
223
|
+
agents: z4.record(z4.string(), agentOverridesSchema),
|
|
224
|
+
defaults: agentOverridesSchema.optional(),
|
|
225
|
+
}),
|
|
226
|
+
handler: async (ctx, input) => {
|
|
227
|
+
const sm = ctx.sessionManager
|
|
228
|
+
const sessionId = SessionId(input.sessionId)
|
|
229
|
+
|
|
230
|
+
const sessionResult = await sm.getSession(sessionId)
|
|
231
|
+
if (!sessionResult.ok) return sessionResult
|
|
232
|
+
const session = sessionResult.value
|
|
233
|
+
|
|
234
|
+
const preset = ctx.presets.get(session.state.presetId)
|
|
235
|
+
if (!preset) return Err(PresetErrors.notFound(session.state.presetId))
|
|
236
|
+
|
|
237
|
+
const unknown = unknownOverrideTargets(preset, input.overrides)
|
|
238
|
+
if (unknown.length > 0) {
|
|
239
|
+
return Err(ValidationErrors.invalid(
|
|
240
|
+
`Unknown agent definitions in overrides: ${unknown.join(', ')}. Preset '${preset.id}' defines: ${knownDefinitionNames(preset).join(', ')}`,
|
|
241
|
+
))
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
await session.setOverrides(input.overrides)
|
|
245
|
+
|
|
246
|
+
// Agents already mid-turn finish on the model they started with: the
|
|
247
|
+
// override is read when the next inference request is built, not applied
|
|
248
|
+
// retroactively to one in flight.
|
|
249
|
+
ctx.logger.info('Session overrides set', { sessionId, overrides: input.overrides })
|
|
250
|
+
|
|
251
|
+
const state = session.state
|
|
252
|
+
return Ok({
|
|
253
|
+
agents: Object.fromEntries(state.overrides.agents),
|
|
254
|
+
defaults: state.overrides.defaults,
|
|
197
255
|
})
|
|
198
256
|
},
|
|
199
257
|
})
|
|
@@ -13,7 +13,7 @@ import { selectPluginState } from '~/core/sessions/reducer.js'
|
|
|
13
13
|
import type { SessionId } from '~/core/sessions/schema.js'
|
|
14
14
|
import { SessionManager } from '~/core/sessions/session-manager.js'
|
|
15
15
|
import type { Session } from '~/core/sessions/session.js'
|
|
16
|
-
import type { SessionState } from '~/core/sessions/state.js'
|
|
16
|
+
import type { SessionOverridesPatch, SessionState } from '~/core/sessions/state.js'
|
|
17
17
|
import { ToolExecutor } from '~/core/tools/executor.js'
|
|
18
18
|
import { silentLogger } from '~/lib/logger/logger.js'
|
|
19
19
|
import { createNodePlatform } from './node-platform.js'
|
|
@@ -145,14 +145,25 @@ export class TestHarness {
|
|
|
145
145
|
/**
|
|
146
146
|
* Create a session and return a TestSession wrapper.
|
|
147
147
|
*/
|
|
148
|
-
async createSession(
|
|
149
|
-
|
|
148
|
+
async createSession(
|
|
149
|
+
presetId: string,
|
|
150
|
+
options?: { workspaceDir?: string; overrides?: SessionOverridesPatch },
|
|
151
|
+
): Promise<TestSession> {
|
|
152
|
+
const result = await this.sessionManager.createSession(presetId, options)
|
|
150
153
|
if (!result.ok) {
|
|
151
154
|
throw new Error(`Failed to create session: ${result.error.type} — ${result.error.message}`)
|
|
152
155
|
}
|
|
153
156
|
return new TestSession(result.value, this)
|
|
154
157
|
}
|
|
155
158
|
|
|
159
|
+
/** Create a session expecting failure, returning the error instead of throwing. */
|
|
160
|
+
async createSessionResult(
|
|
161
|
+
presetId: string,
|
|
162
|
+
options?: { workspaceDir?: string; overrides?: SessionOverridesPatch },
|
|
163
|
+
): Promise<Result<Session, DomainError>> {
|
|
164
|
+
return this.sessionManager.createSession(presetId, options)
|
|
165
|
+
}
|
|
166
|
+
|
|
156
167
|
/**
|
|
157
168
|
* Open an existing session from the event store (replays events, runs onSessionReady hooks).
|
|
158
169
|
* Used to simulate server restart by pairing with another harness over the same eventStore.
|
|
@@ -218,6 +229,14 @@ export class TestSession {
|
|
|
218
229
|
await waitForAllAgentsIdle(this.session, opts)
|
|
219
230
|
}
|
|
220
231
|
|
|
232
|
+
/** Patch session overrides (see Session.setOverrides). */
|
|
233
|
+
async setOverrides(patch: SessionOverridesPatch): Promise<void> {
|
|
234
|
+
const result = await this.session.setOverrides(patch)
|
|
235
|
+
if (!result.ok) {
|
|
236
|
+
throw new Error(`setOverrides failed: ${result.error.type} — ${result.error.message}`)
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
221
240
|
/**
|
|
222
241
|
* Send a message and wait for all agents to become idle.
|
|
223
242
|
*/
|