@gotcos/glasses-server 6.77.0 → 6.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +40 -0
- package/CHANGELOG.md +16 -0
- package/README.md +12 -1
- package/managed-runtime-contract.json +9 -1
- package/package.json +10 -1
- package/server/index.ts +92 -9
- package/server/lib/agent-session-binding-registry.ts +623 -9
- package/server/lib/boot-resume.ts +552 -0
- package/server/lib/claude-bridge.ts +152 -3
- package/server/lib/claude-cli-support.ts +39 -0
- package/server/lib/claude-steer.ts +259 -0
- package/server/lib/env-switch.ts +17 -0
- package/server/lib/managed-restart.ts +8 -0
- package/server/lib/provider-actions.ts +18 -0
- package/server/lib/query-job-coordinator.ts +300 -0
- package/server/lib/query-job-feature.ts +39 -0
- package/server/lib/query-job-resume.ts +269 -0
- package/server/lib/query-job-runtime.ts +23 -3
- package/server/lib/query-job-store.ts +516 -7
- package/server/lib/query-job-types.ts +210 -1
- package/server/lib/queue-turn-evidence.ts +213 -0
- package/server/lib/thread-turn-queue-deliver.ts +6 -1
- package/server/lib/thread-turn-queue-store.ts +13 -4
- package/server/lib/thread-turn-queue.ts +151 -1
- package/server/routes/agent-session-bindings.ts +169 -11
- package/server/routes/health.ts +26 -1
- package/server/routes/maintenance.ts +34 -0
- package/server/routes/meeting.ts +27 -9
- package/server/routes/query-jobs.ts +107 -13
- package/server/routes/recovery.ts +4 -0
- package/server/routes/thread-turn-queue.ts +371 -10
- package/server/scripts/live-cues-v2-notes-mutation-gate.mjs +0 -170
- package/server/scripts/transcript-cleanup-mutation-gate.mjs +0 -219
|
@@ -13,6 +13,7 @@ import { continueFullPermissionsEnabled } from './continue-plan.js'
|
|
|
13
13
|
import { permissionBrokerHealthFields } from './permission-broker.js'
|
|
14
14
|
import { continueLiveEnabled } from './session-peer-inbox-deps.js'
|
|
15
15
|
import { codexLiveQueueEnabled } from './codex-live-queue.js'
|
|
16
|
+
import { sessionSteerEnabled } from './thread-turn-queue.js'
|
|
16
17
|
|
|
17
18
|
export type ProviderName = 'claude' | 'codex' | 'cursor'
|
|
18
19
|
|
|
@@ -33,6 +34,11 @@ export interface ProviderAct {
|
|
|
33
34
|
nativeQueue: ProviderCapability
|
|
34
35
|
keepModel: ProviderCapability
|
|
35
36
|
permissions: ProviderCapability
|
|
37
|
+
/**
|
|
38
|
+
* 6.78.0: can a turn go INTO a busy session's running turn (Sessions steer)? OPTIONAL and
|
|
39
|
+
* ABSENT while `COS_SESSION_STEER` is off, so the eight 6.62 keys stay the whole object.
|
|
40
|
+
*/
|
|
41
|
+
steer?: ProviderCapability
|
|
36
42
|
}
|
|
37
43
|
|
|
38
44
|
export interface ProviderActInput {
|
|
@@ -48,6 +54,8 @@ export interface ProviderActInput {
|
|
|
48
54
|
continueLive: boolean
|
|
49
55
|
/** `COS_CODEX_LIVE_QUEUE` (Codex's app-queue hop, idle and busy). */
|
|
50
56
|
codexLiveQueue: boolean
|
|
57
|
+
/** 6.78.0: `COS_SESSION_STEER`. Absent or false: no `steer` entry at all. */
|
|
58
|
+
sessionSteer?: boolean
|
|
51
59
|
env?: NodeJS.ProcessEnv
|
|
52
60
|
}
|
|
53
61
|
|
|
@@ -63,6 +71,10 @@ export function providerActs(input: ProviderActInput): Record<ProviderName, Prov
|
|
|
63
71
|
const fork = (provider: ProviderName, mode?: string) =>
|
|
64
72
|
(input.forkProviders.includes(provider) ? yes(mode) : no('fork_unsupported'))
|
|
65
73
|
const desk = (key: string) => (cancel[key] === true ? yes('next_command') : no('hooks_not_ready'))
|
|
74
|
+
// 6.78.0: Sessions steer. Claude through the peer inbox (it needs the live hop); Codex has
|
|
75
|
+
// `turn/steer` only in an app-server COS does not hold; Cursor's only door is the Stop hook.
|
|
76
|
+
const steer = input.sessionSteer === true
|
|
77
|
+
const claudeSteer = !attach ? no('attach_disabled') : input.continueLive ? yes('peer_inbox') : no('no_outside_channel')
|
|
66
78
|
return {
|
|
67
79
|
claude: {
|
|
68
80
|
continueIdle: continueIdle(input.continueLive ? 'live' : 'resume'),
|
|
@@ -74,6 +86,7 @@ export function providerActs(input: ProviderActInput): Record<ProviderName, Prov
|
|
|
74
86
|
nativeQueue: no('engine_limit'),
|
|
75
87
|
keepModel: yes(),
|
|
76
88
|
permissions: yes('session'),
|
|
89
|
+
...(steer ? { steer: claudeSteer } : {}),
|
|
77
90
|
},
|
|
78
91
|
codex: {
|
|
79
92
|
continueIdle: continueIdle(input.codexLiveQueue ? 'app_queue' : 'resume'),
|
|
@@ -85,6 +98,7 @@ export function providerActs(input: ProviderActInput): Record<ProviderName, Prov
|
|
|
85
98
|
nativeQueue: yes(),
|
|
86
99
|
keepModel: full ? yes() : no('full_permissions_off'),
|
|
87
100
|
permissions: full ? yes('session_posture') : no('full_permissions_off'),
|
|
101
|
+
...(steer ? { steer: no('no_outside_channel') } : {}),
|
|
88
102
|
},
|
|
89
103
|
cursor: {
|
|
90
104
|
// An idle IDE composer cannot take a message from outside; a CLI chat can.
|
|
@@ -97,6 +111,7 @@ export function providerActs(input: ProviderActInput): Record<ProviderName, Prov
|
|
|
97
111
|
nativeQueue: no('engine_limit'),
|
|
98
112
|
keepModel: full ? yes() : no('full_permissions_off'),
|
|
99
113
|
permissions: full ? yes('run_everything') : no('full_permissions_off'),
|
|
114
|
+
...(steer ? { steer: no('engine_limit') } : {}),
|
|
100
115
|
},
|
|
101
116
|
}
|
|
102
117
|
}
|
|
@@ -145,5 +160,8 @@ export function liveProviderActInput(base: {
|
|
|
145
160
|
brokerEnabled,
|
|
146
161
|
continueLive: continueLiveEnabled(),
|
|
147
162
|
codexLiveQueue: codexLiveQueueEnabled(),
|
|
163
|
+
// 6.78.0: present (and true) only while COS_SESSION_STEER is on, so the input object the
|
|
164
|
+
// 6.62 tests build is unchanged with the flag off.
|
|
165
|
+
...(sessionSteerEnabled() ? { sessionSteer: true } : {}),
|
|
148
166
|
}
|
|
149
167
|
}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { getOrCreateSession } from './conversation.js'
|
|
2
2
|
import {
|
|
3
|
+
QueryJobNotFoundError,
|
|
4
|
+
QueryJobPersistenceError,
|
|
3
5
|
QueryJobProviderOrphanFenceError,
|
|
6
|
+
QueryJobSteerConflictError,
|
|
4
7
|
QueryJobStore,
|
|
5
8
|
QueryJobStoreError,
|
|
6
9
|
type QueryJobAdmissionResult,
|
|
@@ -9,17 +12,30 @@ import {
|
|
|
9
12
|
} from './query-job-store.js'
|
|
10
13
|
import {
|
|
11
14
|
QUERY_JOB_LIMITS,
|
|
15
|
+
QUERY_JOB_STEER_LIMITS,
|
|
16
|
+
QueryJobValidationError,
|
|
12
17
|
isTerminalQueryJobStatus,
|
|
13
18
|
normalizeQueryJobError,
|
|
14
19
|
parsePositiveInteger,
|
|
20
|
+
parseSteerId,
|
|
21
|
+
parseSteerText,
|
|
22
|
+
steerFallback,
|
|
23
|
+
steerReadState,
|
|
24
|
+
steerTextSha256,
|
|
15
25
|
type QueryJobProviderLinkage,
|
|
16
26
|
type QueryJobOutputImageStats,
|
|
17
27
|
type QueryJobRequest,
|
|
18
28
|
type QueryJobSnapshot,
|
|
29
|
+
type QueryJobStatus,
|
|
30
|
+
type QueryJobSteer,
|
|
31
|
+
type QueryJobSteerState,
|
|
19
32
|
type QueryJobStoreHealth,
|
|
20
33
|
} from './query-job-types.js'
|
|
34
|
+
import { SteerPort, steerClock, steerCutoffMs, type PendingSteer, type QueryProvider } from './claude-steer.js'
|
|
21
35
|
import type { MaintenanceWorkLease } from './maintenance-lifecycle.js'
|
|
22
36
|
import { applyTrailToolMode, type JobTrailDraft } from './job-trail.js'
|
|
37
|
+
import { crashLoopParkedError, type BootResumeGate } from './boot-resume.js'
|
|
38
|
+
import { restartNoticeEnabled } from './query-job-feature.js'
|
|
23
39
|
|
|
24
40
|
const CLIENT_JOB_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
|
|
25
41
|
|
|
@@ -63,6 +79,51 @@ export interface QueryJobRunnerContext {
|
|
|
63
79
|
request: QueryJobRequest
|
|
64
80
|
signal: AbortSignal
|
|
65
81
|
callbacks: QueryJobRunnerCallbacks
|
|
82
|
+
/** 6.78.0: present only when this run is steerable (see QueryJobSteerOptions). */
|
|
83
|
+
steer?: SteerPort
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* 6.78.0 Messages steer. A run is steerable only when ALL hold: the switch is on
|
|
88
|
+
* (COS_MESSAGES_STEER), the client asked (`request.steerable`), it is not a dispatch, the
|
|
89
|
+
* provider (resolved as model-router resolves it) is Claude, and the installed CLI takes
|
|
90
|
+
* `--input-format stream-json --replay-user-messages`.
|
|
91
|
+
*/
|
|
92
|
+
export interface QueryJobSteerOptions {
|
|
93
|
+
enabled: () => boolean
|
|
94
|
+
provider: (request: QueryJobRequest) => QueryProvider
|
|
95
|
+
cliSupported: () => Promise<boolean>
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** 6.78.0: what POST /query-jobs/:jobId/steer and the steer lookup answer (contract messages-steer.json). */
|
|
99
|
+
export interface QueryJobSteerReply {
|
|
100
|
+
steerId: string
|
|
101
|
+
state: QueryJobSteerState
|
|
102
|
+
fallback: 'queue_follow_up' | null
|
|
103
|
+
jobStatus: QueryJobStatus
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export class QueryJobSteerLimitError extends Error {
|
|
107
|
+
readonly code = 'steer_limit'
|
|
108
|
+
constructor() {
|
|
109
|
+
super(`This run already has ${QUERY_JOB_STEER_LIMITS.steersPerJob} steers.`)
|
|
110
|
+
this.name = 'QueryJobSteerLimitError'
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export class QueryJobSteerUnknownError extends Error {
|
|
115
|
+
readonly code = 'steer_unknown'
|
|
116
|
+
constructor() {
|
|
117
|
+
super('No steer with this id.')
|
|
118
|
+
this.name = 'QueryJobSteerUnknownError'
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function steerReply(steer: QueryJobSteer, job: Pick<QueryJobSnapshot, 'status'>): QueryJobSteerReply {
|
|
123
|
+
const state = steerReadState(steer.state, job.status)
|
|
124
|
+
// The fallback reads the journaled state: a delivered steer reads not_applied on an ended
|
|
125
|
+
// job, and must still never be resent (P7 S1).
|
|
126
|
+
return { steerId: steer.steerId, state, fallback: steerFallback(steer.state, job.status), jobStatus: job.status }
|
|
66
127
|
}
|
|
67
128
|
|
|
68
129
|
export type QueryJobRunner = (context: QueryJobRunnerContext) => Promise<void>
|
|
@@ -86,8 +147,24 @@ export interface QueryJobCoordinatorOptions {
|
|
|
86
147
|
* refusing its start lease) before the coordinator looks again. Bounded so a Mac
|
|
87
148
|
* that slept through the end of a fence does not wait out a stale timer. */
|
|
88
149
|
holdRecheckMs?: number
|
|
150
|
+
/** 6.78.0: the process's crash-loop gate (lib/boot-resume.ts), read when it is used. */
|
|
151
|
+
bootResumeGate?: () => Pick<BootResumeGate, 'admit' | 'retry' | 'forget' | 'limit'>
|
|
152
|
+
/** 6.78.0 restart truth: defaults to COS_RESTART_NOTICE (on), read at shutdown. */
|
|
153
|
+
restartNotice?: () => boolean
|
|
154
|
+
/** 6.78.0: the longest shutdown waits for steps a run already reported to reach the journal. */
|
|
155
|
+
shutdownDrainMs?: number
|
|
156
|
+
/** 6.78.0: Messages steer. Absent (the default) means no run is ever steerable. */
|
|
157
|
+
steer?: QueryJobSteerOptions
|
|
89
158
|
}
|
|
90
159
|
|
|
160
|
+
/**
|
|
161
|
+
* 6.78.0 restart truth: how long a graceful shutdown lets the steps, chunks and activity a run
|
|
162
|
+
* already reported reach the journal before it writes the interrupt, so the notice names them.
|
|
163
|
+
* All active runs drain at once, so this bounds the whole wait, far inside index.ts's 8-second
|
|
164
|
+
* force exit. A run whose writes are stuck (a broken disk) is interrupted when it runs out.
|
|
165
|
+
*/
|
|
166
|
+
export const SHUTDOWN_CALLBACK_DRAIN_MS = 1_000
|
|
167
|
+
|
|
91
168
|
/** Margin past the fence before the start is retried, so the retry lands after it. */
|
|
92
169
|
const FENCE_HOLD_SLACK_MS = 250
|
|
93
170
|
|
|
@@ -134,6 +211,10 @@ interface ActiveRun {
|
|
|
134
211
|
providerTimeoutPromise: Promise<void>
|
|
135
212
|
resolveProviderTimeout: () => void
|
|
136
213
|
providerTimeoutTimer: ReturnType<typeof setTimeout> | null
|
|
214
|
+
/** 6.78.0: the run's steer port, present only when the run is steerable. */
|
|
215
|
+
steer?: SteerPort
|
|
216
|
+
/** 6.78.0: when the coordinator deadline fires, on the monotonic steer clock (`steerClock`), once the run is running. */
|
|
217
|
+
deadlineAt?: number
|
|
137
218
|
}
|
|
138
219
|
|
|
139
220
|
export interface QueryJobCoordinatorHealth {
|
|
@@ -168,6 +249,8 @@ export class QueryJobCoordinator {
|
|
|
168
249
|
private readonly holds = new Map<string, () => void>()
|
|
169
250
|
/** Admitted jobs inside execute that have not started (or given up) yet. */
|
|
170
251
|
private readonly waiting = new Set<string>()
|
|
252
|
+
/** 6.78.0: steer ports by job, registered before a steerable run's runner starts. */
|
|
253
|
+
private readonly steerPorts = new Map<string, SteerPort>()
|
|
171
254
|
private readonly holdRecheckMs: number
|
|
172
255
|
private admissionTail: Promise<void> = Promise.resolve()
|
|
173
256
|
private shuttingDown = false
|
|
@@ -200,12 +283,53 @@ export class QueryJobCoordinator {
|
|
|
200
283
|
// 6.61.3: work the last boot admitted but never started runs now, with no
|
|
201
284
|
// phone needed to send it again, in admission order. Each takes its own start
|
|
202
285
|
// lease in execute.
|
|
286
|
+
const parked: string[] = []
|
|
203
287
|
for (const { jobId, sessionId } of this.store.takeRequeuedOnBoot()) {
|
|
288
|
+
// 6.78.0: counted BEFORE its start is queued, so a start that kills the process is
|
|
289
|
+
// counted. Past the limit the job is parked: never started again by a boot.
|
|
290
|
+
if (!this.admitBootRequeue(jobId)) { parked.push(jobId); continue }
|
|
204
291
|
queueMicrotask(() => { void this.execute(jobId, sessionId) })
|
|
205
292
|
}
|
|
293
|
+
for (const jobId of parked) await this.parkBootRequeue(jobId)
|
|
206
294
|
return this.getHealth()
|
|
207
295
|
}
|
|
208
296
|
|
|
297
|
+
/** Keyed by lineage (clientJobId), so the app's Retry (a new generation) is the same item. */
|
|
298
|
+
private admitBootRequeue(jobId: string): boolean {
|
|
299
|
+
const gate = this.options.bootResumeGate?.()
|
|
300
|
+
if (!gate) return true
|
|
301
|
+
try {
|
|
302
|
+
return gate.admit('query_job', this.store.lineageOf(jobId) ?? jobId, 'start')
|
|
303
|
+
} catch {
|
|
304
|
+
return true
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* A parked job ends `interrupted` with `restart_limit` and `interruption.resumable: false`, so
|
|
310
|
+
* the app shows it and offers Retry (never Resume: its run never started).
|
|
311
|
+
*/
|
|
312
|
+
private async parkBootRequeue(jobId: string): Promise<void> {
|
|
313
|
+
try {
|
|
314
|
+
const result = await this.store.interruptParked(jobId, crashLoopParkedError(this.options.bootResumeGate?.().limit))
|
|
315
|
+
if (!result.applied) return
|
|
316
|
+
console.warn(`[query-jobs] ${jobId.slice(0, 8)} parked: its start crashed the server on every boot; Retry runs it again`)
|
|
317
|
+
const execution = await this.store.getExecution(jobId)
|
|
318
|
+
await this.projectTerminal(execution.job, execution.request)
|
|
319
|
+
} catch (error) {
|
|
320
|
+
// Left `accepted`: the next boot offers it again and the gate keeps it parked.
|
|
321
|
+
console.error(`[query-jobs] ${jobId.slice(0, 8)} could not be parked:`, error)
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
private bootResumeLineage(op: 'retry' | 'forget', clientJobId: string): void {
|
|
326
|
+
try {
|
|
327
|
+
const gate = this.options.bootResumeGate?.()
|
|
328
|
+
if (op === 'retry') gate?.retry('query_job', clientJobId)
|
|
329
|
+
else gate?.forget('query_job', clientJobId)
|
|
330
|
+
} catch { /* the gate fails open on its own; never fail an admission or a start */ }
|
|
331
|
+
}
|
|
332
|
+
|
|
209
333
|
/** Admission serialization makes first-run session allocation idempotent:
|
|
210
334
|
* two simultaneous retries without a sessionId cannot allocate two sessions
|
|
211
335
|
* and conflict solely because the client had not learned the first one. */
|
|
@@ -224,6 +348,9 @@ export class QueryJobCoordinator {
|
|
|
224
348
|
const admission = await this.store.admit(normalized)
|
|
225
349
|
resolve(admission)
|
|
226
350
|
if (admission.created) {
|
|
351
|
+
// 6.78.0: a new generation of a lineage is the person's Retry (the app's Retry or
|
|
352
|
+
// Resume, or a Messages retry): a parked lineage is unparked and its count starts over.
|
|
353
|
+
if (admission.job.generation > 1) this.bootResumeLineage('retry', admission.job.clientJobId)
|
|
227
354
|
if (maintenanceLease) this.admittedMaintenance.set(admission.job.jobId, maintenanceLease)
|
|
228
355
|
queueMicrotask(() => { void this.execute(admission.job.jobId, admission.job.sessionId) })
|
|
229
356
|
} else {
|
|
@@ -302,6 +429,8 @@ export class QueryJobCoordinator {
|
|
|
302
429
|
try {
|
|
303
430
|
const starting = await this.store.markStarting(jobId, () => !this.shuttingDown)
|
|
304
431
|
if (starting.applied) {
|
|
432
|
+
// 6.78.0: a started job is never handed back by a boot again, so its count is done.
|
|
433
|
+
this.bootResumeLineage('forget', starting.job.clientJobId)
|
|
305
434
|
if (heldSinceMs != null) console.info(`[query-jobs] ${id} starting after a ${Math.round((Date.now() - heldSinceMs) / 1000)}s hold`)
|
|
306
435
|
return { lease }
|
|
307
436
|
}
|
|
@@ -367,6 +496,7 @@ export class QueryJobCoordinator {
|
|
|
367
496
|
this.waiting.add(jobId)
|
|
368
497
|
let release: (() => void) | undefined
|
|
369
498
|
let maintenanceLease: MaintenanceWorkLease | undefined
|
|
499
|
+
let steerMode = false
|
|
370
500
|
let execution
|
|
371
501
|
try {
|
|
372
502
|
if (lockRequest) {
|
|
@@ -387,6 +517,10 @@ export class QueryJobCoordinator {
|
|
|
387
517
|
release?.()
|
|
388
518
|
return
|
|
389
519
|
}
|
|
520
|
+
// 6.78.0: decided before the start (the CLI probe is cached), so nothing awaits between
|
|
521
|
+
// the start and the runner but the steer seeding below.
|
|
522
|
+
// With the switch off nothing is awaited here, so a 6.77.0 start keeps its exact timing.
|
|
523
|
+
if (this.options.steer?.enabled()) steerMode = await this.steerable(execution.request)
|
|
390
524
|
const started = await this.startWhenClear(jobId, lockRequest ? undefined : admitted)
|
|
391
525
|
if (!started) {
|
|
392
526
|
release?.()
|
|
@@ -431,7 +565,23 @@ export class QueryJobCoordinator {
|
|
|
431
565
|
resolveProviderTimeout,
|
|
432
566
|
providerTimeoutTimer: null,
|
|
433
567
|
}
|
|
568
|
+
if (steerMode) {
|
|
569
|
+
active.steer = this.steerPortFor(active)
|
|
570
|
+
this.steerPorts.set(jobId, active.steer)
|
|
571
|
+
}
|
|
434
572
|
this.active.set(jobId, active)
|
|
573
|
+
if (active.steer) {
|
|
574
|
+
// Registered first, then seeded, so a steer journaled meanwhile is offered by one side or
|
|
575
|
+
// both (the port drops a repeat): the ones accepted while the job waited, then, for a
|
|
576
|
+
// Retry or Resume, the ones an interrupted or failed generation did not get to apply.
|
|
577
|
+
await this.seedSteers(active).catch(error => { this.handleCallbackPersistenceFailure(active, error) })
|
|
578
|
+
if (active.terminalPersisted || active.persistenceFailed) return
|
|
579
|
+
if (this.shuttingDown) {
|
|
580
|
+
// As above: never spawn under a closing server. The job stays `starting` for the boot.
|
|
581
|
+
this.abandonActive(active)
|
|
582
|
+
return
|
|
583
|
+
}
|
|
584
|
+
}
|
|
435
585
|
|
|
436
586
|
const callbacks = this.callbacksFor(active)
|
|
437
587
|
try {
|
|
@@ -441,6 +591,7 @@ export class QueryJobCoordinator {
|
|
|
441
591
|
request: execution.request,
|
|
442
592
|
signal: active.controller.signal,
|
|
443
593
|
callbacks,
|
|
594
|
+
...(active.steer ? { steer: active.steer } : {}),
|
|
444
595
|
})
|
|
445
596
|
|
|
446
597
|
const runnerFailed = runnerPromise.then(
|
|
@@ -568,6 +719,8 @@ export class QueryJobCoordinator {
|
|
|
568
719
|
|
|
569
720
|
private armProviderDeadline(active: ActiveRun): void {
|
|
570
721
|
if (active.providerTimeoutTimer || active.terminalPersisted || active.persistenceFailed) return
|
|
722
|
+
// On the monotonic steer clock (P7 S7): decideSteer compares it with the bridge's wall clock.
|
|
723
|
+
active.deadlineAt = steerClock() + this.providerTimeoutMs
|
|
571
724
|
active.providerTimeoutTimer = setTimeout(() => active.resolveProviderTimeout(), this.providerTimeoutMs)
|
|
572
725
|
active.providerTimeoutTimer.unref?.()
|
|
573
726
|
}
|
|
@@ -734,17 +887,141 @@ export class QueryJobCoordinator {
|
|
|
734
887
|
after: number,
|
|
735
888
|
listener: Parameters<QueryJobStore['subscribe']>[3],
|
|
736
889
|
trail?: Parameters<QueryJobStore['subscribe']>[4],
|
|
890
|
+
steer?: Parameters<QueryJobStore['subscribe']>[5],
|
|
737
891
|
): Promise<QueryJobSubscription> {
|
|
892
|
+
if (steer) return this.store.subscribe(jobId, generation, after, listener, trail, steer)
|
|
738
893
|
return trail
|
|
739
894
|
? this.store.subscribe(jobId, generation, after, listener, trail)
|
|
740
895
|
: this.store.subscribe(jobId, generation, after, listener)
|
|
741
896
|
}
|
|
742
897
|
|
|
898
|
+
/**
|
|
899
|
+
* 6.78.0 Messages steer, in the plan's order: validate; look the steer id up across the
|
|
900
|
+
* lineage (the same text replays, other text is a conflict) BEFORE the generation check;
|
|
901
|
+
* decide and journal (fsynced) `accepted`, `too_late` or `unsupported`; then, for an
|
|
902
|
+
* accepted steer, write it through the run's port. The reply is the state after the write:
|
|
903
|
+
* a steer the port could no longer take is journaled `too_late` and answered so.
|
|
904
|
+
*/
|
|
905
|
+
async steer(jobId: string, raw: unknown): Promise<QueryJobSteerReply> {
|
|
906
|
+
const body = raw && typeof raw === 'object' ? raw as Record<string, unknown> : {}
|
|
907
|
+
const generation = parsePositiveInteger(body.generation)
|
|
908
|
+
if (generation == null || generation < 1) throw new QueryJobValidationError('invalid_generation')
|
|
909
|
+
const steerId = parseSteerId(body.steerId)
|
|
910
|
+
const text = parseSteerText(body.text)
|
|
911
|
+
await this.store.init()
|
|
912
|
+
const clientJobId = this.store.lineageOf(jobId)
|
|
913
|
+
if (!clientJobId) throw new QueryJobNotFoundError(jobId)
|
|
914
|
+
const known = await this.store.findSteer(clientJobId, steerId)
|
|
915
|
+
if (known) {
|
|
916
|
+
if (known.textSha256 !== steerTextSha256(text)) throw new QueryJobSteerConflictError()
|
|
917
|
+
return steerReply(known.steer, known.job)
|
|
918
|
+
}
|
|
919
|
+
const job = await this.store.getSnapshot(jobId, generation)
|
|
920
|
+
if (await this.store.steerCount(jobId) >= QUERY_JOB_STEER_LIMITS.steersPerJob) throw new QueryJobSteerLimitError()
|
|
921
|
+
const decided = await this.decideSteer(job)
|
|
922
|
+
let result = await this.store.appendSteer(jobId, { steerId, state: decided, text, create: true })
|
|
923
|
+
if (result.applied && result.steer.state === 'accepted') {
|
|
924
|
+
const port = this.steerPorts.get(jobId)
|
|
925
|
+
if (port && port.offer({ steerId, text }) === 'closed') {
|
|
926
|
+
result = await this.store.appendSteer(jobId, { steerId, state: 'too_late' })
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
return steerReply(result.steer, result.job)
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
/** 6.78.0: GET /query-jobs/steers/:steerId?clientJobId=, from the newest generation. */
|
|
933
|
+
async steerStatus(clientJobId: string, rawSteerId: unknown): Promise<QueryJobSteerReply> {
|
|
934
|
+
const steerId = parseSteerId(rawSteerId)
|
|
935
|
+
const known = await this.store.findSteer(clientJobId, steerId)
|
|
936
|
+
if (!known) throw new QueryJobSteerUnknownError()
|
|
937
|
+
return steerReply(known.steer, known.job)
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
private async steerable(request: QueryJobRequest): Promise<boolean> {
|
|
941
|
+
const options = this.options.steer
|
|
942
|
+
if (!options || !options.enabled() || request.steerable !== true || request.dispatch) return false
|
|
943
|
+
try {
|
|
944
|
+
if (options.provider(request) !== 'claude') return false
|
|
945
|
+
return await options.cliSupported()
|
|
946
|
+
} catch {
|
|
947
|
+
return false
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
/** The state a new steer is journaled with (before the port is tried). */
|
|
952
|
+
private async decideSteer(job: QueryJobSnapshot): Promise<'accepted' | 'too_late' | 'unsupported'> {
|
|
953
|
+
if (isTerminalQueryJobStatus(job.status) || job.status === 'answer_ready') return 'too_late'
|
|
954
|
+
const active = this.active.get(job.jobId)
|
|
955
|
+
if (!active) {
|
|
956
|
+
// Not started yet (waiting its turn, fenced, or held by a drain): it reads the predicate.
|
|
957
|
+
const execution = await this.store.getExecution(job.jobId)
|
|
958
|
+
return await this.steerable(execution.request) ? 'accepted' : 'unsupported'
|
|
959
|
+
}
|
|
960
|
+
const port = active.steer
|
|
961
|
+
if (!port) return 'unsupported'
|
|
962
|
+
if (port.isClosed) return 'too_late'
|
|
963
|
+
// Monotonic, as the timers that end the run are (P7 S7): after a Mac sleep Date.now()
|
|
964
|
+
// has jumped and every steer would read too late.
|
|
965
|
+
const now = steerClock()
|
|
966
|
+
const wallMax = port.wallMax()
|
|
967
|
+
const margin = steerCutoffMs(wallMax ?? this.providerTimeoutMs)
|
|
968
|
+
const wallLeft = port.wallRemainingMs(now)
|
|
969
|
+
if (wallLeft !== null && wallLeft < margin) return 'too_late'
|
|
970
|
+
if (active.deadlineAt !== undefined && active.deadlineAt - now < margin) return 'too_late'
|
|
971
|
+
return 'accepted'
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
/** A steer port whose events journal through the run's ordered callback tail. */
|
|
975
|
+
private steerPortFor(active: ActiveRun): SteerPort {
|
|
976
|
+
const journal = (operation: () => Promise<unknown>) => {
|
|
977
|
+
void this.enqueueCallback(active, async () => {
|
|
978
|
+
try {
|
|
979
|
+
await operation()
|
|
980
|
+
} catch (error) {
|
|
981
|
+
// A broken journal is the run's failure; a steer bookkeeping refusal is not.
|
|
982
|
+
if (error instanceof QueryJobPersistenceError || !(error instanceof QueryJobStoreError)) throw error
|
|
983
|
+
console.warn(`[query-jobs] ${active.jobId.slice(0, 8)} steer not journaled:`, (error as Error).message)
|
|
984
|
+
}
|
|
985
|
+
})
|
|
986
|
+
}
|
|
987
|
+
const jobId = active.jobId
|
|
988
|
+
return new SteerPort({
|
|
989
|
+
onIncluded: steers => journal(async () => {
|
|
990
|
+
for (const steer of steers) {
|
|
991
|
+
await this.store.appendSteer(jobId, steer.carriedFrom !== undefined
|
|
992
|
+
? { steerId: steer.steerId, state: 'included', text: steer.text, carriedFrom: steer.carriedFrom, create: true }
|
|
993
|
+
: { steerId: steer.steerId, state: 'included' })
|
|
994
|
+
}
|
|
995
|
+
}),
|
|
996
|
+
onDelivered: steerId => journal(() => this.store.appendSteer(jobId, { steerId, state: 'delivered' })),
|
|
997
|
+
onFolded: steerIds => journal(async () => {
|
|
998
|
+
for (const steerId of steerIds) await this.store.appendSteer(jobId, { steerId, state: 'folded' })
|
|
999
|
+
}),
|
|
1000
|
+
onCheckpoint: text => journal(() => this.store.appendSteerCheckpoint(jobId, text)),
|
|
1001
|
+
})
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
/** Offer the run's port what it must start with: this job's accepted steers, then carried ones. */
|
|
1005
|
+
private async seedSteers(active: ActiveRun): Promise<void> {
|
|
1006
|
+
const port = active.steer!
|
|
1007
|
+
const own: PendingSteer[] = (await this.store.queuedSteers(active.jobId))
|
|
1008
|
+
.filter(steer => steer.state === 'accepted' && steer.carriedFrom === undefined)
|
|
1009
|
+
.map(steer => ({ steerId: steer.steerId, text: steer.text }))
|
|
1010
|
+
for (const steer of own) port.offer(steer)
|
|
1011
|
+
if (active.generation > 1) {
|
|
1012
|
+
for (const steer of await this.store.listCarryableSteers(active.request.clientJobId, active.generation)) port.offer(steer)
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
|
|
743
1016
|
async shutdown(reason = 'server_shutdown'): Promise<void> {
|
|
744
1017
|
this.shuttingDown = true
|
|
745
1018
|
// Held jobs stay `accepted` in the journal; the next boot re-queues them.
|
|
746
1019
|
for (const end of [...this.holds.values()]) end()
|
|
747
1020
|
await this.admissionTail.catch(() => {})
|
|
1021
|
+
// 6.78.0 restart truth: the notice is built from the journal, so the steps a run reported
|
|
1022
|
+
// before this point reach it first (bounded; see SHUTDOWN_CALLBACK_DRAIN_MS). Off, the
|
|
1023
|
+
// interrupt is written at once, as 6.77.0 did, ahead of any queued step.
|
|
1024
|
+
if ((this.options.restartNotice ?? restartNoticeEnabled)()) await this.drainCallbacks([...this.active.values()])
|
|
748
1025
|
for (const active of [...this.active.values()]) {
|
|
749
1026
|
try {
|
|
750
1027
|
const result = await this.store.interrupt(active.jobId, reason)
|
|
@@ -759,9 +1036,25 @@ export class QueryJobCoordinator {
|
|
|
759
1036
|
}
|
|
760
1037
|
}
|
|
761
1038
|
|
|
1039
|
+
/** Every run's queued callback writes, all at once, for at most the drain bound. Never throws. */
|
|
1040
|
+
private async drainCallbacks(runs: ActiveRun[]): Promise<void> {
|
|
1041
|
+
if (runs.length === 0) return
|
|
1042
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
1043
|
+
const bound = new Promise<void>(resolve => {
|
|
1044
|
+
timer = setTimeout(resolve, Math.max(0, this.options.shutdownDrainMs ?? SHUTDOWN_CALLBACK_DRAIN_MS))
|
|
1045
|
+
timer.unref?.()
|
|
1046
|
+
})
|
|
1047
|
+
try {
|
|
1048
|
+
await Promise.race([Promise.all(runs.map(run => run.callbackTail.catch(() => {}))), bound])
|
|
1049
|
+
} finally {
|
|
1050
|
+
if (timer) clearTimeout(timer)
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
|
|
762
1054
|
private finishActive(active: ActiveRun): void {
|
|
763
1055
|
if (active.terminalPersisted) return
|
|
764
1056
|
active.terminalPersisted = true
|
|
1057
|
+
this.releaseSteerPort(active)
|
|
765
1058
|
active.resolveTerminal()
|
|
766
1059
|
if (active.providerTimeoutTimer) clearTimeout(active.providerTimeoutTimer)
|
|
767
1060
|
active.providerTimeoutTimer = null
|
|
@@ -775,7 +1068,14 @@ export class QueryJobCoordinator {
|
|
|
775
1068
|
}
|
|
776
1069
|
}
|
|
777
1070
|
|
|
1071
|
+
private releaseSteerPort(active: ActiveRun): void {
|
|
1072
|
+
if (!active.steer) return
|
|
1073
|
+
active.steer.close()
|
|
1074
|
+
if (this.steerPorts.get(active.jobId) === active.steer) this.steerPorts.delete(active.jobId)
|
|
1075
|
+
}
|
|
1076
|
+
|
|
778
1077
|
private abandonActive(active: ActiveRun): void {
|
|
1078
|
+
this.releaseSteerPort(active)
|
|
779
1079
|
if (active.providerTimeoutTimer) clearTimeout(active.providerTimeoutTimer)
|
|
780
1080
|
active.providerTimeoutTimer = null
|
|
781
1081
|
if (active.partialTimer) clearTimeout(active.partialTimer)
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readSwitch } from './env-switch.js'
|
|
1
2
|
import { QUERY_JOB_PROTOCOL_VERSION } from './query-job-types.js'
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -19,6 +20,44 @@ export function messagesTrailEnabled(env: NodeJS.ProcessEnv = process.env): bool
|
|
|
19
20
|
return env.COS_MESSAGES_TRAIL !== '0'
|
|
20
21
|
}
|
|
21
22
|
|
|
23
|
+
// 6.78.0 switches all read through `readSwitch` (lib/env-switch.ts): `1`, `true` or `on`
|
|
24
|
+
// force a switch on, `0`, `false` or `off` force it off, anything else is its DEFAULT.
|
|
25
|
+
const triState = readSwitch
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* 6.78.0 restart truth, default ON. An Ask a restart cut off (crash then boot, or a graceful
|
|
29
|
+
* shutdown) carries `error.cause` and an `interruption` record naming the step it was cut off
|
|
30
|
+
* during and the side-effect steps already done; a crash-loop-parked one carries its
|
|
31
|
+
* `interruption` too. Off is the MVP-C build exactly: the boot path's old message, the raw
|
|
32
|
+
* `server_shutdown` message on a graceful shutdown, and no `interruption` anywhere.
|
|
33
|
+
*/
|
|
34
|
+
export function restartNoticeEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
|
35
|
+
return triState(env.COS_RESTART_NOTICE, true)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* 6.78.0 (MVP-D release 1): the delta-only chunk WRITER, default OFF. The reader is always
|
|
40
|
+
* on, so a journal a writer-on build wrote reads back whole on this build and on every later
|
|
41
|
+
* one. The writer ships on by default one release after its reader.
|
|
42
|
+
*/
|
|
43
|
+
export function queryJobDeltaJournalEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
|
44
|
+
return triState(env.COS_QUERY_JOB_DELTA_JOURNAL, false)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* 6.78.0 Messages steer, default OFF in 6.78.0 (on for the candidate; the default flips after
|
|
49
|
+
* the field window). Read per job and per request, so a restart applies a change.
|
|
50
|
+
*/
|
|
51
|
+
export function messagesSteerEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
|
52
|
+
return triState(env.COS_MESSAGES_STEER, false)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** 6.78.0: `capabilities.durableQueryJobs.steer`, or undefined (the key is then absent). */
|
|
56
|
+
export function messagesSteerCapability(input: { jobsEnabled: boolean; cliSupported: boolean | null }, env: NodeJS.ProcessEnv = process.env): { protocolVersion: 1; providers: ['claude'] } | undefined {
|
|
57
|
+
if (!input.jobsEnabled || input.cliSupported !== true || !messagesSteerEnabled(env)) return undefined
|
|
58
|
+
return { protocolVersion: 1, providers: ['claude'] }
|
|
59
|
+
}
|
|
60
|
+
|
|
22
61
|
export function durableQueryJobsCapability(): {
|
|
23
62
|
enabled: boolean
|
|
24
63
|
protocolVersion: number
|