@gotcos/glasses-server 6.77.0 → 6.78.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +40 -0
- package/CHANGELOG.md +24 -0
- package/README.md +12 -1
- package/managed-runtime-contract.json +9 -1
- package/package.json +10 -1
- package/server/index.ts +96 -10
- package/server/lib/agent-session-binding-registry.ts +623 -9
- package/server/lib/boot-resume.ts +552 -0
- package/server/lib/claude-bridge.ts +152 -3
- package/server/lib/claude-cli-support.ts +39 -0
- package/server/lib/claude-steer.ts +259 -0
- package/server/lib/env-switch.ts +17 -0
- package/server/lib/managed-restart.ts +8 -0
- package/server/lib/native-head.ts +65 -6
- package/server/lib/provider-actions.ts +18 -0
- package/server/lib/query-job-coordinator.ts +300 -0
- package/server/lib/query-job-feature.ts +39 -0
- package/server/lib/query-job-resume.ts +269 -0
- package/server/lib/query-job-runtime.ts +23 -3
- package/server/lib/query-job-store.ts +516 -7
- package/server/lib/query-job-types.ts +210 -1
- package/server/lib/queue-turn-evidence.ts +213 -0
- package/server/lib/thread-turn-queue-deliver.ts +6 -1
- package/server/lib/thread-turn-queue-store.ts +13 -4
- package/server/lib/thread-turn-queue.ts +151 -1
- package/server/routes/agent-session-bindings.ts +169 -11
- package/server/routes/health.ts +26 -1
- package/server/routes/maintenance.ts +34 -0
- package/server/routes/meeting.ts +27 -9
- package/server/routes/query-jobs.ts +107 -13
- package/server/routes/recovery.ts +4 -0
- package/server/routes/thread-turn-queue.ts +371 -10
- package/server/scripts/live-cues-v2-notes-mutation-gate.mjs +0 -170
- package/server/scripts/transcript-cleanup-mutation-gate.mjs +0 -219
|
@@ -56,6 +56,10 @@ export type QueryJobEventType =
|
|
|
56
56
|
// and reaches only a subscriber that asked for the trail. Its data is one draft plus
|
|
57
57
|
// `trailSeq`, a dense 1..n per job.
|
|
58
58
|
| 'trail'
|
|
59
|
+
// 6.78.0 (Messages steer): a steer's state, or the answer checkpoint of a steered run. A
|
|
60
|
+
// JOURNAL row type only, like `trail`: never a job event, never an eventSeq or a ring slot,
|
|
61
|
+
// never `updatedAt`. Builds before 6.78.0 skip it as malformed.
|
|
62
|
+
| 'steer'
|
|
59
63
|
|
|
60
64
|
export type QueryJobActivityMode = 'off' | 'status' | 'preview'
|
|
61
65
|
export type QueryJobActivityKind = 'status' | 'input' | 'output' | 'gap'
|
|
@@ -128,6 +132,12 @@ export interface QueryJobRequest {
|
|
|
128
132
|
* it names the session, it does not change what is asked.
|
|
129
133
|
*/
|
|
130
134
|
sessionName?: string
|
|
135
|
+
/**
|
|
136
|
+
* 6.78.0: the client can steer this run (it read `durableQueryJobs.steer`). Only `true` is
|
|
137
|
+
* kept. Excluded from the fingerprint: it says how the answer may be guided, not what is
|
|
138
|
+
* asked, and a build that does not know it must still hydrate the job.
|
|
139
|
+
*/
|
|
140
|
+
steerable?: true
|
|
131
141
|
}
|
|
132
142
|
|
|
133
143
|
/** 6.58.2: longest session name kept (the Claude sidebar shows about 40 characters). */
|
|
@@ -198,6 +208,195 @@ export interface QueryJobError {
|
|
|
198
208
|
message: string
|
|
199
209
|
retryable?: boolean
|
|
200
210
|
retryAfterMs?: number
|
|
211
|
+
/** 6.78.0: why the server ended the job, from QUERY_JOB_ERROR_CAUSES only. Additive: a
|
|
212
|
+
* client that does not read it sees the 6.77.0 error. */
|
|
213
|
+
cause?: string
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** 6.78.0: the only `error.cause` values a journaled error keeps. An allowlist, so no
|
|
217
|
+
* provider error (an Error's own `cause` included) can start carrying one. */
|
|
218
|
+
export const QUERY_JOB_ERROR_CAUSES: ReadonlySet<string> = new Set(['server_restarted', 'server_shutdown', 'crash_loop_parked'])
|
|
219
|
+
|
|
220
|
+
/** 6.78.0: why a restart ended a job: a crash then boot, a graceful shutdown, or crash-loop parking. */
|
|
221
|
+
export const QUERY_JOB_RESTART_CAUSES = ['server_restarted', 'server_shutdown', 'crash_loop_parked'] as const
|
|
222
|
+
export type QueryJobRestartCause = typeof QUERY_JOB_RESTART_CAUSES[number]
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* 6.78.0: the message of a job either restart path cut off. The WHOLE message fits the lens's
|
|
226
|
+
* 32-column footer lead (P7 S8: the 34-character first copy was cut mid-word by companion
|
|
227
|
+
* 6.10.644). The hold menu offers Retry or Resume from `interruption.resumable`.
|
|
228
|
+
*/
|
|
229
|
+
export const QUERY_JOB_RESTART_MESSAGE = 'Server restarted. Tap to retry.'
|
|
230
|
+
|
|
231
|
+
/** 6.78.0: the bounds of `snapshot.interruption` (contract interrupted-job.json `limits`). */
|
|
232
|
+
export const QUERY_JOB_INTERRUPTION_LIMITS = Object.freeze({
|
|
233
|
+
alreadyDoneMax: 5,
|
|
234
|
+
alreadyDoneEntryChars: 60,
|
|
235
|
+
cutOffDuringChars: 60,
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* 6.78.0: what a restart cut off, built once from the job's own trail when the job ends
|
|
240
|
+
* (lib/query-job-resume.ts `buildInterruption`) and journaled with the terminal record.
|
|
241
|
+
*
|
|
242
|
+
* cutOffDuring the step that was running, or null when none was (or none is known)
|
|
243
|
+
* alreadyDone side-effect steps that finished (reads and searches are not listed), the
|
|
244
|
+
* newest QUERY_JOB_INTERRUPTION_LIMITS.alreadyDoneMax of them
|
|
245
|
+
* incomplete the list may be missing steps: tool activity was off, the trail hit its
|
|
246
|
+
* bound, a step had no outcome to match, older steps were dropped from the
|
|
247
|
+
* list, or the provider started and left no trail at all
|
|
248
|
+
* resumable the app may offer Resume (false for a crash-loop-parked job)
|
|
249
|
+
*
|
|
250
|
+
* Additive: absent on every job a restart did not end, and with COS_RESTART_NOTICE=0.
|
|
251
|
+
*/
|
|
252
|
+
export interface QueryJobInterruption {
|
|
253
|
+
cause: QueryJobRestartCause
|
|
254
|
+
at: string
|
|
255
|
+
cutOffDuring: string | null
|
|
256
|
+
alreadyDone: string[]
|
|
257
|
+
incomplete: boolean
|
|
258
|
+
resumable: boolean
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const RESTART_CAUSES: ReadonlySet<string> = new Set(QUERY_JOB_RESTART_CAUSES)
|
|
262
|
+
|
|
263
|
+
/** One line of the interruption record: redacted like activity, one line, at most `max` characters. */
|
|
264
|
+
export function interruptionLine(value: unknown, max: number): string {
|
|
265
|
+
if (typeof value !== 'string' || !value.replace(CONTROL_RE, '').trim()) return ''
|
|
266
|
+
const text = sanitizeQueryJobActivity(value).text.replace(/\s+/g, ' ').trim()
|
|
267
|
+
return text.length <= max ? text : `${text.slice(0, Math.max(0, max - 3)).trimEnd()}...`
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** The only shape a journaled interruption may have, from anything. Undefined when it is not one. */
|
|
271
|
+
export function parseQueryJobInterruption(raw: unknown): QueryJobInterruption | undefined {
|
|
272
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined
|
|
273
|
+
const r = raw as Record<string, unknown>
|
|
274
|
+
if (typeof r.cause !== 'string' || !RESTART_CAUSES.has(r.cause)) return undefined
|
|
275
|
+
if (typeof r.at !== 'string' || !Number.isFinite(Date.parse(r.at))) return undefined
|
|
276
|
+
if (typeof r.incomplete !== 'boolean' || typeof r.resumable !== 'boolean' || !Array.isArray(r.alreadyDone)) return undefined
|
|
277
|
+
const limits = QUERY_JOB_INTERRUPTION_LIMITS
|
|
278
|
+
const cutOffDuring = typeof r.cutOffDuring === 'string' ? interruptionLine(r.cutOffDuring, limits.cutOffDuringChars) || null : null
|
|
279
|
+
const alreadyDone = r.alreadyDone
|
|
280
|
+
.filter((entry): entry is string => typeof entry === 'string')
|
|
281
|
+
.map(entry => interruptionLine(entry, limits.alreadyDoneEntryChars))
|
|
282
|
+
.filter(Boolean)
|
|
283
|
+
.slice(-limits.alreadyDoneMax)
|
|
284
|
+
return {
|
|
285
|
+
cause: r.cause as QueryJobRestartCause,
|
|
286
|
+
at: r.at,
|
|
287
|
+
cutOffDuring,
|
|
288
|
+
alreadyDone,
|
|
289
|
+
incomplete: r.incomplete,
|
|
290
|
+
resumable: r.resumable,
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** 6.78.0 Messages steer: the bounds (contract messages-steer.json `limits`). */
|
|
295
|
+
export const QUERY_JOB_STEER_LIMITS = Object.freeze({
|
|
296
|
+
steerChars: 4_000,
|
|
297
|
+
/** Steers a client may send to one job. Carried steers (from an earlier generation) do not count. */
|
|
298
|
+
steersPerJob: 8,
|
|
299
|
+
})
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* 6.78.0: a steer's state (contract messages-steer.json `states`).
|
|
303
|
+
*
|
|
304
|
+
* accepted journaled; written to the CLI, or waiting for the first line
|
|
305
|
+
* delivered the CLI replayed it: it has it
|
|
306
|
+
* folded a result covered it: the answer includes it
|
|
307
|
+
* included composed into the first prompt (sent before the start, or carried from an
|
|
308
|
+
* interrupted or failed generation)
|
|
309
|
+
* too_late the run could not take it any more (answering, ended, or near a deadline)
|
|
310
|
+
* unsupported this run cannot be steered (Codex, Cursor, Ollama, a dispatch, an old client)
|
|
311
|
+
* not_applied NEVER journaled: read for an accepted or delivered steer of an ended job
|
|
312
|
+
*/
|
|
313
|
+
export const QUERY_JOB_STEER_STATES = ['accepted', 'delivered', 'folded', 'included', 'too_late', 'unsupported', 'not_applied'] as const
|
|
314
|
+
export type QueryJobSteerState = typeof QUERY_JOB_STEER_STATES[number]
|
|
315
|
+
export type QueryJobJournaledSteerState = Exclude<QueryJobSteerState, 'not_applied'>
|
|
316
|
+
const JOURNALED_STEER_STATES: ReadonlySet<string> = new Set(QUERY_JOB_STEER_STATES.filter(state => state !== 'not_applied'))
|
|
317
|
+
|
|
318
|
+
export interface QueryJobSteer {
|
|
319
|
+
steerId: string
|
|
320
|
+
state: QueryJobSteerState
|
|
321
|
+
/** When the current state was journaled. */
|
|
322
|
+
at: string
|
|
323
|
+
text: string
|
|
324
|
+
/** The generation it was first sent to, when it was carried into this one. */
|
|
325
|
+
carriedFrom?: number
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** A steer state only moves forward: accepted, then delivered, then a final state. */
|
|
329
|
+
export function steerStateRank(state: string): number {
|
|
330
|
+
return state === 'accepted' ? 1 : state === 'delivered' ? 2 : 3
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** The state a client reads: an accepted or delivered steer of an ended job did not apply. */
|
|
334
|
+
export function steerReadState(state: QueryJobSteerState, jobStatus: QueryJobStatus): QueryJobSteerState {
|
|
335
|
+
return isTerminalQueryJobStatus(jobStatus) && (state === 'accepted' || state === 'delivered') ? 'not_applied' : state
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/**
|
|
339
|
+
* What the client does with a steer that will not reach this run. `queue_follow_up`: send it as
|
|
340
|
+
* the next Ask. Null: nothing (it applied, it may have applied, it is carried by Retry or
|
|
341
|
+
* Resume, or the run was canceled).
|
|
342
|
+
*
|
|
343
|
+
* Decided from the JOURNALED state, never the read one (P7 S1). A steer the CLI replayed
|
|
344
|
+
* (`delivered`) reached the run: a deadline, inactivity, a restart or a second-turn error can
|
|
345
|
+
* still complete the job from its checkpoint, and the read state is then `not_applied`, but
|
|
346
|
+
* the guidance may already be in the answer or in Claude's session. Sending it again as an Ask
|
|
347
|
+
* would apply it twice, which the plan forbids, so a delivered steer never falls back.
|
|
348
|
+
*/
|
|
349
|
+
export function steerFallback(journaled: QueryJobSteerState, jobStatus: QueryJobStatus): 'queue_follow_up' | null {
|
|
350
|
+
if (journaled === 'too_late' || journaled === 'unsupported') return 'queue_follow_up'
|
|
351
|
+
if (journaled === 'delivered') return null
|
|
352
|
+
if (steerReadState(journaled, jobStatus) === 'not_applied' && jobStatus === 'completed') return 'queue_follow_up'
|
|
353
|
+
return null
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** One `steer` journal row: a state, or a steered run's answer checkpoint. */
|
|
357
|
+
export type QueryJobSteerRow =
|
|
358
|
+
| { kind: 'state'; steerId: string; state: QueryJobJournaledSteerState; textSha256: string; text?: string; carriedFrom?: number }
|
|
359
|
+
| { kind: 'checkpoint'; text: string; truncated: boolean }
|
|
360
|
+
|
|
361
|
+
/** The only shapes a `steer` row may have. Null means malformed. */
|
|
362
|
+
export function parseQueryJobSteerRow(raw: unknown): QueryJobSteerRow | null {
|
|
363
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null
|
|
364
|
+
const r = raw as Record<string, unknown>
|
|
365
|
+
if (r.checkpoint && typeof r.checkpoint === 'object') {
|
|
366
|
+
const c = r.checkpoint as Record<string, unknown>
|
|
367
|
+
if (typeof c.text !== 'string' || c.text.length > QUERY_JOB_LIMITS.partialChars) return null
|
|
368
|
+
return { kind: 'checkpoint', text: c.text, truncated: c.truncated === true }
|
|
369
|
+
}
|
|
370
|
+
if (typeof r.steerId !== 'string' || !CLIENT_JOB_ID_RE.test(r.steerId)) return null
|
|
371
|
+
if (typeof r.state !== 'string' || !JOURNALED_STEER_STATES.has(r.state)) return null
|
|
372
|
+
if (typeof r.textSha256 !== 'string' || !/^[0-9a-f]{64}$/.test(r.textSha256)) return null
|
|
373
|
+
if (r.text !== undefined && (typeof r.text !== 'string' || r.text.length > QUERY_JOB_STEER_LIMITS.steerChars)) return null
|
|
374
|
+
const carriedFrom = r.carriedFrom === undefined ? undefined : Number(r.carriedFrom)
|
|
375
|
+
if (carriedFrom !== undefined && (!Number.isSafeInteger(carriedFrom) || carriedFrom < 1)) return null
|
|
376
|
+
return {
|
|
377
|
+
kind: 'state', steerId: r.steerId, state: r.state as QueryJobJournaledSteerState, textSha256: r.textSha256,
|
|
378
|
+
...(typeof r.text === 'string' ? { text: r.text } : {}),
|
|
379
|
+
...(carriedFrom !== undefined ? { carriedFrom } : {}),
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/** Steer text as the journal and the CLI get it: control characters out, trimmed, bounded. */
|
|
384
|
+
export function parseSteerText(raw: unknown): string {
|
|
385
|
+
if (typeof raw !== 'string') throw new QueryJobValidationError('invalid_steer_text')
|
|
386
|
+
const text = raw.replace(CONTROL_RE, '').trim()
|
|
387
|
+
if (!text) throw new QueryJobValidationError('invalid_steer_text')
|
|
388
|
+
if (text.length > QUERY_JOB_STEER_LIMITS.steerChars) throw new QueryJobValidationError('steer_text_too_large')
|
|
389
|
+
return text
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** A steer id is a client-minted UUID v4, lowercased. */
|
|
393
|
+
export function parseSteerId(raw: unknown): string {
|
|
394
|
+
if (typeof raw !== 'string' || !CLIENT_JOB_ID_RE.test(raw.toLowerCase())) throw new QueryJobValidationError('invalid_steer_id')
|
|
395
|
+
return raw.toLowerCase()
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
export function steerTextSha256(text: string): string {
|
|
399
|
+
return createHash('sha256').update(text, 'utf8').digest('hex')
|
|
201
400
|
}
|
|
202
401
|
|
|
203
402
|
export interface QueryJobActivity {
|
|
@@ -239,6 +438,14 @@ export interface QueryJobSnapshot extends QueryJobProviderLinkage {
|
|
|
239
438
|
responseTruncated?: boolean
|
|
240
439
|
outputImageStats?: QueryJobOutputImageStats
|
|
241
440
|
error?: QueryJobError
|
|
441
|
+
/** 6.78.0: what a restart cut off (see QueryJobInterruption). Only on a job a restart ended. */
|
|
442
|
+
interruption?: QueryJobInterruption
|
|
443
|
+
/**
|
|
444
|
+
* 6.78.0: the steers sent to this job, in arrival order, with the state JOURNALED (a reader
|
|
445
|
+
* computes `not_applied`, see steerReadState). ABSENT on a job with no steer, and stripped
|
|
446
|
+
* from every snapshot sent to a client that did not ask with `?steer=1`.
|
|
447
|
+
*/
|
|
448
|
+
steers?: QueryJobSteer[]
|
|
242
449
|
activity: QueryJobActivity[]
|
|
243
450
|
/** The readable trail (6.52.0), bounded by `QUERY_JOB_LIMITS.trailEntries` and
|
|
244
451
|
* `trailChars`, rebuilt from the journal on restart. ABSENT, never `[]`, on a job
|
|
@@ -450,6 +657,7 @@ export function parseQueryJobRequest(raw: unknown, options: ParseQueryJobRequest
|
|
|
450
657
|
...(input.handoffLatest === true ? { handoffLatest: true } : {}),
|
|
451
658
|
...(clientQueueItemId ? { clientQueueItemId } : {}),
|
|
452
659
|
...(sessionName ? { sessionName } : {}),
|
|
660
|
+
...(input.steerable === true ? { steerable: true as const } : {}),
|
|
453
661
|
attachmentIds,
|
|
454
662
|
attachmentRefs,
|
|
455
663
|
activityToolMode,
|
|
@@ -519,7 +727,7 @@ export const FINGERPRINT_KEYS = [
|
|
|
519
727
|
/** The request keys deliberately OUTSIDE the identity. Every key of
|
|
520
728
|
* `QueryJobRequest` must appear in exactly one of the two lists: adding a key
|
|
521
729
|
* without naming it here or above fails to compile, so the author picks a side. */
|
|
522
|
-
export const FINGERPRINT_EXCLUDED = ['origin', 'dispatch', 'sessionName'] as const satisfies readonly (keyof QueryJobRequest)[]
|
|
730
|
+
export const FINGERPRINT_EXCLUDED = ['origin', 'dispatch', 'sessionName', 'steerable'] as const satisfies readonly (keyof QueryJobRequest)[]
|
|
523
731
|
type FingerprintUncovered = Exclude<keyof QueryJobRequest, typeof FINGERPRINT_KEYS[number] | typeof FINGERPRINT_EXCLUDED[number]>
|
|
524
732
|
export const FINGERPRINT_KEYS_COVER_REQUEST: FingerprintUncovered extends never ? true : never = true
|
|
525
733
|
|
|
@@ -593,6 +801,7 @@ export function normalizeQueryJobError(error: unknown, fallbackCode = 'query_job
|
|
|
593
801
|
...(candidate.retryable === true ? { retryable: true } : {}),
|
|
594
802
|
...(typeof candidate.retryAfterMs === 'number' && Number.isFinite(candidate.retryAfterMs)
|
|
595
803
|
? { retryAfterMs: Math.max(0, Math.ceil(candidate.retryAfterMs)) } : {}),
|
|
804
|
+
...(typeof candidate.cause === 'string' && QUERY_JOB_ERROR_CAUSES.has(candidate.cause) ? { cause: candidate.cause } : {}),
|
|
596
805
|
}
|
|
597
806
|
}
|
|
598
807
|
|
|
@@ -16,6 +16,9 @@
|
|
|
16
16
|
// throw is `false` for each, which is what the composition root did before the move.
|
|
17
17
|
|
|
18
18
|
import { cancelVoidsOpenTurn, type ThreadCancel } from './session-cancel.js'
|
|
19
|
+
import { unwrapPeerMessage } from './session-stream-events.js'
|
|
20
|
+
import type { TurnAttemptLookup } from './agent-session-binding-registry.js'
|
|
21
|
+
import type { QueuedThreadTurn } from './thread-turn-queue.js'
|
|
19
22
|
|
|
20
23
|
/** The hook signal fields these questions read (a `SessionSignal` satisfies it). */
|
|
21
24
|
export interface TurnEvidenceSignal {
|
|
@@ -135,3 +138,213 @@ export function makeQueueTurnEvidence(deps: QueueTurnEvidenceDeps): QueueTurnEvi
|
|
|
135
138
|
|
|
136
139
|
return { turnEnded, turnOpen }
|
|
137
140
|
}
|
|
141
|
+
|
|
142
|
+
// =====================================================================================
|
|
143
|
+
// 6.78.0 (P3): WHAT HAPPENED TO A TURN A RESTART CUT OFF, AND WHETHER A KEY WAS ALREADY SENT
|
|
144
|
+
// =====================================================================================
|
|
145
|
+
//
|
|
146
|
+
// Delivering a prompt is NOT replay-safe: a second copy lands in a real conversation. So
|
|
147
|
+
// every answer below comes from POSITIVE evidence, and absence of evidence is "unknown",
|
|
148
|
+
// never "not sent", with exactly one exception that is itself a positive fact: a row the
|
|
149
|
+
// 6.78 drain stamped (`journal: 1`) whose attempt journal holds no attempt at all. The
|
|
150
|
+
// journal is written in the same commit as the pin, and nothing is sent before the pin, so
|
|
151
|
+
// that absence proves the turn never reached a delivery hop.
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
/** What a `delivering` row from a dead boot is settled as. */
|
|
155
|
+
export type DeliveryEvidence = 'delivered' | 'not_delivered' | 'unknown'
|
|
156
|
+
|
|
157
|
+
/** A transcript row this far before the attempt still counts (one Mac, one clock; a writer may stamp before it appends). */
|
|
158
|
+
export const EVIDENCE_BACKDATE_MS = 2_000
|
|
159
|
+
|
|
160
|
+
/** The facts the reconcile decides on, each read by the caller. */
|
|
161
|
+
export interface ReconcileFacts {
|
|
162
|
+
row: Pick<QueuedThreadTurn, 'queuedAt' | 'journal' | 'mode'>
|
|
163
|
+
/** `outcome` of the terminal ledger record for this key on the target, or null when there is none. */
|
|
164
|
+
ledgerOutcome: string | null
|
|
165
|
+
/** An exact, wrapper-aware transcript match after the attempt began. */
|
|
166
|
+
transcriptMatch: boolean
|
|
167
|
+
/** A Codex native queue row with the same text. */
|
|
168
|
+
codexQueueMatch: boolean
|
|
169
|
+
/** The attempt journal's answer, or null when the journal could not be read. */
|
|
170
|
+
attempt: TurnAttemptLookup | null
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* THE RECONCILE ORDER (plan v3.1 P3, V3 W4). Pure, every branch resolves:
|
|
175
|
+
* 1. a `completed` or `queued` ledger record, or an attempt that exited `sent`: delivered
|
|
176
|
+
* 2. an exact transcript match after the attempt began: delivered (it may upgrade an ambiguous record)
|
|
177
|
+
* 3. a Codex queue row with the same text: delivered
|
|
178
|
+
* 4. an ambiguous ledger record, an unstamped (legacy) row, a steer row, or an unreadable
|
|
179
|
+
* journal, with none of the above: unknown
|
|
180
|
+
* 5. not_delivered ONLY when the attempt says `not_sent`, or a stamped turn row has no
|
|
181
|
+
* attempt at all and the journal cannot have shed one; anything else is unknown
|
|
182
|
+
*/
|
|
183
|
+
export function reconcileVerdict(facts: ReconcileFacts): DeliveryEvidence {
|
|
184
|
+
if (facts.ledgerOutcome === 'completed' || facts.ledgerOutcome === 'queued') return 'delivered'
|
|
185
|
+
if (facts.attempt?.attempt?.exit === 'sent') return 'delivered'
|
|
186
|
+
if (facts.transcriptMatch) return 'delivered'
|
|
187
|
+
if (facts.codexQueueMatch) return 'delivered'
|
|
188
|
+
if (facts.ledgerOutcome === 'ambiguous') return 'unknown'
|
|
189
|
+
if (facts.row.journal !== 1) return 'unknown'
|
|
190
|
+
if (facts.row.mode === 'steer') return 'unknown'
|
|
191
|
+
if (facts.attempt === null) return 'unknown'
|
|
192
|
+
const attempt = facts.attempt.attempt
|
|
193
|
+
if (attempt !== null) return attempt.exit === 'not_sent' ? 'not_delivered' : 'unknown'
|
|
194
|
+
const shed = facts.attempt.evictedThrough
|
|
195
|
+
if (shed !== null && shed >= facts.row.queuedAt) return 'unknown'
|
|
196
|
+
return 'not_delivered'
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function rowTime(row: Record<string, unknown>): number | null {
|
|
200
|
+
if (typeof row.timestamp !== 'string') return null
|
|
201
|
+
const ms = Date.parse(row.timestamp)
|
|
202
|
+
return Number.isFinite(ms) ? ms : null
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function textOf(content: unknown): string | null {
|
|
206
|
+
if (typeof content === 'string') return content
|
|
207
|
+
if (!Array.isArray(content)) return null
|
|
208
|
+
const parts: string[] = []
|
|
209
|
+
for (const block of content) {
|
|
210
|
+
if (!block || typeof block !== 'object') continue
|
|
211
|
+
const b = block as Record<string, unknown>
|
|
212
|
+
if (b.type === 'tool_result') return null
|
|
213
|
+
if (typeof b.text === 'string' && (b.type === 'text' || b.type === 'input_text' || b.type === undefined)) parts.push(b.text)
|
|
214
|
+
}
|
|
215
|
+
return parts.length > 0 ? parts.join('\n') : null
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** The words, as typed or out of Claude Code's peer wrapper, equal the prompt exactly. */
|
|
219
|
+
function sameWords(text: string | null, want: string): boolean {
|
|
220
|
+
if (text === null) return false
|
|
221
|
+
if (text.trim() === want) return true
|
|
222
|
+
return unwrapPeerMessage(text) === want
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Does this transcript tail show the session took exactly these words, at or after
|
|
227
|
+
* `sinceMs`? WRAPPER-AWARE AND EXACT: the whole prompt, trimmed, either as typed or inside
|
|
228
|
+
* Claude Code's peer-message wrapper. A prefix, a substring or a row without a timestamp is
|
|
229
|
+
* never evidence, because a false "delivered" loses a turn the person is waiting on.
|
|
230
|
+
*
|
|
231
|
+
* claude a user row (including the `isMeta` peer row), an `enqueue` queue-operation,
|
|
232
|
+
* a `queued_command` attachment
|
|
233
|
+
* codex an `event_msg` user_message, or a user `response_item` message
|
|
234
|
+
*/
|
|
235
|
+
export function transcriptShowsPrompt(provider: string, lines: readonly string[] | null, prompt: string, sinceMs: number): boolean {
|
|
236
|
+
const want = typeof prompt === 'string' ? prompt.trim() : ''
|
|
237
|
+
if (!want || !lines || !Number.isFinite(sinceMs)) return false
|
|
238
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
239
|
+
const line = typeof lines[i] === 'string' ? lines[i].trim() : ''
|
|
240
|
+
if (!line.startsWith('{')) continue
|
|
241
|
+
let row: Record<string, unknown>
|
|
242
|
+
try { row = JSON.parse(line) as Record<string, unknown> } catch { continue }
|
|
243
|
+
if (!row || typeof row !== 'object') continue
|
|
244
|
+
const at = rowTime(row)
|
|
245
|
+
if (at === null || at < sinceMs) continue
|
|
246
|
+
if (provider === 'claude') {
|
|
247
|
+
if (row.type === 'user') {
|
|
248
|
+
const message = row.message as Record<string, unknown> | undefined
|
|
249
|
+
if (message && typeof message === 'object' && sameWords(textOf(message.content), want)) return true
|
|
250
|
+
} else if (row.type === 'queue-operation' && row.operation === 'enqueue') {
|
|
251
|
+
if (sameWords(textOf(row.content), want)) return true
|
|
252
|
+
} else if (row.type === 'attachment') {
|
|
253
|
+
const attachment = row.attachment as Record<string, unknown> | undefined
|
|
254
|
+
if (attachment && attachment.type === 'queued_command' && sameWords(textOf(attachment.prompt ?? attachment.content), want)) return true
|
|
255
|
+
}
|
|
256
|
+
} else if (provider === 'codex') {
|
|
257
|
+
const payload = row.payload as Record<string, unknown> | undefined
|
|
258
|
+
if (!payload || typeof payload !== 'object') continue
|
|
259
|
+
if (row.type === 'event_msg' && payload.type === 'user_message' && sameWords(typeof payload.message === 'string' ? payload.message : null, want)) return true
|
|
260
|
+
if (row.type === 'response_item' && payload.type === 'message' && payload.role === 'user' && sameWords(textOf(payload.content), want)) return true
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return false
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** Where `makeDeliveryEvidence` reads each fact. Every read may throw; a throw is unknown. */
|
|
267
|
+
export interface DeliveryEvidenceDeps {
|
|
268
|
+
/** The terminal ledger record's `outcome` for this key on the target, or null. */
|
|
269
|
+
ledgerOutcome: (provider: string, threadId: string, clientTurnId: string) => string | null
|
|
270
|
+
attempt: (provider: string, threadId: string, clientTurnId: string) => TurnAttemptLookup | null
|
|
271
|
+
/** The thread's transcript tail as lines, or null when unreadable. */
|
|
272
|
+
transcriptLines: (provider: string, threadId: string) => readonly string[] | null
|
|
273
|
+
/** The plain text of every row in the Codex app's own queue for this thread. */
|
|
274
|
+
codexQueueTexts?: (threadId: string) => Promise<readonly string[]> | readonly string[]
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** `ThreadTurnQueueDeps.deliveryEvidence`, built from the real reads. Never throws. */
|
|
278
|
+
export function makeDeliveryEvidence(deps: DeliveryEvidenceDeps): (turn: QueuedThreadTurn) => Promise<DeliveryEvidence> {
|
|
279
|
+
return async turn => {
|
|
280
|
+
try {
|
|
281
|
+
const ledgerOutcome = deps.ledgerOutcome(turn.provider, turn.threadId, turn.clientTurnId)
|
|
282
|
+
const attempt = deps.attempt(turn.provider, turn.threadId, turn.clientTurnId)
|
|
283
|
+
const since = (attempt?.attempt?.at ?? turn.queuedAt) - EVIDENCE_BACKDATE_MS
|
|
284
|
+
const transcriptMatch = (turn.provider === 'claude' || turn.provider === 'codex')
|
|
285
|
+
&& transcriptShowsPrompt(turn.provider, deps.transcriptLines(turn.provider, turn.threadId), turn.prompt, since)
|
|
286
|
+
let codexQueueMatch = false
|
|
287
|
+
if (!transcriptMatch && turn.provider === 'codex' && typeof deps.codexQueueTexts === 'function') {
|
|
288
|
+
const texts = await deps.codexQueueTexts(turn.threadId)
|
|
289
|
+
codexQueueMatch = texts.some(text => typeof text === 'string' && text.trim() === turn.prompt.trim())
|
|
290
|
+
}
|
|
291
|
+
return reconcileVerdict({ row: turn, ledgerOutcome, transcriptMatch, codexQueueMatch, attempt })
|
|
292
|
+
} catch {
|
|
293
|
+
return 'unknown'
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** A key the turns route must replay instead of sending. */
|
|
299
|
+
export interface TurnReceipt {
|
|
300
|
+
kind: 'delivered' | 'unknown'
|
|
301
|
+
source: 'queue' | 'attempt' | 'transcript'
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Which queue rows are receipts. ONLY a row whose outcome is settled: delivered, or refused
|
|
306
|
+
* with an unknown outcome. Never `waiting` or `delivering`: the drainer posts the turns route
|
|
307
|
+
* with the row's OWN key while the row is `delivering`, so counting that row would make the
|
|
308
|
+
* queue refuse its own delivery (V2 B2, V3 B3).
|
|
309
|
+
*/
|
|
310
|
+
export function isReceiptRow(row: Pick<QueuedThreadTurn, 'status' | 'deliveryState'>): boolean {
|
|
311
|
+
return row.status === 'delivered' || (row.status === 'refused' && row.deliveryState === 'unknown')
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
export interface TurnReceiptDeps {
|
|
315
|
+
queueRows: (provider: string, threadId: string) => readonly QueuedThreadTurn[]
|
|
316
|
+
attempt: (provider: string, threadId: string, clientTurnId: string) => TurnAttemptLookup | null
|
|
317
|
+
transcriptLines?: (provider: string, threadId: string) => readonly string[] | null
|
|
318
|
+
/** This process's boot: a `pending` attempt of THIS boot is a turn in flight, not an orphan. */
|
|
319
|
+
bootId: string
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* The turns POST's receipt check (6.78.0), after the ledger replay and before the claim.
|
|
324
|
+
*
|
|
325
|
+
* queue row delivered delivered
|
|
326
|
+
* queue row refused, delivery unknown unknown
|
|
327
|
+
* attempt sent delivered
|
|
328
|
+
* attempt unknown, or pending from a unknown, unless the transcript now shows the
|
|
329
|
+
* dead boot exact words after it began (then delivered)
|
|
330
|
+
* anything else (no row, not_sent, null: not a receipt; the route's own gates
|
|
331
|
+
* pending in this boot) decide (a turn in flight meets the claim)
|
|
332
|
+
*/
|
|
333
|
+
export function makeTurnReceiptLookup(deps: TurnReceiptDeps): (provider: string, threadId: string, clientTurnId: string, prompt: string) => TurnReceipt | null {
|
|
334
|
+
return (provider, threadId, clientTurnId, prompt) => {
|
|
335
|
+
const row = deps.queueRows(provider, threadId).find(t => t.clientTurnId === clientTurnId)
|
|
336
|
+
if (row && isReceiptRow(row)) return { kind: row.status === 'delivered' ? 'delivered' : 'unknown', source: 'queue' }
|
|
337
|
+
const lookup = deps.attempt(provider, threadId, clientTurnId)
|
|
338
|
+
const attempt = lookup?.attempt ?? null
|
|
339
|
+
if (attempt === null) return null
|
|
340
|
+
if (attempt.exit === 'sent') return { kind: 'delivered', source: 'attempt' }
|
|
341
|
+
const unknown = attempt.exit === 'unknown' || (attempt.exit === 'pending' && attempt.bootId !== deps.bootId)
|
|
342
|
+
if (!unknown) return null
|
|
343
|
+
if (typeof deps.transcriptLines === 'function') {
|
|
344
|
+
let lines: readonly string[] | null = null
|
|
345
|
+
try { lines = deps.transcriptLines(provider, threadId) } catch { lines = null }
|
|
346
|
+
if (transcriptShowsPrompt(provider, lines, prompt, attempt.at - EVIDENCE_BACKDATE_MS)) return { kind: 'delivered', source: 'transcript' }
|
|
347
|
+
}
|
|
348
|
+
return { kind: 'unknown', source: 'attempt' }
|
|
349
|
+
}
|
|
350
|
+
}
|
|
@@ -77,7 +77,7 @@ export async function deliverQueuedTurnOverLoopback(
|
|
|
77
77
|
turn: QueuedThreadTurn,
|
|
78
78
|
port: number,
|
|
79
79
|
token: string,
|
|
80
|
-
): Promise<{ ok: boolean; reason?: string; serverRetryable?: boolean }> {
|
|
80
|
+
): Promise<{ ok: boolean; reason?: string; serverRetryable?: boolean; replayed?: boolean; deliveryState?: string }> {
|
|
81
81
|
// 6.62.0 /qa (Q3): repeat the parking client's acknowledgement, and only that.
|
|
82
82
|
const noteHeaders: Record<string, string> = turn.continueNoteAck === true ? { 'X-COS-Continue-Note': '1' } : {}
|
|
83
83
|
try {
|
|
@@ -131,6 +131,11 @@ export async function deliverQueuedTurnOverLoopback(
|
|
|
131
131
|
ok: false,
|
|
132
132
|
reason: String(sent.body.reason ?? sent.body.error ?? `turn_${sent.status}`),
|
|
133
133
|
serverRetryable: typeof sent.body.retryable === 'boolean' ? sent.body.retryable : undefined,
|
|
134
|
+
// 6.78.0: a REPLAY of a key whose delivery is unknown (a receipt, or an ambiguous
|
|
135
|
+
// ledger record). The drain settles it at once instead of retrying the key until the
|
|
136
|
+
// attempt ceiling: the route will give the same answer every time, by design.
|
|
137
|
+
...(sent.body.replayed === true ? { replayed: true } : {}),
|
|
138
|
+
...(typeof sent.body.deliveryState === 'string' ? { deliveryState: sent.body.deliveryState } : {}),
|
|
134
139
|
}
|
|
135
140
|
} catch (error) {
|
|
136
141
|
return { ok: false, reason: error instanceof Error ? error.message : 'deliver_failed' }
|
|
@@ -12,7 +12,7 @@ import { nativeQueuedCounts } from './codex-queue-control.js'
|
|
|
12
12
|
|
|
13
13
|
import { closeSync, constants, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, readSync } from 'node:fs'
|
|
14
14
|
import { join } from 'node:path'
|
|
15
|
-
import { atomicWriteFileSync } from './atomic-fs.js'
|
|
15
|
+
import { atomicWriteFileSync, durableAtomicWriteFileSync } from './atomic-fs.js'
|
|
16
16
|
import { dataPath } from './data-dir.js'
|
|
17
17
|
import { turnFromTail, type SessionStreamProvider, type TurnFromTail } from './session-stream-events.js'
|
|
18
18
|
import { pruneQueue, type QueuedThreadTurn } from './thread-turn-queue.js'
|
|
@@ -64,9 +64,18 @@ export function readQueue(provider: string, threadId: string, now: number): Queu
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
-
/**
|
|
68
|
-
|
|
69
|
-
|
|
67
|
+
/**
|
|
68
|
+
* Replace a thread's queue. Atomic, so a crash mid-write cannot truncate it.
|
|
69
|
+
*
|
|
70
|
+
* 6.78.0: `durable` also fsyncs the bytes and the directory before returning. Used for the
|
|
71
|
+
* one write that must survive a power cut as well as a crash: a row entering `delivering`,
|
|
72
|
+
* which is the intent the restart reconcile reads. Same bytes either way.
|
|
73
|
+
*/
|
|
74
|
+
export function writeQueue(provider: string, threadId: string, queue: readonly QueuedThreadTurn[], options: { durable?: boolean } = {}): void {
|
|
75
|
+
const path = queuePath(provider, threadId)
|
|
76
|
+
const text = `${JSON.stringify(queue, null, 2)}\n`
|
|
77
|
+
if (options.durable === true) durableAtomicWriteFileSync(path, text)
|
|
78
|
+
else atomicWriteFileSync(path, text)
|
|
70
79
|
}
|
|
71
80
|
|
|
72
81
|
/**
|