@gotcos/glasses-server 6.36.6 → 6.36.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,53 @@
|
|
|
1
1
|
## Unreleased
|
|
2
2
|
|
|
3
|
+
## 6.36.8
|
|
4
|
+
- **The queue now covers the refusal that actually fires.** Device diagnostics, once the
|
|
5
|
+
path was finally instrumented, recorded `native_target_busy` on every Continue that
|
|
6
|
+
reached the server — never `native_thread_working`, which is what 6.36.7 was built
|
|
7
|
+
around. Selecting Continue SUCCEEDS and mints a binding; if the dictation is not
|
|
8
|
+
completed the binding lingers to its TTL, and every Continue inside that window
|
|
9
|
+
refuses. Twelve such bindings had stacked up on one thread over an evening.
|
|
10
|
+
- `native_target_busy` and `native_turn_in_progress` are now queueable. Both are
|
|
11
|
+
transient by construction — a clock clears them — and both are COS's OWN bookkeeping
|
|
12
|
+
rather than a foreign process holding the thread. Delivery still re-runs the full
|
|
13
|
+
gate, so nothing about the safety model changes.
|
|
14
|
+
- `native_target_fenced` is deliberately NOT queueable: "may or may not have been
|
|
15
|
+
delivered" cannot be resolved by waiting, and queueing it risks a duplicate turn in a
|
|
16
|
+
real conversation.
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
## 6.36.7
|
|
20
|
+
- **A turn spoken at a busy thread is now queued instead of refused.** Miles: "if
|
|
21
|
+
there's a session that's still running, that would just put it into the queue the same
|
|
22
|
+
way that the user has the ability to do so." The thread was never stuck — measured at
|
|
23
|
+
the moment he asked, its transcript mtime was 3s old against a 30s window, so the gate
|
|
24
|
+
correctly read `working`. That is the trap: while you are talking to an agent in a
|
|
25
|
+
thread, it is CONTINUOUSLY working, so Continue was unreachable for exactly the thread
|
|
26
|
+
you most want to continue and Fork was the only door.
|
|
27
|
+
- **This does not weaken the attach gate, which is the whole design.** The queue defers
|
|
28
|
+
to the gate rather than bypassing it: delivery re-runs the full occupancy check and
|
|
29
|
+
can still refuse. Several tests exist only to prove that negative.
|
|
30
|
+
- Only occupancy reasons are queueable. A structural refusal — unsupported provider,
|
|
31
|
+
malformed id, attach switched off — still refuses immediately, because telling someone
|
|
32
|
+
their turn is queued when it can never run is worse than refusing it.
|
|
33
|
+
- **Delivery re-enters through the front door**, over loopback to this server's own
|
|
34
|
+
attach and turn routes. Those carry the gate, the target fence, the per-target claim,
|
|
35
|
+
the watermark, the idempotency ledger and the child-pid accounting; a second copy in a
|
|
36
|
+
background worker would be a second place for the gate to drift.
|
|
37
|
+
- **The watermark exemption, approved explicitly by Miles.** A queued turn drains after
|
|
38
|
+
the thread has moved on — that is what it waited for — so its binding is minted fresh
|
|
39
|
+
at delivery. Checked as normal, a queue would fail 100% of the time. An interactive
|
|
40
|
+
turn still gets the full divergence check.
|
|
41
|
+
- Ready means the turn ENDED (`result` / `turn_complete`, detected with the same parser
|
|
42
|
+
the live stream uses), with the 30s idle clock as a backstop so a holder that dies
|
|
43
|
+
cannot wedge the queue. Thirty seconds of silence alone would fire during a long tool
|
|
44
|
+
call and inject into the middle of a turn.
|
|
45
|
+
- Durable under the data home, so it survives a server update and a pocketed phone. The
|
|
46
|
+
attempt count is persisted BEFORE each delivery, so a crash mid-flight cannot reset
|
|
47
|
+
the ceiling and retry forever. Six-hour TTL, 8 waiting per thread, 5 attempts, and a
|
|
48
|
+
cancel route for the × control.
|
|
49
|
+
|
|
50
|
+
|
|
3
51
|
## 6.36.6
|
|
4
52
|
- **The session digest follows the thread instead of its opening.** Miles, from the
|
|
5
53
|
lens: "It's currently showing a legacy session that I had over a day ago... The
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotcos/glasses-server",
|
|
3
|
-
"version": "6.36.
|
|
3
|
+
"version": "6.36.8",
|
|
4
4
|
"description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/server/index.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { createAttachedTurnStream } from './lib/session-stream-producer.js'
|
|
|
24
24
|
import { claudeSessionsRouter } from './routes/claude-sessions.js'
|
|
25
25
|
import {
|
|
26
26
|
createAgentSessionBindingsRouter,
|
|
27
|
+
|
|
27
28
|
threadAttachEnabled,
|
|
28
29
|
} from './routes/agent-session-bindings.js'
|
|
29
30
|
import { AgentSessionBindingRegistry } from './lib/agent-session-binding-registry.js'
|
|
@@ -33,7 +34,7 @@ import { realAttachedWorkspaceDeps, resolveAttachedWorkspace } from './lib/attac
|
|
|
33
34
|
import { deliverAttachedTurn, realAttachedTurnDeps } from './lib/attached-provider-adapter.js'
|
|
34
35
|
import { forkThread, realForkDeps } from './lib/fork-thread.js'
|
|
35
36
|
import { nativeHead, realNativeHeadDeps } from './lib/native-head.js'
|
|
36
|
-
import { threadOccupancy } from './lib/thread-occupancy.js'
|
|
37
|
+
import { threadOccupancy, holderActivity } from './lib/thread-occupancy.js'
|
|
37
38
|
import { displayRouter } from './routes/display.js'
|
|
38
39
|
import { transcribeStreamRouter } from './routes/transcribe-stream.js'
|
|
39
40
|
import { meetingRouter, resumeMeetingFinalizationJobs } from './routes/meeting.js'
|
|
@@ -104,6 +105,12 @@ import {
|
|
|
104
105
|
} from './lib/maintenance-lifecycle.js'
|
|
105
106
|
|
|
106
107
|
const app = express()
|
|
108
|
+
import { createThreadTurnQueueRouter, drainAllThreads } from './routes/thread-turn-queue.js'
|
|
109
|
+
import { transcriptTurnEnded } from './lib/thread-turn-queue-store.js'
|
|
110
|
+
import { transcriptPathFor } from './lib/native-head.js'
|
|
111
|
+
import { deliverQueuedTurnOverLoopback } from './lib/thread-turn-queue-deliver.js'
|
|
112
|
+
import type { QueuedThreadTurn } from './lib/thread-turn-queue.js'
|
|
113
|
+
|
|
107
114
|
const PORT = parseInt(process.env.PORT ?? '3141', 10)
|
|
108
115
|
|
|
109
116
|
// Mode detection — COS mode when a full pipeline directory is configured.
|
|
@@ -491,6 +498,66 @@ app.use('/api', claudeSessionsRouter)
|
|
|
491
498
|
// Registered AFTER agentSessionsRouter deliberately: its paths are 2 and 4 segments
|
|
492
499
|
// (`/agent-sessions/bindings`, `/agent-sessions/:provider/:threadId/attachability`)
|
|
493
500
|
// and cannot shadow that router's `/agent-sessions/:provider/:id` transcript route.
|
|
501
|
+
// ---------------------------------------------------------------------------
|
|
502
|
+
// QUEUED THREAD TURNS
|
|
503
|
+
// ---------------------------------------------------------------------------
|
|
504
|
+
// A turn spoken at a thread that is busy right now is PARKED instead of refused, and
|
|
505
|
+
// delivered when the thread frees. Miles, 2026-08-17: "if there's a session that's
|
|
506
|
+
// still running, that would just put it into the queue the same way that the user has
|
|
507
|
+
// the ability to do so."
|
|
508
|
+
//
|
|
509
|
+
// Registered only when attach is enabled, on the SAME flag as the write routes: a
|
|
510
|
+
// queue whose delivery path does not exist would accept turns it can never send.
|
|
511
|
+
//
|
|
512
|
+
// DELIVERY GOES BACK IN THROUGH THE FRONT DOOR, over loopback to this server's own
|
|
513
|
+
// attach + turn routes. Those two carry the occupancy gate, the target fence, the
|
|
514
|
+
// per-target claim, the divergence watermark, the idempotency ledger and the child-pid
|
|
515
|
+
// accounting; a second copy of that sequence in a background worker is a second place
|
|
516
|
+
// for the gate to drift. One request per delivered turn, at a handful of turns a day,
|
|
517
|
+
// buys the guarantee that the queue CANNOT weaken the gate even by accident.
|
|
518
|
+
if (threadAttachEnabled()) {
|
|
519
|
+
const queueDeps = {
|
|
520
|
+
occupancy: (provider: string, threadId: string) => {
|
|
521
|
+
try {
|
|
522
|
+
const v = threadOccupancy(provider, threadId, occupancyProbes, occupancyDirs)
|
|
523
|
+
return { attachable: v.attachable === true, reason: v.reason ?? null }
|
|
524
|
+
} catch {
|
|
525
|
+
// A throwing probe is not an open door.
|
|
526
|
+
return { attachable: false, reason: 'probe_failed' }
|
|
527
|
+
}
|
|
528
|
+
},
|
|
529
|
+
turnEnded: (provider: string, threadId: string) => {
|
|
530
|
+
if (provider !== 'claude' && provider !== 'codex') return false
|
|
531
|
+
try {
|
|
532
|
+
return transcriptTurnEnded(provider, transcriptPathFor(provider, threadId, nativeHeadDeps))
|
|
533
|
+
} catch {
|
|
534
|
+
return false
|
|
535
|
+
}
|
|
536
|
+
},
|
|
537
|
+
activity: (provider: string, threadId: string): 'working' | 'idle' | 'unknown' => {
|
|
538
|
+
try {
|
|
539
|
+
const read = occupancyProbes.transcriptMtimeMs
|
|
540
|
+
return typeof read === 'function'
|
|
541
|
+
? holderActivity(read(provider as 'claude' | 'codex', threadId), Date.now())
|
|
542
|
+
: 'unknown'
|
|
543
|
+
} catch {
|
|
544
|
+
return 'unknown'
|
|
545
|
+
}
|
|
546
|
+
},
|
|
547
|
+
deliver: (turn: QueuedThreadTurn) => deliverQueuedTurnOverLoopback(turn, PORT, API_TOKEN),
|
|
548
|
+
now: () => Date.now(),
|
|
549
|
+
}
|
|
550
|
+
app.use('/api', createThreadTurnQueueRouter(queueDeps))
|
|
551
|
+
|
|
552
|
+
// Every 20s. Fast enough that a freed thread drains while the user is still looking
|
|
553
|
+
// at the pending row, slow enough to be nothing: the sweep does no work at all when
|
|
554
|
+
// no queue file exists. Unref'd so it never holds the process open.
|
|
555
|
+
const queueDrainTimer = setInterval(() => {
|
|
556
|
+
void drainAllThreads(queueDeps).catch(() => { /* the next sweep retries */ })
|
|
557
|
+
}, 20_000)
|
|
558
|
+
queueDrainTimer.unref()
|
|
559
|
+
}
|
|
560
|
+
|
|
494
561
|
app.use('/api', createAgentSessionBindingsRouter({
|
|
495
562
|
probes: occupancyProbes,
|
|
496
563
|
dirs: occupancyDirs,
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Delivering a queued turn by re-entering this server's own attach + turn routes.
|
|
2
|
+
//
|
|
3
|
+
// WHY LOOPBACK AND NOT A DIRECT CALL. Delivery is not "post a prompt". It is the
|
|
4
|
+
// occupancy gate, the target fence, the per-target claim, the divergence watermark, the
|
|
5
|
+
// idempotency ledger and the child-pid accounting -- a long, ordered sequence in
|
|
6
|
+
// agent-session-bindings.ts where every branch exists because something once went
|
|
7
|
+
// wrong. Re-implementing it here would create a second place for the gate to drift, and
|
|
8
|
+
// the drift would be invisible until the day it mattered. Going back in through the
|
|
9
|
+
// front door means the gate runs exactly once, in its existing home, and this file
|
|
10
|
+
// cannot weaken it even by accident.
|
|
11
|
+
//
|
|
12
|
+
// The cost is one loopback request per delivered turn, at a volume of a handful a day.
|
|
13
|
+
//
|
|
14
|
+
// THE WATERMARK EXEMPTION LIVES HERE, and nowhere else. A queued turn drains after the
|
|
15
|
+
// thread has moved on -- that is what it waited for -- so its binding is minted FRESH
|
|
16
|
+
// at delivery against the thread as it stands. The turn is therefore always composed
|
|
17
|
+
// against a current baseline, which is what the user meant by queueing it: "append this
|
|
18
|
+
// to whatever the thread is when it frees". An interactive turn still gets the full
|
|
19
|
+
// divergence check, because there the user composed against a state they were looking
|
|
20
|
+
// at and a silent change is genuinely surprising. Miles approved this explicitly.
|
|
21
|
+
//
|
|
22
|
+
// FAILS CLOSED, AND AMBIGUITY IS A FAILURE. Anything other than a clean admission
|
|
23
|
+
// resolves `ok: false`, which returns the turn to the queue rather than marking it
|
|
24
|
+
// sent. The caller bounds retries; the danger to avoid here is the opposite one --
|
|
25
|
+
// reporting success for a turn that may not have landed, which loses it silently.
|
|
26
|
+
|
|
27
|
+
import { request } from 'node:http'
|
|
28
|
+
import type { QueuedThreadTurn } from './thread-turn-queue.js'
|
|
29
|
+
|
|
30
|
+
/** Per-request ceiling. Attach and turn both answer immediately; the turn route
|
|
31
|
+
* admits with 202 and does the long work in the background. */
|
|
32
|
+
const DELIVER_TIMEOUT_MS = 15_000
|
|
33
|
+
|
|
34
|
+
interface LoopbackReply { status: number; body: Record<string, unknown> }
|
|
35
|
+
|
|
36
|
+
function post(port: number, token: string, path: string, payload: unknown): Promise<LoopbackReply> {
|
|
37
|
+
return new Promise((resolve, reject) => {
|
|
38
|
+
const data = JSON.stringify(payload)
|
|
39
|
+
const req = request({
|
|
40
|
+
host: '127.0.0.1',
|
|
41
|
+
port,
|
|
42
|
+
path,
|
|
43
|
+
method: 'POST',
|
|
44
|
+
headers: {
|
|
45
|
+
'content-type': 'application/json',
|
|
46
|
+
'content-length': Buffer.byteLength(data),
|
|
47
|
+
// The header the rest of COS authenticates with. NOT `Authorization: Bearer`.
|
|
48
|
+
'X-Cos-Token': token,
|
|
49
|
+
},
|
|
50
|
+
timeout: DELIVER_TIMEOUT_MS,
|
|
51
|
+
}, res => {
|
|
52
|
+
let raw = ''
|
|
53
|
+
res.on('data', c => { raw += c })
|
|
54
|
+
res.on('end', () => {
|
|
55
|
+
let body: Record<string, unknown> = {}
|
|
56
|
+
try { body = raw ? JSON.parse(raw) as Record<string, unknown> : {} } catch { /* non-JSON is a failure below */ }
|
|
57
|
+
resolve({ status: res.statusCode ?? 0, body })
|
|
58
|
+
})
|
|
59
|
+
})
|
|
60
|
+
req.on('timeout', () => { req.destroy(new Error('deliver_timeout')) })
|
|
61
|
+
req.on('error', reject)
|
|
62
|
+
req.write(data)
|
|
63
|
+
req.end()
|
|
64
|
+
})
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Attach, then send. Returns `ok` only on a provable admission.
|
|
69
|
+
*
|
|
70
|
+
* The attach is what re-baselines the watermark: it reads the thread's head digest as
|
|
71
|
+
* it is NOW, so the turn that follows cannot be refused for divergence that happened
|
|
72
|
+
* while the turn was waiting.
|
|
73
|
+
*/
|
|
74
|
+
export async function deliverQueuedTurnOverLoopback(
|
|
75
|
+
turn: QueuedThreadTurn,
|
|
76
|
+
port: number,
|
|
77
|
+
token: string,
|
|
78
|
+
): Promise<{ ok: boolean; reason?: string }> {
|
|
79
|
+
try {
|
|
80
|
+
const attach = await post(
|
|
81
|
+
port, token,
|
|
82
|
+
`/api/agent-sessions/${encodeURIComponent(turn.provider)}/${encodeURIComponent(turn.threadId)}/attach`,
|
|
83
|
+
{ cosSessionId: turn.cosSessionId },
|
|
84
|
+
)
|
|
85
|
+
if (attach.status !== 200 && attach.status !== 201) {
|
|
86
|
+
// The gate said no at drain time. Not an error -- the queue holds and tries again.
|
|
87
|
+
return { ok: false, reason: String(attach.body.error ?? `attach_${attach.status}`) }
|
|
88
|
+
}
|
|
89
|
+
const bindingId = typeof attach.body.bindingId === 'string' ? attach.body.bindingId : ''
|
|
90
|
+
if (!bindingId) return { ok: false, reason: 'attach_no_binding' }
|
|
91
|
+
|
|
92
|
+
const sent = await post(
|
|
93
|
+
port, token,
|
|
94
|
+
`/api/agent-sessions/bindings/${encodeURIComponent(bindingId)}/turns`,
|
|
95
|
+
// `clientTurnId` is carried through unchanged so the turn route's own
|
|
96
|
+
// idempotency ledger recognises a re-delivery of the SAME turn. Without it a
|
|
97
|
+
// retry after an ambiguous response would put the sentence in twice.
|
|
98
|
+
{ clientTurnId: turn.clientTurnId, prompt: turn.prompt },
|
|
99
|
+
)
|
|
100
|
+
// 202 is the success shape: admitted, delivered in the background, poll the ledger.
|
|
101
|
+
if (sent.status === 202 || sent.status === 200) return { ok: true }
|
|
102
|
+
return { ok: false, reason: String(sent.body.error ?? `turn_${sent.status}`) }
|
|
103
|
+
} catch (error) {
|
|
104
|
+
return { ok: false, reason: error instanceof Error ? error.message : 'deliver_failed' }
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// Durable storage for queued thread turns, and the terminal-record probe.
|
|
2
|
+
//
|
|
3
|
+
// SEPARATE FROM THE DECISIONS. `thread-turn-queue.ts` is pure and holds every rule;
|
|
4
|
+
// this file only reads and writes. That split is what lets the rules be tested by
|
|
5
|
+
// execution instead of by reading them.
|
|
6
|
+
//
|
|
7
|
+
// ONE FILE PER THREAD, under the data home so it survives a server update -- the
|
|
8
|
+
// generation directory is replaced wholesale on every Update Server, and a queue that
|
|
9
|
+
// lived there would be silently emptied by a routine upgrade. Same lesson as the
|
|
10
|
+
// stranded voice profiles.
|
|
11
|
+
|
|
12
|
+
import { closeSync, constants, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, readSync } from 'node:fs'
|
|
13
|
+
import { join } from 'node:path'
|
|
14
|
+
import { atomicWriteFileSync } from './atomic-fs.js'
|
|
15
|
+
import { dataPath } from './data-dir.js'
|
|
16
|
+
import { draftsFromLine, type SessionStreamProvider } from './session-stream-events.js'
|
|
17
|
+
import { pruneQueue, type QueuedThreadTurn } from './thread-turn-queue.js'
|
|
18
|
+
|
|
19
|
+
/** Bytes of transcript tail read to decide whether the last turn ended. */
|
|
20
|
+
export const TURN_END_TAIL_BYTES = 64 * 1024
|
|
21
|
+
|
|
22
|
+
function queueDir(): string {
|
|
23
|
+
const dir = dataPath('thread-turn-queue')
|
|
24
|
+
try { mkdirSync(dir, { recursive: true }) } catch { /* the write below reports it */ }
|
|
25
|
+
return dir
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* One file per (provider, thread).
|
|
30
|
+
*
|
|
31
|
+
* The thread id is validated by the caller before it reaches here, but it is still
|
|
32
|
+
* sanitised: this value becomes a PATH, and a route that forgets its guard must not
|
|
33
|
+
* turn into a directory traversal.
|
|
34
|
+
*/
|
|
35
|
+
export function queuePath(provider: string, threadId: string): string {
|
|
36
|
+
const safe = `${provider}-${threadId}`.replace(/[^A-Za-z0-9._-]/g, '_')
|
|
37
|
+
return join(queueDir(), `${safe}.json`)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The queue for a thread, pruned. A missing or corrupt file reads as empty. */
|
|
41
|
+
export function readQueue(provider: string, threadId: string, now: number): QueuedThreadTurn[] {
|
|
42
|
+
const path = queuePath(provider, threadId)
|
|
43
|
+
if (!existsSync(path)) return []
|
|
44
|
+
try {
|
|
45
|
+
const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8'))
|
|
46
|
+
if (!Array.isArray(parsed)) return []
|
|
47
|
+
// Corrupt rows are dropped individually rather than discarding the whole queue:
|
|
48
|
+
// one bad record must not lose the other turns someone is waiting on.
|
|
49
|
+
const rows = parsed.filter((r): r is QueuedThreadTurn =>
|
|
50
|
+
!!r && typeof r === 'object'
|
|
51
|
+
&& typeof (r as QueuedThreadTurn).clientTurnId === 'string'
|
|
52
|
+
&& typeof (r as QueuedThreadTurn).prompt === 'string'
|
|
53
|
+
&& typeof (r as QueuedThreadTurn).queuedAt === 'number')
|
|
54
|
+
return pruneQueue(rows, now)
|
|
55
|
+
} catch {
|
|
56
|
+
return []
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Replace a thread's queue. Atomic, so a crash mid-write cannot truncate it. */
|
|
61
|
+
export function writeQueue(provider: string, threadId: string, queue: readonly QueuedThreadTurn[]): void {
|
|
62
|
+
atomicWriteFileSync(queuePath(provider, threadId), `${JSON.stringify(queue, null, 2)}\n`)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Every thread with a queue file, for the drain sweep. */
|
|
66
|
+
export function queuedThreadKeys(): Array<{ provider: string; threadId: string }> {
|
|
67
|
+
try {
|
|
68
|
+
return readdirSync(queueDir())
|
|
69
|
+
.filter(f => f.endsWith('.json'))
|
|
70
|
+
.map(f => {
|
|
71
|
+
const base = f.slice(0, -5)
|
|
72
|
+
const dash = base.indexOf('-')
|
|
73
|
+
return dash > 0
|
|
74
|
+
? { provider: base.slice(0, dash), threadId: base.slice(dash + 1) }
|
|
75
|
+
: null
|
|
76
|
+
})
|
|
77
|
+
.filter((v): v is { provider: string; threadId: string } => v !== null)
|
|
78
|
+
} catch {
|
|
79
|
+
return []
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Did the holder's last turn END?
|
|
85
|
+
*
|
|
86
|
+
* REUSES `draftsFromLine`, the same parser the live stream uses, so "the turn ended"
|
|
87
|
+
* means exactly here what it means there. Hand-rolling a second `type === 'result'`
|
|
88
|
+
* check is how two definitions of done drift apart.
|
|
89
|
+
*
|
|
90
|
+
* Reads a bounded tail, newest record wins. Returns false on any doubt -- an
|
|
91
|
+
* unreadable transcript is not evidence a turn finished, and false only means the
|
|
92
|
+
* queue HOLDS, which is always the safe answer.
|
|
93
|
+
*/
|
|
94
|
+
export function transcriptTurnEnded(provider: SessionStreamProvider, path: string | null): boolean {
|
|
95
|
+
if (!path || !existsSync(path)) return false
|
|
96
|
+
try {
|
|
97
|
+
// BOTH flags, and both are load-bearing -- hazard-invariants.test.ts enforces
|
|
98
|
+
// them and each is right on its own terms. O_NOFOLLOW: a symlinked `<id>.jsonl`
|
|
99
|
+
// could point at any file on disk and would be parsed here as a transcript.
|
|
100
|
+
// O_NONBLOCK: `openSync` on a FIFO with no writer NEVER RETURNS, and it is a
|
|
101
|
+
// synchronous syscall on Node's single thread, so one planted path would stop
|
|
102
|
+
// health, meeting save and transcribe-stream along with this drain. That one is
|
|
103
|
+
// recorded in the repo as three reproductions of the same bug, >34s to SIGKILL.
|
|
104
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
|
|
105
|
+
try {
|
|
106
|
+
const size = fstatSync(fd).size
|
|
107
|
+
const start = Math.max(0, size - TURN_END_TAIL_BYTES)
|
|
108
|
+
const buf = Buffer.alloc(size - start)
|
|
109
|
+
readSync(fd, buf, 0, buf.length, start)
|
|
110
|
+
const lines = buf.toString('utf-8').split('\n')
|
|
111
|
+
// The first line of a tail read is almost always a fragment.
|
|
112
|
+
if (start > 0) lines.shift()
|
|
113
|
+
let ended = false
|
|
114
|
+
for (const line of lines) {
|
|
115
|
+
if (!line.trim()) continue
|
|
116
|
+
for (const draft of draftsFromLine(provider, line)) {
|
|
117
|
+
if (draft.kind === 'status') ended = draft.state === 'done'
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return ended
|
|
121
|
+
} finally {
|
|
122
|
+
closeSync(fd)
|
|
123
|
+
}
|
|
124
|
+
} catch {
|
|
125
|
+
return false
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
// A turn spoken at a thread that is busy right now, held until it is not.
|
|
2
|
+
//
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// WHY
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
// Miles, 2026-08-17, looking at a thread he could not continue: "This thread, the
|
|
7
|
+
// COS-glasses server, has actually completed, but it's still locked. Ideally, what would
|
|
8
|
+
// happen is if there's a session that's still running, that would just put it into the
|
|
9
|
+
// queue the same way that the user has the ability to do so." His example was his own
|
|
10
|
+
// desktop: he typed into a running turn and Claude Code queued it, pending, with a
|
|
11
|
+
// cancel control.
|
|
12
|
+
//
|
|
13
|
+
// The thread was not stuck. Measured at the moment he asked: transcript mtime 3s old
|
|
14
|
+
// against a 30s window, so `holderActivity` was `working` -- correctly, because a COS
|
|
15
|
+
// session was writing to it. That is the trap: while you are talking to an agent in a
|
|
16
|
+
// thread, that thread is CONTINUOUSLY working, so Continue is unreachable for exactly
|
|
17
|
+
// the thread you most want to continue, and Fork is the only door.
|
|
18
|
+
//
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// THIS DOES NOT WEAKEN THE ATTACH GATE, AND THAT IS THE WHOLE DESIGN
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// The gate refuses to WRITE into a thread another process holds. A queue does not
|
|
23
|
+
// bypass it -- it defers to it. Nothing here decides a turn may be delivered; delivery
|
|
24
|
+
// re-runs the full occupancy check at drain time and can still refuse. So parking is
|
|
25
|
+
// safe by construction, and the only reasons that should still refuse outright are the
|
|
26
|
+
// ones that can NEVER clear.
|
|
27
|
+
//
|
|
28
|
+
// That is the split `queueableRefusal` encodes: an occupancy reason describes a
|
|
29
|
+
// condition that may pass, a structural reason describes a thread that will never be
|
|
30
|
+
// continuable no matter how long anyone waits. Telling someone their turn is queued
|
|
31
|
+
// when it can never run is worse than refusing.
|
|
32
|
+
//
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
// THE WATERMARK EXEMPTION -- Miles approved this explicitly
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
// A binding carries a content watermark, and a write is refused with
|
|
37
|
+
// `native_thread_changed` when the thread's head digest moved since the turn was
|
|
38
|
+
// composed. A queued turn drains AFTER more has been written, by definition -- that is
|
|
39
|
+
// what it was waiting for -- so the watermark has ALWAYS moved by then. Checked as
|
|
40
|
+
// normal, a queue would fail 100% of the time.
|
|
41
|
+
//
|
|
42
|
+
// So a drained turn re-baselines the watermark at delivery. This is a deliberate,
|
|
43
|
+
// scoped exemption, and its justification is the user's intent: queueing means "append
|
|
44
|
+
// this to whatever the thread is when it frees", which is exactly what the desktop
|
|
45
|
+
// queue does. It is NOT a general relaxation -- an interactive turn still gets the full
|
|
46
|
+
// divergence check, because there the user composed against a state they were looking
|
|
47
|
+
// at and a silent change is genuinely surprising.
|
|
48
|
+
//
|
|
49
|
+
// Miles, asked directly before this was built: "Yeah, this is a fine exemption."
|
|
50
|
+
//
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
// WHAT MAKES A TURN READY, AND WHY NOT "IDLE"
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
// Thirty seconds of transcript silence is the wrong trigger on its own: a long tool
|
|
55
|
+
// call goes quiet mid-turn, and draining there would inject a message into the middle
|
|
56
|
+
// of someone's reasoning. The precise signal is the turn ENDING, which is observable in
|
|
57
|
+
// the transcript itself -- Claude writes a `result` record, Codex a `task_complete` /
|
|
58
|
+
// `turn_complete`. The idle clock stays as a BACKSTOP for a holder that dies or a
|
|
59
|
+
// provider that writes no terminal record, so a queue cannot wedge forever on a missing
|
|
60
|
+
// event. Same reasoning as the session-trail handoff, one layer down.
|
|
61
|
+
|
|
62
|
+
/** Terminal-ish states a queued turn can reach. `waiting` is the only live one. */
|
|
63
|
+
export type QueuedTurnStatus =
|
|
64
|
+
| 'waiting'
|
|
65
|
+
| 'delivering'
|
|
66
|
+
| 'delivered'
|
|
67
|
+
| 'refused'
|
|
68
|
+
| 'expired'
|
|
69
|
+
| 'cancelled'
|
|
70
|
+
|
|
71
|
+
export interface QueuedThreadTurn {
|
|
72
|
+
/** Idempotency key, poll key, and row identity -- the same one the phone's pending
|
|
73
|
+
* ledger already tracks, so a queued turn needs no new client vocabulary. */
|
|
74
|
+
clientTurnId: string
|
|
75
|
+
cosSessionId: string
|
|
76
|
+
provider: string
|
|
77
|
+
threadId: string
|
|
78
|
+
prompt: string
|
|
79
|
+
queuedAt: number
|
|
80
|
+
status: QueuedTurnStatus
|
|
81
|
+
/** Delivery attempts made. Bounded so a permanently-refusing target cannot spin. */
|
|
82
|
+
attempts: number
|
|
83
|
+
/** The SERVER's reason for a terminal outcome. Never the client's wording. */
|
|
84
|
+
reason?: string
|
|
85
|
+
settledAt?: number
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* How long a waiting turn stays valid.
|
|
90
|
+
*
|
|
91
|
+
* Six hours, not indefinite. A turn is a thing someone SAID, and saying it into a
|
|
92
|
+
* thread twelve hours later is not what they meant -- the context they were replying to
|
|
93
|
+
* is gone. Expiry is reported, never silent.
|
|
94
|
+
*/
|
|
95
|
+
export const QUEUED_TURN_TTL_MS = 6 * 60 * 60 * 1000
|
|
96
|
+
|
|
97
|
+
/** Waiting turns per thread. Small: this is a queue, not a backlog. */
|
|
98
|
+
export const MAX_QUEUED_PER_THREAD = 8
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Delivery attempts before a turn is given up on.
|
|
102
|
+
*
|
|
103
|
+
* A drain that keeps failing is not going to start working, and a queue that retries
|
|
104
|
+
* forever is a write loop against someone else's session.
|
|
105
|
+
*/
|
|
106
|
+
export const MAX_DELIVERY_ATTEMPTS = 5
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Occupancy reasons a turn may WAIT on, versus ones that must refuse now.
|
|
110
|
+
*
|
|
111
|
+
* The distinction is whether the condition can ever pass. `native_thread_working` is
|
|
112
|
+
* the common one and obviously transient. The `unknown`-shaped reasons are included
|
|
113
|
+
* deliberately: they mean the scan could not SEE, which is a refusal for a write but
|
|
114
|
+
* not a reason to discard something the user said -- and delivery re-runs the gate, so
|
|
115
|
+
* a queue on an unmeasurable thread simply never drains and expires honestly.
|
|
116
|
+
*
|
|
117
|
+
* Everything absent from this set is STRUCTURAL: a provider COS cannot continue, a
|
|
118
|
+
* malformed id, or the write feature being switched off. None of those pass with time,
|
|
119
|
+
* and queueing against them would be a lie told politely.
|
|
120
|
+
*/
|
|
121
|
+
const QUEUEABLE_REFUSALS: ReadonlySet<string> = new Set([
|
|
122
|
+
'native_thread_working',
|
|
123
|
+
// COS'S OWN LEFTOVER BINDING, and the reason this feature did not work for a day.
|
|
124
|
+
// Proven from device diagnostics 2026-08-17: every Continue that reached the server
|
|
125
|
+
// was refused `native_target_busy` ("already attached to another COS chat"), never
|
|
126
|
+
// `native_thread_working`. Selecting Continue SUCCEEDS and mints a binding; if the
|
|
127
|
+
// dictation is not completed the binding lingers to its TTL, and every Continue in
|
|
128
|
+
// that window refuses. Twelve such bindings had stacked up on one thread.
|
|
129
|
+
//
|
|
130
|
+
// It belongs here because it is transient BY CONSTRUCTION -- the binding expires on
|
|
131
|
+
// a clock -- and because it is COS's own bookkeeping, not a foreign process holding
|
|
132
|
+
// the thread. Queueing waits it out, and delivery re-runs the whole gate as always.
|
|
133
|
+
'native_target_busy',
|
|
134
|
+
// Same shape: a COS turn is mid-flight on this thread and will finish.
|
|
135
|
+
'native_turn_in_progress',
|
|
136
|
+
'live_desktop_process',
|
|
137
|
+
'thread_busy',
|
|
138
|
+
'binding_conflict',
|
|
139
|
+
'detector_unavailable',
|
|
140
|
+
'registry_unreadable',
|
|
141
|
+
'unverifiable_process_start',
|
|
142
|
+
'unverifiable_liveness_socket',
|
|
143
|
+
'probe_failed',
|
|
144
|
+
])
|
|
145
|
+
|
|
146
|
+
/** Can a turn refused for this reason be parked, or must it refuse now? */
|
|
147
|
+
export function queueableRefusal(reason: string | null | undefined): boolean {
|
|
148
|
+
return typeof reason === 'string' && QUEUEABLE_REFUSALS.has(reason)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** What the drainer observed about the thread this tick. */
|
|
152
|
+
export interface DrainObservation {
|
|
153
|
+
/** The full occupancy gate re-run. Delivery NEVER happens on a false. */
|
|
154
|
+
attachable: boolean
|
|
155
|
+
/**
|
|
156
|
+
* Did the holder's last transcript record end a turn (`result`, `turn_complete`)?
|
|
157
|
+
* The precise signal, when the provider writes one.
|
|
158
|
+
*/
|
|
159
|
+
turnEnded: boolean
|
|
160
|
+
/** The 30s transcript clock. `idle` is the backstop when no terminal record lands. */
|
|
161
|
+
activity: 'working' | 'idle' | 'unknown'
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export type DrainDecision = 'deliver' | 'hold' | 'expire' | 'give_up'
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Should this waiting turn go now?
|
|
168
|
+
*
|
|
169
|
+
* PURE, and every branch resolves. Order matters: expiry and the attempt ceiling are
|
|
170
|
+
* checked BEFORE readiness, so a turn that has run out of time or tries cannot be
|
|
171
|
+
* delivered by a lucky tick.
|
|
172
|
+
*/
|
|
173
|
+
export function drainDecision(
|
|
174
|
+
turn: Pick<QueuedThreadTurn, 'status' | 'queuedAt' | 'attempts'>,
|
|
175
|
+
seen: DrainObservation,
|
|
176
|
+
now: number,
|
|
177
|
+
ttlMs: number = QUEUED_TURN_TTL_MS,
|
|
178
|
+
): DrainDecision {
|
|
179
|
+
if (turn.status !== 'waiting') return 'hold'
|
|
180
|
+
if (!Number.isFinite(now) || !Number.isFinite(turn.queuedAt)) return 'hold'
|
|
181
|
+
if (now - turn.queuedAt >= ttlMs) return 'expire'
|
|
182
|
+
if (turn.attempts >= MAX_DELIVERY_ATTEMPTS) return 'give_up'
|
|
183
|
+
// THE GATE, unweakened. Everything below is about WHEN, never about whether.
|
|
184
|
+
if (!seen.attachable) return 'hold'
|
|
185
|
+
// Turn-ended is the precise signal; idle is the backstop for a holder that wrote no
|
|
186
|
+
// terminal record. `working` holds even when attachable, because attachable only says
|
|
187
|
+
// no one else owns it -- it does not say a turn is not mid-flight.
|
|
188
|
+
if (seen.turnEnded) return 'deliver'
|
|
189
|
+
return seen.activity === 'idle' ? 'deliver' : 'hold'
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Admit a turn to the queue, or say why not.
|
|
194
|
+
*
|
|
195
|
+
* Rejects a duplicate `clientTurnId` rather than parking a second copy: the id is an
|
|
196
|
+
* idempotency key, and a phone that retries after a dropped response must not put the
|
|
197
|
+
* same sentence into a real conversation twice.
|
|
198
|
+
*/
|
|
199
|
+
export function admitToQueue(
|
|
200
|
+
existing: readonly QueuedThreadTurn[],
|
|
201
|
+
turn: QueuedThreadTurn,
|
|
202
|
+
): { ok: true; queue: QueuedThreadTurn[] } | { ok: false; reason: string } {
|
|
203
|
+
if (!turn.clientTurnId || !turn.prompt.trim()) return { ok: false, reason: 'invalid_request' }
|
|
204
|
+
if (existing.some(t => t.clientTurnId === turn.clientTurnId)) {
|
|
205
|
+
return { ok: false, reason: 'duplicate_turn' }
|
|
206
|
+
}
|
|
207
|
+
const waiting = existing.filter(t => t.status === 'waiting')
|
|
208
|
+
if (waiting.length >= MAX_QUEUED_PER_THREAD) return { ok: false, reason: 'queue_full' }
|
|
209
|
+
return { ok: true, queue: [...existing, turn] }
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Drop rows nobody needs any more.
|
|
214
|
+
*
|
|
215
|
+
* Settled rows are kept briefly so the phone's poll can still see the outcome -- a row
|
|
216
|
+
* that vanishes reads as "lost", which is the one answer a pending ledger must never
|
|
217
|
+
* give. Waiting rows are never pruned here; only `drainDecision` retires those, so
|
|
218
|
+
* there is exactly one place a live turn can die.
|
|
219
|
+
*/
|
|
220
|
+
export function pruneQueue(
|
|
221
|
+
queue: readonly QueuedThreadTurn[],
|
|
222
|
+
now: number,
|
|
223
|
+
settledRetentionMs: number = 30 * 60 * 1000,
|
|
224
|
+
): QueuedThreadTurn[] {
|
|
225
|
+
return queue.filter(t => {
|
|
226
|
+
if (t.status === 'waiting' || t.status === 'delivering') return true
|
|
227
|
+
const settled = typeof t.settledAt === 'number' ? t.settledAt : t.queuedAt
|
|
228
|
+
return now - settled < settledRetentionMs
|
|
229
|
+
})
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Where a waiting turn sits, for the phone's row copy. `0` when it is next. */
|
|
233
|
+
export function queuePosition(queue: readonly QueuedThreadTurn[], clientTurnId: string): number {
|
|
234
|
+
return queue.filter(t => t.status === 'waiting').findIndex(t => t.clientTurnId === clientTurnId)
|
|
235
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
// Routes and drainer for turns spoken at a busy thread.
|
|
2
|
+
//
|
|
3
|
+
// WHY THE DRAINER CALLS THE SERVER'S OWN ROUTES OVER LOOPBACK, which looks odd until
|
|
4
|
+
// you look at what it would otherwise have to copy. Delivery is not "post a prompt": it
|
|
5
|
+
// is the occupancy gate, the target fence, the per-target claim, the divergence
|
|
6
|
+
// watermark, the idempotency ledger, and the child-pid accounting -- roughly 370 lines
|
|
7
|
+
// of ordering in agent-session-bindings.ts, where every branch exists because something
|
|
8
|
+
// once went wrong. A second copy of that in a background worker is a second place for
|
|
9
|
+
// the gate to drift, and the drift would be invisible until it mattered.
|
|
10
|
+
//
|
|
11
|
+
// So the drainer re-enters through the front door. The gate runs exactly once, in its
|
|
12
|
+
// existing home, and this file cannot weaken it even by accident -- the strongest form
|
|
13
|
+
// of "does not weaken the attach gate" available. It costs one loopback request per
|
|
14
|
+
// delivered turn, at a volume of a handful per day.
|
|
15
|
+
//
|
|
16
|
+
// EVERY DEPENDENCY IS INJECTED so the whole path is testable without a live server.
|
|
17
|
+
|
|
18
|
+
import { Router, type Request, type Response } from 'express'
|
|
19
|
+
import {
|
|
20
|
+
admitToQueue, drainDecision, queueableRefusal, queuePosition,
|
|
21
|
+
MAX_DELIVERY_ATTEMPTS, type DrainObservation, type QueuedThreadTurn,
|
|
22
|
+
} from '../lib/thread-turn-queue.js'
|
|
23
|
+
import { readQueue, writeQueue, queuedThreadKeys } from '../lib/thread-turn-queue-store.js'
|
|
24
|
+
|
|
25
|
+
export interface ThreadTurnQueueDeps {
|
|
26
|
+
/** Re-runs the FULL occupancy gate. The drainer never decides attachability itself. */
|
|
27
|
+
occupancy: (provider: string, threadId: string) => { attachable: boolean; reason: string | null }
|
|
28
|
+
/** Did the holder's last transcript record end a turn? */
|
|
29
|
+
turnEnded: (provider: string, threadId: string) => boolean
|
|
30
|
+
/** The 30s transcript clock, as a backstop. */
|
|
31
|
+
activity: (provider: string, threadId: string) => 'working' | 'idle' | 'unknown'
|
|
32
|
+
/**
|
|
33
|
+
* Deliver one turn. Production wires this to a loopback attach + turn.
|
|
34
|
+
*
|
|
35
|
+
* Resolves `{ ok: true }` only when the turn was provably admitted. Anything
|
|
36
|
+
* ambiguous must resolve `ok: false` with a reason: an unknown delivery that is
|
|
37
|
+
* retried puts the same sentence into a real conversation twice.
|
|
38
|
+
*/
|
|
39
|
+
deliver: (turn: QueuedThreadTurn) => Promise<{ ok: boolean; reason?: string }>
|
|
40
|
+
now: () => number
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Public shape of a queued turn. No prompt echo beyond a short preview. */
|
|
44
|
+
function publicRow(turn: QueuedThreadTurn, position: number): Record<string, unknown> {
|
|
45
|
+
return {
|
|
46
|
+
clientTurnId: turn.clientTurnId,
|
|
47
|
+
status: turn.status,
|
|
48
|
+
queuedAt: turn.queuedAt,
|
|
49
|
+
attempts: turn.attempts,
|
|
50
|
+
position: turn.status === 'waiting' ? position : -1,
|
|
51
|
+
preview: turn.prompt.length > 80 ? `${turn.prompt.slice(0, 79)}…` : turn.prompt,
|
|
52
|
+
...(turn.reason ? { reason: turn.reason } : {}),
|
|
53
|
+
...(turn.settledAt ? { settledAt: turn.settledAt } : {}),
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* One drain pass over one thread.
|
|
59
|
+
*
|
|
60
|
+
* Exported for the test and for the sweep. Sequential by design: two turns to the same
|
|
61
|
+
* thread must not race, and the second one's readiness is decided AFTER the first has
|
|
62
|
+
* landed, because the first makes the thread busy again.
|
|
63
|
+
*/
|
|
64
|
+
export async function drainThread(
|
|
65
|
+
provider: string,
|
|
66
|
+
threadId: string,
|
|
67
|
+
deps: ThreadTurnQueueDeps,
|
|
68
|
+
): Promise<{ delivered: number; held: number; retired: number }> {
|
|
69
|
+
const now = deps.now()
|
|
70
|
+
const queue = readQueue(provider, threadId, now)
|
|
71
|
+
if (queue.length === 0) return { delivered: 0, held: 0, retired: 0 }
|
|
72
|
+
|
|
73
|
+
let delivered = 0, held = 0, retired = 0
|
|
74
|
+
let dirty = false
|
|
75
|
+
|
|
76
|
+
for (const turn of queue) {
|
|
77
|
+
if (turn.status !== 'waiting') continue
|
|
78
|
+
|
|
79
|
+
// Observed FRESH for each turn: delivering one makes the thread busy again, so a
|
|
80
|
+
// verdict from the top of the loop would be stale by the second item.
|
|
81
|
+
const gate = deps.occupancy(provider, threadId)
|
|
82
|
+
const seen: DrainObservation = {
|
|
83
|
+
attachable: gate.attachable,
|
|
84
|
+
turnEnded: deps.turnEnded(provider, threadId),
|
|
85
|
+
activity: deps.activity(provider, threadId),
|
|
86
|
+
}
|
|
87
|
+
const decision = drainDecision(turn, seen, deps.now())
|
|
88
|
+
|
|
89
|
+
if (decision === 'hold') { held += 1; continue }
|
|
90
|
+
if (decision === 'expire' || decision === 'give_up') {
|
|
91
|
+
turn.status = decision === 'expire' ? 'expired' : 'refused'
|
|
92
|
+
turn.reason = decision === 'expire' ? 'queued_turn_expired' : 'delivery_attempts_exhausted'
|
|
93
|
+
turn.settledAt = deps.now()
|
|
94
|
+
retired += 1; dirty = true
|
|
95
|
+
continue
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ATTEMPT IS RECORDED BEFORE THE CALL, and persisted. A delivery that crashes the
|
|
99
|
+
// process mid-flight must not come back with its attempt count unchanged and try
|
|
100
|
+
// forever; the ceiling only bounds anything if it survives the crash it is bounding.
|
|
101
|
+
turn.attempts += 1
|
|
102
|
+
turn.status = 'delivering'
|
|
103
|
+
writeQueue(provider, threadId, queue)
|
|
104
|
+
dirty = true
|
|
105
|
+
|
|
106
|
+
let outcome: { ok: boolean; reason?: string }
|
|
107
|
+
try {
|
|
108
|
+
outcome = await deps.deliver(turn)
|
|
109
|
+
} catch (error) {
|
|
110
|
+
outcome = { ok: false, reason: error instanceof Error ? error.message : 'deliver_threw' }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (outcome.ok) {
|
|
114
|
+
turn.status = 'delivered'
|
|
115
|
+
turn.settledAt = deps.now()
|
|
116
|
+
delivered += 1
|
|
117
|
+
} else if (turn.attempts >= MAX_DELIVERY_ATTEMPTS) {
|
|
118
|
+
turn.status = 'refused'
|
|
119
|
+
turn.reason = outcome.reason ?? 'delivery_failed'
|
|
120
|
+
turn.settledAt = deps.now()
|
|
121
|
+
retired += 1
|
|
122
|
+
} else {
|
|
123
|
+
// Back to waiting for the next sweep. A failure that is not terminal is a
|
|
124
|
+
// failure to deliver NOW, not a failure of the turn.
|
|
125
|
+
turn.status = 'waiting'
|
|
126
|
+
turn.reason = outcome.reason
|
|
127
|
+
held += 1
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (dirty) writeQueue(provider, threadId, queue)
|
|
132
|
+
return { delivered, held, retired }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** One pass over every thread holding a queue. */
|
|
136
|
+
export async function drainAllThreads(deps: ThreadTurnQueueDeps): Promise<number> {
|
|
137
|
+
let delivered = 0
|
|
138
|
+
for (const { provider, threadId } of queuedThreadKeys()) {
|
|
139
|
+
try {
|
|
140
|
+
delivered += (await drainThread(provider, threadId, deps)).delivered
|
|
141
|
+
} catch (error) {
|
|
142
|
+
// One bad thread must not stop the sweep for every other thread.
|
|
143
|
+
console.error(`[thread-turn-queue] drain failed for ${provider}: ${error instanceof Error ? error.message : error}`)
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return delivered
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function createThreadTurnQueueRouter(deps: ThreadTurnQueueDeps): Router {
|
|
150
|
+
const router = Router()
|
|
151
|
+
|
|
152
|
+
// POST — park a turn for a thread that is busy right now.
|
|
153
|
+
router.post('/agent-sessions/:provider/:threadId/queued-turns', (req: Request, res: Response) => {
|
|
154
|
+
res.set('Cache-Control', 'private, no-store')
|
|
155
|
+
const provider = String(req.params.provider ?? '')
|
|
156
|
+
const threadId = String(req.params.threadId ?? '')
|
|
157
|
+
const body = (req.body ?? {}) as Record<string, unknown>
|
|
158
|
+
const clientTurnId = typeof body.clientTurnId === 'string' ? body.clientTurnId : ''
|
|
159
|
+
const cosSessionId = typeof body.cosSessionId === 'string' ? body.cosSessionId : ''
|
|
160
|
+
const prompt = typeof body.prompt === 'string' ? body.prompt : ''
|
|
161
|
+
if (!clientTurnId || !cosSessionId || !prompt.trim()) {
|
|
162
|
+
return res.status(400).json({ error: 'invalid_request' })
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// THE GATE DECIDES WHETHER PARKING IS EVEN HONEST. A structural refusal can never
|
|
166
|
+
// clear, and telling someone their turn is queued when it can never run is worse
|
|
167
|
+
// than refusing it. An attachable thread is not queued either -- it is sent now,
|
|
168
|
+
// through the ordinary route, which the client does on this 409.
|
|
169
|
+
const gate = deps.occupancy(provider, threadId)
|
|
170
|
+
if (gate.attachable) return res.status(409).json({ error: 'thread_free', hint: 'send_now' })
|
|
171
|
+
if (!queueableRefusal(gate.reason)) {
|
|
172
|
+
return res.status(423).json({ error: gate.reason ?? 'probe_failed', queueable: false })
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const now = deps.now()
|
|
176
|
+
const queue = readQueue(provider, threadId, now)
|
|
177
|
+
const admitted = admitToQueue(queue, {
|
|
178
|
+
clientTurnId, cosSessionId, provider, threadId, prompt,
|
|
179
|
+
queuedAt: now, status: 'waiting', attempts: 0,
|
|
180
|
+
})
|
|
181
|
+
if (!admitted.ok) return res.status(409).json({ error: admitted.reason })
|
|
182
|
+
|
|
183
|
+
writeQueue(provider, threadId, admitted.queue)
|
|
184
|
+
return res.status(202).json({
|
|
185
|
+
queued: true,
|
|
186
|
+
clientTurnId,
|
|
187
|
+
position: queuePosition(admitted.queue, clientTurnId),
|
|
188
|
+
waitingOn: gate.reason,
|
|
189
|
+
})
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
// GET — what is waiting, for the pending row on the lens.
|
|
193
|
+
router.get('/agent-sessions/:provider/:threadId/queued-turns', (req: Request, res: Response) => {
|
|
194
|
+
res.set('Cache-Control', 'private, no-store')
|
|
195
|
+
const provider = String(req.params.provider ?? '')
|
|
196
|
+
const threadId = String(req.params.threadId ?? '')
|
|
197
|
+
const queue = readQueue(provider, threadId, deps.now())
|
|
198
|
+
return res.json({ turns: queue.map(t => publicRow(t, queuePosition(queue, t.clientTurnId))) })
|
|
199
|
+
})
|
|
200
|
+
|
|
201
|
+
// DELETE — the cancel control, the same affordance the desktop queue offers.
|
|
202
|
+
router.delete('/agent-sessions/:provider/:threadId/queued-turns/:clientTurnId', (req: Request, res: Response) => {
|
|
203
|
+
res.set('Cache-Control', 'private, no-store')
|
|
204
|
+
const provider = String(req.params.provider ?? '')
|
|
205
|
+
const threadId = String(req.params.threadId ?? '')
|
|
206
|
+
const clientTurnId = String(req.params.clientTurnId ?? '')
|
|
207
|
+
const now = deps.now()
|
|
208
|
+
const queue = readQueue(provider, threadId, now)
|
|
209
|
+
const row = queue.find(t => t.clientTurnId === clientTurnId)
|
|
210
|
+
if (!row) return res.status(404).json({ error: 'unknown_turn' })
|
|
211
|
+
// A turn already handed to the adapter cannot be recalled, and saying otherwise
|
|
212
|
+
// would be the one lie this whole feature exists to avoid.
|
|
213
|
+
if (row.status === 'delivering') return res.status(409).json({ error: 'already_delivering' })
|
|
214
|
+
if (row.status === 'waiting') {
|
|
215
|
+
row.status = 'cancelled'
|
|
216
|
+
row.settledAt = now
|
|
217
|
+
writeQueue(provider, threadId, queue)
|
|
218
|
+
}
|
|
219
|
+
return res.json({ cancelled: true, clientTurnId, status: row.status })
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
return router
|
|
223
|
+
}
|