@gotcos/glasses-server 6.36.5 → 6.36.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/package.json +1 -1
- package/server/index.ts +68 -1
- package/server/lib/agent-session-store.ts +69 -4
- package/server/lib/thread-turn-queue-deliver.ts +106 -0
- package/server/lib/thread-turn-queue-store.ts +127 -0
- package/server/lib/thread-turn-queue.ts +222 -0
- package/server/routes/thread-turn-queue.ts +223 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,63 @@
|
|
|
1
1
|
## Unreleased
|
|
2
2
|
|
|
3
|
+
## 6.36.7
|
|
4
|
+
- **A turn spoken at a busy thread is now queued instead of refused.** Miles: "if
|
|
5
|
+
there's a session that's still running, that would just put it into the queue the same
|
|
6
|
+
way that the user has the ability to do so." The thread was never stuck — measured at
|
|
7
|
+
the moment he asked, its transcript mtime was 3s old against a 30s window, so the gate
|
|
8
|
+
correctly read `working`. That is the trap: while you are talking to an agent in a
|
|
9
|
+
thread, it is CONTINUOUSLY working, so Continue was unreachable for exactly the thread
|
|
10
|
+
you most want to continue and Fork was the only door.
|
|
11
|
+
- **This does not weaken the attach gate, which is the whole design.** The queue defers
|
|
12
|
+
to the gate rather than bypassing it: delivery re-runs the full occupancy check and
|
|
13
|
+
can still refuse. Several tests exist only to prove that negative.
|
|
14
|
+
- Only occupancy reasons are queueable. A structural refusal — unsupported provider,
|
|
15
|
+
malformed id, attach switched off — still refuses immediately, because telling someone
|
|
16
|
+
their turn is queued when it can never run is worse than refusing it.
|
|
17
|
+
- **Delivery re-enters through the front door**, over loopback to this server's own
|
|
18
|
+
attach and turn routes. Those carry the gate, the target fence, the per-target claim,
|
|
19
|
+
the watermark, the idempotency ledger and the child-pid accounting; a second copy in a
|
|
20
|
+
background worker would be a second place for the gate to drift.
|
|
21
|
+
- **The watermark exemption, approved explicitly by Miles.** A queued turn drains after
|
|
22
|
+
the thread has moved on — that is what it waited for — so its binding is minted fresh
|
|
23
|
+
at delivery. Checked as normal, a queue would fail 100% of the time. An interactive
|
|
24
|
+
turn still gets the full divergence check.
|
|
25
|
+
- Ready means the turn ENDED (`result` / `turn_complete`, detected with the same parser
|
|
26
|
+
the live stream uses), with the 30s idle clock as a backstop so a holder that dies
|
|
27
|
+
cannot wedge the queue. Thirty seconds of silence alone would fire during a long tool
|
|
28
|
+
call and inject into the middle of a turn.
|
|
29
|
+
- Durable under the data home, so it survives a server update and a pocketed phone. The
|
|
30
|
+
attempt count is persisted BEFORE each delivery, so a crash mid-flight cannot reset
|
|
31
|
+
the ceiling and retry forever. Six-hour TTL, 8 waiting per thread, 5 attempts, and a
|
|
32
|
+
cancel route for the × control.
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
## 6.36.6
|
|
36
|
+
- **The session digest follows the thread instead of its opening.** Miles, from the
|
|
37
|
+
lens: "It's currently showing a legacy session that I had over a day ago... The
|
|
38
|
+
discussion should show the questions that we're asking and a summary of those most
|
|
39
|
+
recent things, not something that's the 'first' message." The whole budget now goes
|
|
40
|
+
to recency. `DIGEST_HEAD_TURNS` drops from 2 to 0 — a deliberate reversal of the
|
|
41
|
+
earlier rule that reserved the opening ask because it "frames everything after it".
|
|
42
|
+
The opening is not lost: `first_prompt` still carries it in full.
|
|
43
|
+
- **Harness-injected rows no longer render as things you asked.** A slash-command body
|
|
44
|
+
and a compaction preamble are both written as USER rows, so both appeared in the
|
|
45
|
+
DISCUSSION list. Filtered on the STRUCTURAL flags Claude already sets — `isMeta` and
|
|
46
|
+
`isCompactSummary` — not on a markdown heuristic that could misfire on a real paste.
|
|
47
|
+
The compaction preamble is also filtered by text as a fallback for providers that
|
|
48
|
+
emit no flag, and that filter reaches titles and the search index too: the preamble
|
|
49
|
+
is byte-identical across every compacted session, so as a title or a search hit it
|
|
50
|
+
distinguishes nothing.
|
|
51
|
+
- **On a truncated read the head window no longer feeds the recency list.** The head
|
|
52
|
+
window IS the session opening, and on a large session the 60-turn window never fills,
|
|
53
|
+
so those turns survived at the top of the digest indefinitely. Measured on a real
|
|
54
|
+
94 MiB session: both leading bullets were head-window turns from the previous day.
|
|
55
|
+
A whole-file read is untouched — it yields `tail: true` for every line.
|
|
56
|
+
- Verified by parsing real transcripts, not fixtures: the 94 MiB session now leads with
|
|
57
|
+
the two most recent asks, and an ordinary 2.7 MiB session renders nine recent turns
|
|
58
|
+
in reading order.
|
|
59
|
+
|
|
60
|
+
|
|
3
61
|
## 6.36.5
|
|
4
62
|
- **A speaker merge now reaches the meetings, not just the voice store.**
|
|
5
63
|
`merge-profiles` folded two profiles together and relabelled the calibration log,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotcos/glasses-server",
|
|
3
|
-
"version": "6.36.
|
|
3
|
+
"version": "6.36.7",
|
|
4
4
|
"description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/server/index.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { createAttachedTurnStream } from './lib/session-stream-producer.js'
|
|
|
24
24
|
import { claudeSessionsRouter } from './routes/claude-sessions.js'
|
|
25
25
|
import {
|
|
26
26
|
createAgentSessionBindingsRouter,
|
|
27
|
+
|
|
27
28
|
threadAttachEnabled,
|
|
28
29
|
} from './routes/agent-session-bindings.js'
|
|
29
30
|
import { AgentSessionBindingRegistry } from './lib/agent-session-binding-registry.js'
|
|
@@ -33,7 +34,7 @@ import { realAttachedWorkspaceDeps, resolveAttachedWorkspace } from './lib/attac
|
|
|
33
34
|
import { deliverAttachedTurn, realAttachedTurnDeps } from './lib/attached-provider-adapter.js'
|
|
34
35
|
import { forkThread, realForkDeps } from './lib/fork-thread.js'
|
|
35
36
|
import { nativeHead, realNativeHeadDeps } from './lib/native-head.js'
|
|
36
|
-
import { threadOccupancy } from './lib/thread-occupancy.js'
|
|
37
|
+
import { threadOccupancy, holderActivity } from './lib/thread-occupancy.js'
|
|
37
38
|
import { displayRouter } from './routes/display.js'
|
|
38
39
|
import { transcribeStreamRouter } from './routes/transcribe-stream.js'
|
|
39
40
|
import { meetingRouter, resumeMeetingFinalizationJobs } from './routes/meeting.js'
|
|
@@ -104,6 +105,12 @@ import {
|
|
|
104
105
|
} from './lib/maintenance-lifecycle.js'
|
|
105
106
|
|
|
106
107
|
const app = express()
|
|
108
|
+
import { createThreadTurnQueueRouter, drainAllThreads } from './routes/thread-turn-queue.js'
|
|
109
|
+
import { transcriptTurnEnded } from './lib/thread-turn-queue-store.js'
|
|
110
|
+
import { transcriptPathFor } from './lib/native-head.js'
|
|
111
|
+
import { deliverQueuedTurnOverLoopback } from './lib/thread-turn-queue-deliver.js'
|
|
112
|
+
import type { QueuedThreadTurn } from './lib/thread-turn-queue.js'
|
|
113
|
+
|
|
107
114
|
const PORT = parseInt(process.env.PORT ?? '3141', 10)
|
|
108
115
|
|
|
109
116
|
// Mode detection — COS mode when a full pipeline directory is configured.
|
|
@@ -491,6 +498,66 @@ app.use('/api', claudeSessionsRouter)
|
|
|
491
498
|
// Registered AFTER agentSessionsRouter deliberately: its paths are 2 and 4 segments
|
|
492
499
|
// (`/agent-sessions/bindings`, `/agent-sessions/:provider/:threadId/attachability`)
|
|
493
500
|
// and cannot shadow that router's `/agent-sessions/:provider/:id` transcript route.
|
|
501
|
+
// ---------------------------------------------------------------------------
|
|
502
|
+
// QUEUED THREAD TURNS
|
|
503
|
+
// ---------------------------------------------------------------------------
|
|
504
|
+
// A turn spoken at a thread that is busy right now is PARKED instead of refused, and
|
|
505
|
+
// delivered when the thread frees. Miles, 2026-08-17: "if there's a session that's
|
|
506
|
+
// still running, that would just put it into the queue the same way that the user has
|
|
507
|
+
// the ability to do so."
|
|
508
|
+
//
|
|
509
|
+
// Registered only when attach is enabled, on the SAME flag as the write routes: a
|
|
510
|
+
// queue whose delivery path does not exist would accept turns it can never send.
|
|
511
|
+
//
|
|
512
|
+
// DELIVERY GOES BACK IN THROUGH THE FRONT DOOR, over loopback to this server's own
|
|
513
|
+
// attach + turn routes. Those two carry the occupancy gate, the target fence, the
|
|
514
|
+
// per-target claim, the divergence watermark, the idempotency ledger and the child-pid
|
|
515
|
+
// accounting; a second copy of that sequence in a background worker is a second place
|
|
516
|
+
// for the gate to drift. One request per delivered turn, at a handful of turns a day,
|
|
517
|
+
// buys the guarantee that the queue CANNOT weaken the gate even by accident.
|
|
518
|
+
if (threadAttachEnabled()) {
|
|
519
|
+
const queueDeps = {
|
|
520
|
+
occupancy: (provider: string, threadId: string) => {
|
|
521
|
+
try {
|
|
522
|
+
const v = threadOccupancy(provider, threadId, occupancyProbes, occupancyDirs)
|
|
523
|
+
return { attachable: v.attachable === true, reason: v.reason ?? null }
|
|
524
|
+
} catch {
|
|
525
|
+
// A throwing probe is not an open door.
|
|
526
|
+
return { attachable: false, reason: 'probe_failed' }
|
|
527
|
+
}
|
|
528
|
+
},
|
|
529
|
+
turnEnded: (provider: string, threadId: string) => {
|
|
530
|
+
if (provider !== 'claude' && provider !== 'codex') return false
|
|
531
|
+
try {
|
|
532
|
+
return transcriptTurnEnded(provider, transcriptPathFor(provider, threadId, nativeHeadDeps))
|
|
533
|
+
} catch {
|
|
534
|
+
return false
|
|
535
|
+
}
|
|
536
|
+
},
|
|
537
|
+
activity: (provider: string, threadId: string): 'working' | 'idle' | 'unknown' => {
|
|
538
|
+
try {
|
|
539
|
+
const read = occupancyProbes.transcriptMtimeMs
|
|
540
|
+
return typeof read === 'function'
|
|
541
|
+
? holderActivity(read(provider as 'claude' | 'codex', threadId), Date.now())
|
|
542
|
+
: 'unknown'
|
|
543
|
+
} catch {
|
|
544
|
+
return 'unknown'
|
|
545
|
+
}
|
|
546
|
+
},
|
|
547
|
+
deliver: (turn: QueuedThreadTurn) => deliverQueuedTurnOverLoopback(turn, PORT, API_TOKEN),
|
|
548
|
+
now: () => Date.now(),
|
|
549
|
+
}
|
|
550
|
+
app.use('/api', createThreadTurnQueueRouter(queueDeps))
|
|
551
|
+
|
|
552
|
+
// Every 20s. Fast enough that a freed thread drains while the user is still looking
|
|
553
|
+
// at the pending row, slow enough to be nothing: the sweep does no work at all when
|
|
554
|
+
// no queue file exists. Unref'd so it never holds the process open.
|
|
555
|
+
const queueDrainTimer = setInterval(() => {
|
|
556
|
+
void drainAllThreads(queueDeps).catch(() => { /* the next sweep retries */ })
|
|
557
|
+
}, 20_000)
|
|
558
|
+
queueDrainTimer.unref()
|
|
559
|
+
}
|
|
560
|
+
|
|
494
561
|
app.use('/api', createAgentSessionBindingsRouter({
|
|
495
562
|
probes: occupancyProbes,
|
|
496
563
|
dirs: occupancyDirs,
|
|
@@ -103,6 +103,22 @@ export function isKeepWarmSessionTitle(title: string): boolean {
|
|
|
103
103
|
return t.startsWith('this is an automated local readiness check')
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
+
/**
|
|
107
|
+
* The compaction preamble, which the harness writes as a USER turn.
|
|
108
|
+
*
|
|
109
|
+
* Miles's 2026-08-17 screenshot had this sitting in the DISCUSSION list as though he
|
|
110
|
+
* had asked it: "This session is being continued from a previous conversation… Summary:
|
|
111
|
+
* ## 1. Primary Request and Intent". It is scaffolding, and it lands in the digest at
|
|
112
|
+
* full length because it IS recent -- compaction happens mid-session -- so recency
|
|
113
|
+
* ordering alone does not remove it.
|
|
114
|
+
*
|
|
115
|
+
* Measured across the 25 most recently written transcripts on this machine: 29
|
|
116
|
+
* occurrences, exactly ONE distinct opening. Anchored to that opening rather than to a
|
|
117
|
+
* loose "continued from" match, so a human sentence about continuing a conversation is
|
|
118
|
+
* not mistaken for it.
|
|
119
|
+
*/
|
|
120
|
+
const COMPACTION_PREAMBLE = /^this session is being continued from a previous conversation/i
|
|
121
|
+
|
|
106
122
|
export function isWrapperPrompt(text: string): boolean {
|
|
107
123
|
const trimmed = text.trim()
|
|
108
124
|
return trimmed.startsWith('<')
|
|
@@ -110,6 +126,10 @@ export function isWrapperPrompt(text: string): boolean {
|
|
|
110
126
|
|| trimmed.startsWith('You are an agent')
|
|
111
127
|
|| trimmed.startsWith('You are QA Agent')
|
|
112
128
|
|| trimmed.startsWith('Message Type:')
|
|
129
|
+
// Filtered HERE rather than only in the digest, so it also stops titling a session
|
|
130
|
+
// and stops being indexed for search. It is byte-identical across every compacted
|
|
131
|
+
// session, so as a title or a search hit it distinguishes nothing.
|
|
132
|
+
|| COMPACTION_PREAMBLE.test(trimmed)
|
|
113
133
|
}
|
|
114
134
|
|
|
115
135
|
export function firstLineTitle(text: string): string {
|
|
@@ -249,7 +269,27 @@ export const DISCUSSION_DIGEST_MAX = 2000
|
|
|
249
269
|
const DIGEST_TURN_MAX = 220
|
|
250
270
|
|
|
251
271
|
/** Turns kept from the START. The opening ask frames everything after it. */
|
|
252
|
-
|
|
272
|
+
/**
|
|
273
|
+
* User turns reserved from the START of the session before recency gets the budget.
|
|
274
|
+
*
|
|
275
|
+
* ZERO, AND THAT IS A REVERSAL. This was 2, on the reasoning that the opening ask
|
|
276
|
+
* "frames everything after it" -- see the head-first block in composeDiscussionDigest,
|
|
277
|
+
* which was itself a fix for a digest that spent everything on the tail.
|
|
278
|
+
*
|
|
279
|
+
* Miles overruled it from hardware, 2026-08-17, looking at a digest whose first two
|
|
280
|
+
* bullets were a question from the previous day and a compaction preamble: "It needs to
|
|
281
|
+
* refresh to the most recent response inside of the thread. The discussion should show
|
|
282
|
+
* the questions that we're asking and a summary of those most recent things, not
|
|
283
|
+
* something that's the 'first' message."
|
|
284
|
+
*
|
|
285
|
+
* The opening ask is NOT lost -- it is still carried, in full, in `first_prompt`, which
|
|
286
|
+
* the digest and the row label both fall back to. What changes is that it stops
|
|
287
|
+
* occupying the two most valuable slots in a list about what is happening NOW.
|
|
288
|
+
*
|
|
289
|
+
* Left as a constant rather than deleting the head path: this is a judgement call about
|
|
290
|
+
* emphasis, and one number is the whole reversal if Miles wants some framing back.
|
|
291
|
+
*/
|
|
292
|
+
const DIGEST_HEAD_TURNS = 0
|
|
253
293
|
|
|
254
294
|
/** Recent turns retained while streaming. Comfortably more than 2000 chars can
|
|
255
295
|
* render (~20-25 at typical length), so the budget and not the buffer decides
|
|
@@ -1341,7 +1381,32 @@ export async function parseAgentSession(
|
|
|
1341
1381
|
// reports what was actually dropped rather than what this buffer happens to hold.
|
|
1342
1382
|
const digestHead: string[] = []
|
|
1343
1383
|
const digestRecent: string[] = []
|
|
1344
|
-
|
|
1384
|
+
/**
|
|
1385
|
+
* Add a user turn to the recency digest -- or decline to.
|
|
1386
|
+
*
|
|
1387
|
+
* TWO DECLINES, both found by parsing Miles's own 94 MiB session rather than by
|
|
1388
|
+
* reasoning about the code.
|
|
1389
|
+
*
|
|
1390
|
+
* `injected`: STRUCTURAL, not a heuristic. Claude marks a slash-command body with
|
|
1391
|
+
* `isMeta: true` and a compaction preamble with `isCompactSummary: true`. Both are
|
|
1392
|
+
* written as USER rows, so both rendered in the digest as things Miles had asked --
|
|
1393
|
+
* his screenshot led with "This session is being continued from a previous
|
|
1394
|
+
* conversation… Summary: ## 1. Primary Request and Intent". Neither is an ask, and
|
|
1395
|
+
* both are recent, so recency ordering alone leaves them exactly where they were.
|
|
1396
|
+
*
|
|
1397
|
+
* `fromTail` on a TRUNCATED read: the head window IS the session opening. Feeding it
|
|
1398
|
+
* into a recency list is the very thing Miles objected to -- "not something that's
|
|
1399
|
+
* the 'first' message" -- and on a large session the 60-turn window never fills, so
|
|
1400
|
+
* those opening turns survive to the top of the digest forever. Measured on his
|
|
1401
|
+
* session: the two leading bullets were both head-window turns from the previous day.
|
|
1402
|
+
* The head is still read and still published, as `first_prompt`.
|
|
1403
|
+
*
|
|
1404
|
+
* A whole-file read yields `tail: true` for every line, so a normal session is
|
|
1405
|
+
* unaffected -- this narrows only the partial read.
|
|
1406
|
+
*/
|
|
1407
|
+
const collectTurn = (text: string, fromTail = true, injected = false): void => {
|
|
1408
|
+
if (injected) return
|
|
1409
|
+
if (truncated && !fromTail) return
|
|
1345
1410
|
// Reuse the wrapper filter the rest of this module already trusts. Without it the
|
|
1346
1411
|
// opening turns of a slash-command session render as
|
|
1347
1412
|
// "<command-message>cos-glasses</command-message>…" — scaffolding, not the ask,
|
|
@@ -1413,7 +1478,7 @@ export async function parseAgentSession(
|
|
|
1413
1478
|
}
|
|
1414
1479
|
if (obj.type === 'user') {
|
|
1415
1480
|
userCount += 1
|
|
1416
|
-
collectTurn(text)
|
|
1481
|
+
collectTurn(text, tail, obj.isMeta === true || obj.isCompactSummary === true)
|
|
1417
1482
|
if (!firstPrompt) firstPrompt = firstLineTitle(text)
|
|
1418
1483
|
if (!title) title = firstLineTitle(text)
|
|
1419
1484
|
} else {
|
|
@@ -1459,7 +1524,7 @@ export async function parseAgentSession(
|
|
|
1459
1524
|
if (obj.role === 'user') {
|
|
1460
1525
|
userCount += 1
|
|
1461
1526
|
const query = cursorUserTitle(text, true) ?? cursorUserTitle(text, false)
|
|
1462
|
-
collectTurn(query ?? text)
|
|
1527
|
+
collectTurn(query ?? text, tail)
|
|
1463
1528
|
if (query) {
|
|
1464
1529
|
title = query
|
|
1465
1530
|
if (!firstPrompt) firstPrompt = query
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Delivering a queued turn by re-entering this server's own attach + turn routes.
|
|
2
|
+
//
|
|
3
|
+
// WHY LOOPBACK AND NOT A DIRECT CALL. Delivery is not "post a prompt". It is the
|
|
4
|
+
// occupancy gate, the target fence, the per-target claim, the divergence watermark, the
|
|
5
|
+
// idempotency ledger and the child-pid accounting -- a long, ordered sequence in
|
|
6
|
+
// agent-session-bindings.ts where every branch exists because something once went
|
|
7
|
+
// wrong. Re-implementing it here would create a second place for the gate to drift, and
|
|
8
|
+
// the drift would be invisible until the day it mattered. Going back in through the
|
|
9
|
+
// front door means the gate runs exactly once, in its existing home, and this file
|
|
10
|
+
// cannot weaken it even by accident.
|
|
11
|
+
//
|
|
12
|
+
// The cost is one loopback request per delivered turn, at a volume of a handful a day.
|
|
13
|
+
//
|
|
14
|
+
// THE WATERMARK EXEMPTION LIVES HERE, and nowhere else. A queued turn drains after the
|
|
15
|
+
// thread has moved on -- that is what it waited for -- so its binding is minted FRESH
|
|
16
|
+
// at delivery against the thread as it stands. The turn is therefore always composed
|
|
17
|
+
// against a current baseline, which is what the user meant by queueing it: "append this
|
|
18
|
+
// to whatever the thread is when it frees". An interactive turn still gets the full
|
|
19
|
+
// divergence check, because there the user composed against a state they were looking
|
|
20
|
+
// at and a silent change is genuinely surprising. Miles approved this explicitly.
|
|
21
|
+
//
|
|
22
|
+
// FAILS CLOSED, AND AMBIGUITY IS A FAILURE. Anything other than a clean admission
|
|
23
|
+
// resolves `ok: false`, which returns the turn to the queue rather than marking it
|
|
24
|
+
// sent. The caller bounds retries; the danger to avoid here is the opposite one --
|
|
25
|
+
// reporting success for a turn that may not have landed, which loses it silently.
|
|
26
|
+
|
|
27
|
+
import { request } from 'node:http'
|
|
28
|
+
import type { QueuedThreadTurn } from './thread-turn-queue.js'
|
|
29
|
+
|
|
30
|
+
/** Per-request ceiling. Attach and turn both answer immediately; the turn route
|
|
31
|
+
* admits with 202 and does the long work in the background. */
|
|
32
|
+
const DELIVER_TIMEOUT_MS = 15_000
|
|
33
|
+
|
|
34
|
+
interface LoopbackReply { status: number; body: Record<string, unknown> }
|
|
35
|
+
|
|
36
|
+
function post(port: number, token: string, path: string, payload: unknown): Promise<LoopbackReply> {
|
|
37
|
+
return new Promise((resolve, reject) => {
|
|
38
|
+
const data = JSON.stringify(payload)
|
|
39
|
+
const req = request({
|
|
40
|
+
host: '127.0.0.1',
|
|
41
|
+
port,
|
|
42
|
+
path,
|
|
43
|
+
method: 'POST',
|
|
44
|
+
headers: {
|
|
45
|
+
'content-type': 'application/json',
|
|
46
|
+
'content-length': Buffer.byteLength(data),
|
|
47
|
+
// The header the rest of COS authenticates with. NOT `Authorization: Bearer`.
|
|
48
|
+
'X-Cos-Token': token,
|
|
49
|
+
},
|
|
50
|
+
timeout: DELIVER_TIMEOUT_MS,
|
|
51
|
+
}, res => {
|
|
52
|
+
let raw = ''
|
|
53
|
+
res.on('data', c => { raw += c })
|
|
54
|
+
res.on('end', () => {
|
|
55
|
+
let body: Record<string, unknown> = {}
|
|
56
|
+
try { body = raw ? JSON.parse(raw) as Record<string, unknown> : {} } catch { /* non-JSON is a failure below */ }
|
|
57
|
+
resolve({ status: res.statusCode ?? 0, body })
|
|
58
|
+
})
|
|
59
|
+
})
|
|
60
|
+
req.on('timeout', () => { req.destroy(new Error('deliver_timeout')) })
|
|
61
|
+
req.on('error', reject)
|
|
62
|
+
req.write(data)
|
|
63
|
+
req.end()
|
|
64
|
+
})
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Attach, then send. Returns `ok` only on a provable admission.
|
|
69
|
+
*
|
|
70
|
+
* The attach is what re-baselines the watermark: it reads the thread's head digest as
|
|
71
|
+
* it is NOW, so the turn that follows cannot be refused for divergence that happened
|
|
72
|
+
* while the turn was waiting.
|
|
73
|
+
*/
|
|
74
|
+
export async function deliverQueuedTurnOverLoopback(
|
|
75
|
+
turn: QueuedThreadTurn,
|
|
76
|
+
port: number,
|
|
77
|
+
token: string,
|
|
78
|
+
): Promise<{ ok: boolean; reason?: string }> {
|
|
79
|
+
try {
|
|
80
|
+
const attach = await post(
|
|
81
|
+
port, token,
|
|
82
|
+
`/api/agent-sessions/${encodeURIComponent(turn.provider)}/${encodeURIComponent(turn.threadId)}/attach`,
|
|
83
|
+
{ cosSessionId: turn.cosSessionId },
|
|
84
|
+
)
|
|
85
|
+
if (attach.status !== 200 && attach.status !== 201) {
|
|
86
|
+
// The gate said no at drain time. Not an error -- the queue holds and tries again.
|
|
87
|
+
return { ok: false, reason: String(attach.body.error ?? `attach_${attach.status}`) }
|
|
88
|
+
}
|
|
89
|
+
const bindingId = typeof attach.body.bindingId === 'string' ? attach.body.bindingId : ''
|
|
90
|
+
if (!bindingId) return { ok: false, reason: 'attach_no_binding' }
|
|
91
|
+
|
|
92
|
+
const sent = await post(
|
|
93
|
+
port, token,
|
|
94
|
+
`/api/agent-sessions/bindings/${encodeURIComponent(bindingId)}/turns`,
|
|
95
|
+
// `clientTurnId` is carried through unchanged so the turn route's own
|
|
96
|
+
// idempotency ledger recognises a re-delivery of the SAME turn. Without it a
|
|
97
|
+
// retry after an ambiguous response would put the sentence in twice.
|
|
98
|
+
{ clientTurnId: turn.clientTurnId, prompt: turn.prompt },
|
|
99
|
+
)
|
|
100
|
+
// 202 is the success shape: admitted, delivered in the background, poll the ledger.
|
|
101
|
+
if (sent.status === 202 || sent.status === 200) return { ok: true }
|
|
102
|
+
return { ok: false, reason: String(sent.body.error ?? `turn_${sent.status}`) }
|
|
103
|
+
} catch (error) {
|
|
104
|
+
return { ok: false, reason: error instanceof Error ? error.message : 'deliver_failed' }
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// Durable storage for queued thread turns, and the terminal-record probe.
|
|
2
|
+
//
|
|
3
|
+
// SEPARATE FROM THE DECISIONS. `thread-turn-queue.ts` is pure and holds every rule;
|
|
4
|
+
// this file only reads and writes. That split is what lets the rules be tested by
|
|
5
|
+
// execution instead of by reading them.
|
|
6
|
+
//
|
|
7
|
+
// ONE FILE PER THREAD, under the data home so it survives a server update -- the
|
|
8
|
+
// generation directory is replaced wholesale on every Update Server, and a queue that
|
|
9
|
+
// lived there would be silently emptied by a routine upgrade. Same lesson as the
|
|
10
|
+
// stranded voice profiles.
|
|
11
|
+
|
|
12
|
+
import { closeSync, constants, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, readSync } from 'node:fs'
|
|
13
|
+
import { join } from 'node:path'
|
|
14
|
+
import { atomicWriteFileSync } from './atomic-fs.js'
|
|
15
|
+
import { dataPath } from './data-dir.js'
|
|
16
|
+
import { draftsFromLine, type SessionStreamProvider } from './session-stream-events.js'
|
|
17
|
+
import { pruneQueue, type QueuedThreadTurn } from './thread-turn-queue.js'
|
|
18
|
+
|
|
19
|
+
/** Bytes of transcript tail read to decide whether the last turn ended. */
|
|
20
|
+
export const TURN_END_TAIL_BYTES = 64 * 1024
|
|
21
|
+
|
|
22
|
+
function queueDir(): string {
|
|
23
|
+
const dir = dataPath('thread-turn-queue')
|
|
24
|
+
try { mkdirSync(dir, { recursive: true }) } catch { /* the write below reports it */ }
|
|
25
|
+
return dir
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* One file per (provider, thread).
|
|
30
|
+
*
|
|
31
|
+
* The thread id is validated by the caller before it reaches here, but it is still
|
|
32
|
+
* sanitised: this value becomes a PATH, and a route that forgets its guard must not
|
|
33
|
+
* turn into a directory traversal.
|
|
34
|
+
*/
|
|
35
|
+
export function queuePath(provider: string, threadId: string): string {
|
|
36
|
+
const safe = `${provider}-${threadId}`.replace(/[^A-Za-z0-9._-]/g, '_')
|
|
37
|
+
return join(queueDir(), `${safe}.json`)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The queue for a thread, pruned. A missing or corrupt file reads as empty. */
|
|
41
|
+
export function readQueue(provider: string, threadId: string, now: number): QueuedThreadTurn[] {
|
|
42
|
+
const path = queuePath(provider, threadId)
|
|
43
|
+
if (!existsSync(path)) return []
|
|
44
|
+
try {
|
|
45
|
+
const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8'))
|
|
46
|
+
if (!Array.isArray(parsed)) return []
|
|
47
|
+
// Corrupt rows are dropped individually rather than discarding the whole queue:
|
|
48
|
+
// one bad record must not lose the other turns someone is waiting on.
|
|
49
|
+
const rows = parsed.filter((r): r is QueuedThreadTurn =>
|
|
50
|
+
!!r && typeof r === 'object'
|
|
51
|
+
&& typeof (r as QueuedThreadTurn).clientTurnId === 'string'
|
|
52
|
+
&& typeof (r as QueuedThreadTurn).prompt === 'string'
|
|
53
|
+
&& typeof (r as QueuedThreadTurn).queuedAt === 'number')
|
|
54
|
+
return pruneQueue(rows, now)
|
|
55
|
+
} catch {
|
|
56
|
+
return []
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Replace a thread's queue. Atomic, so a crash mid-write cannot truncate it. */
|
|
61
|
+
export function writeQueue(provider: string, threadId: string, queue: readonly QueuedThreadTurn[]): void {
|
|
62
|
+
atomicWriteFileSync(queuePath(provider, threadId), `${JSON.stringify(queue, null, 2)}\n`)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Every thread with a queue file, for the drain sweep. */
|
|
66
|
+
export function queuedThreadKeys(): Array<{ provider: string; threadId: string }> {
|
|
67
|
+
try {
|
|
68
|
+
return readdirSync(queueDir())
|
|
69
|
+
.filter(f => f.endsWith('.json'))
|
|
70
|
+
.map(f => {
|
|
71
|
+
const base = f.slice(0, -5)
|
|
72
|
+
const dash = base.indexOf('-')
|
|
73
|
+
return dash > 0
|
|
74
|
+
? { provider: base.slice(0, dash), threadId: base.slice(dash + 1) }
|
|
75
|
+
: null
|
|
76
|
+
})
|
|
77
|
+
.filter((v): v is { provider: string; threadId: string } => v !== null)
|
|
78
|
+
} catch {
|
|
79
|
+
return []
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Did the holder's last turn END?
|
|
85
|
+
*
|
|
86
|
+
* REUSES `draftsFromLine`, the same parser the live stream uses, so "the turn ended"
|
|
87
|
+
* means exactly here what it means there. Hand-rolling a second `type === 'result'`
|
|
88
|
+
* check is how two definitions of done drift apart.
|
|
89
|
+
*
|
|
90
|
+
* Reads a bounded tail, newest record wins. Returns false on any doubt -- an
|
|
91
|
+
* unreadable transcript is not evidence a turn finished, and false only means the
|
|
92
|
+
* queue HOLDS, which is always the safe answer.
|
|
93
|
+
*/
|
|
94
|
+
export function transcriptTurnEnded(provider: SessionStreamProvider, path: string | null): boolean {
|
|
95
|
+
if (!path || !existsSync(path)) return false
|
|
96
|
+
try {
|
|
97
|
+
// BOTH flags, and both are load-bearing -- hazard-invariants.test.ts enforces
|
|
98
|
+
// them and each is right on its own terms. O_NOFOLLOW: a symlinked `<id>.jsonl`
|
|
99
|
+
// could point at any file on disk and would be parsed here as a transcript.
|
|
100
|
+
// O_NONBLOCK: `openSync` on a FIFO with no writer NEVER RETURNS, and it is a
|
|
101
|
+
// synchronous syscall on Node's single thread, so one planted path would stop
|
|
102
|
+
// health, meeting save and transcribe-stream along with this drain. That one is
|
|
103
|
+
// recorded in the repo as three reproductions of the same bug, >34s to SIGKILL.
|
|
104
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
|
|
105
|
+
try {
|
|
106
|
+
const size = fstatSync(fd).size
|
|
107
|
+
const start = Math.max(0, size - TURN_END_TAIL_BYTES)
|
|
108
|
+
const buf = Buffer.alloc(size - start)
|
|
109
|
+
readSync(fd, buf, 0, buf.length, start)
|
|
110
|
+
const lines = buf.toString('utf-8').split('\n')
|
|
111
|
+
// The first line of a tail read is almost always a fragment.
|
|
112
|
+
if (start > 0) lines.shift()
|
|
113
|
+
let ended = false
|
|
114
|
+
for (const line of lines) {
|
|
115
|
+
if (!line.trim()) continue
|
|
116
|
+
for (const draft of draftsFromLine(provider, line)) {
|
|
117
|
+
if (draft.kind === 'status') ended = draft.state === 'done'
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return ended
|
|
121
|
+
} finally {
|
|
122
|
+
closeSync(fd)
|
|
123
|
+
}
|
|
124
|
+
} catch {
|
|
125
|
+
return false
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
// A turn spoken at a thread that is busy right now, held until it is not.
|
|
2
|
+
//
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// WHY
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
// Miles, 2026-08-17, looking at a thread he could not continue: "This thread, the
|
|
7
|
+
// COS-glasses server, has actually completed, but it's still locked. Ideally, what would
|
|
8
|
+
// happen is if there's a session that's still running, that would just put it into the
|
|
9
|
+
// queue the same way that the user has the ability to do so." His example was his own
|
|
10
|
+
// desktop: he typed into a running turn and Claude Code queued it, pending, with a
|
|
11
|
+
// cancel control.
|
|
12
|
+
//
|
|
13
|
+
// The thread was not stuck. Measured at the moment he asked: transcript mtime 3s old
|
|
14
|
+
// against a 30s window, so `holderActivity` was `working` -- correctly, because a COS
|
|
15
|
+
// session was writing to it. That is the trap: while you are talking to an agent in a
|
|
16
|
+
// thread, that thread is CONTINUOUSLY working, so Continue is unreachable for exactly
|
|
17
|
+
// the thread you most want to continue, and Fork is the only door.
|
|
18
|
+
//
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// THIS DOES NOT WEAKEN THE ATTACH GATE, AND THAT IS THE WHOLE DESIGN
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// The gate refuses to WRITE into a thread another process holds. A queue does not
|
|
23
|
+
// bypass it -- it defers to it. Nothing here decides a turn may be delivered; delivery
|
|
24
|
+
// re-runs the full occupancy check at drain time and can still refuse. So parking is
|
|
25
|
+
// safe by construction, and the only reasons that should still refuse outright are the
|
|
26
|
+
// ones that can NEVER clear.
|
|
27
|
+
//
|
|
28
|
+
// That is the split `queueableRefusal` encodes: an occupancy reason describes a
|
|
29
|
+
// condition that may pass, a structural reason describes a thread that will never be
|
|
30
|
+
// continuable no matter how long anyone waits. Telling someone their turn is queued
|
|
31
|
+
// when it can never run is worse than refusing.
|
|
32
|
+
//
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
// THE WATERMARK EXEMPTION -- Miles approved this explicitly
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
// A binding carries a content watermark, and a write is refused with
|
|
37
|
+
// `native_thread_changed` when the thread's head digest moved since the turn was
|
|
38
|
+
// composed. A queued turn drains AFTER more has been written, by definition -- that is
|
|
39
|
+
// what it was waiting for -- so the watermark has ALWAYS moved by then. Checked as
|
|
40
|
+
// normal, a queue would fail 100% of the time.
|
|
41
|
+
//
|
|
42
|
+
// So a drained turn re-baselines the watermark at delivery. This is a deliberate,
|
|
43
|
+
// scoped exemption, and its justification is the user's intent: queueing means "append
|
|
44
|
+
// this to whatever the thread is when it frees", which is exactly what the desktop
|
|
45
|
+
// queue does. It is NOT a general relaxation -- an interactive turn still gets the full
|
|
46
|
+
// divergence check, because there the user composed against a state they were looking
|
|
47
|
+
// at and a silent change is genuinely surprising.
|
|
48
|
+
//
|
|
49
|
+
// Miles, asked directly before this was built: "Yeah, this is a fine exemption."
|
|
50
|
+
//
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
// WHAT MAKES A TURN READY, AND WHY NOT "IDLE"
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
// Thirty seconds of transcript silence is the wrong trigger on its own: a long tool
|
|
55
|
+
// call goes quiet mid-turn, and draining there would inject a message into the middle
|
|
56
|
+
// of someone's reasoning. The precise signal is the turn ENDING, which is observable in
|
|
57
|
+
// the transcript itself -- Claude writes a `result` record, Codex a `task_complete` /
|
|
58
|
+
// `turn_complete`. The idle clock stays as a BACKSTOP for a holder that dies or a
|
|
59
|
+
// provider that writes no terminal record, so a queue cannot wedge forever on a missing
|
|
60
|
+
// event. Same reasoning as the session-trail handoff, one layer down.
|
|
61
|
+
|
|
62
|
+
/** Terminal-ish states a queued turn can reach. `waiting` is the only live one. */
|
|
63
|
+
export type QueuedTurnStatus =
|
|
64
|
+
| 'waiting'
|
|
65
|
+
| 'delivering'
|
|
66
|
+
| 'delivered'
|
|
67
|
+
| 'refused'
|
|
68
|
+
| 'expired'
|
|
69
|
+
| 'cancelled'
|
|
70
|
+
|
|
71
|
+
export interface QueuedThreadTurn {
|
|
72
|
+
/** Idempotency key, poll key, and row identity -- the same one the phone's pending
|
|
73
|
+
* ledger already tracks, so a queued turn needs no new client vocabulary. */
|
|
74
|
+
clientTurnId: string
|
|
75
|
+
cosSessionId: string
|
|
76
|
+
provider: string
|
|
77
|
+
threadId: string
|
|
78
|
+
prompt: string
|
|
79
|
+
queuedAt: number
|
|
80
|
+
status: QueuedTurnStatus
|
|
81
|
+
/** Delivery attempts made. Bounded so a permanently-refusing target cannot spin. */
|
|
82
|
+
attempts: number
|
|
83
|
+
/** The SERVER's reason for a terminal outcome. Never the client's wording. */
|
|
84
|
+
reason?: string
|
|
85
|
+
settledAt?: number
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* How long a waiting turn stays valid.
|
|
90
|
+
*
|
|
91
|
+
* Six hours, not indefinite. A turn is a thing someone SAID, and saying it into a
|
|
92
|
+
* thread twelve hours later is not what they meant -- the context they were replying to
|
|
93
|
+
* is gone. Expiry is reported, never silent.
|
|
94
|
+
*/
|
|
95
|
+
export const QUEUED_TURN_TTL_MS = 6 * 60 * 60 * 1000
|
|
96
|
+
|
|
97
|
+
/** Waiting turns per thread. Small: this is a queue, not a backlog. */
|
|
98
|
+
export const MAX_QUEUED_PER_THREAD = 8
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Delivery attempts before a turn is given up on.
|
|
102
|
+
*
|
|
103
|
+
* A drain that keeps failing is not going to start working, and a queue that retries
|
|
104
|
+
* forever is a write loop against someone else's session.
|
|
105
|
+
*/
|
|
106
|
+
export const MAX_DELIVERY_ATTEMPTS = 5
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Occupancy reasons a turn may WAIT on, versus ones that must refuse now.
|
|
110
|
+
*
|
|
111
|
+
* The distinction is whether the condition can ever pass. `native_thread_working` is
|
|
112
|
+
* the common one and obviously transient. The `unknown`-shaped reasons are included
|
|
113
|
+
* deliberately: they mean the scan could not SEE, which is a refusal for a write but
|
|
114
|
+
* not a reason to discard something the user said -- and delivery re-runs the gate, so
|
|
115
|
+
* a queue on an unmeasurable thread simply never drains and expires honestly.
|
|
116
|
+
*
|
|
117
|
+
* Everything absent from this set is STRUCTURAL: a provider COS cannot continue, a
|
|
118
|
+
* malformed id, or the write feature being switched off. None of those pass with time,
|
|
119
|
+
* and queueing against them would be a lie told politely.
|
|
120
|
+
*/
|
|
121
|
+
const QUEUEABLE_REFUSALS: ReadonlySet<string> = new Set([
|
|
122
|
+
'native_thread_working',
|
|
123
|
+
'live_desktop_process',
|
|
124
|
+
'thread_busy',
|
|
125
|
+
'binding_conflict',
|
|
126
|
+
'detector_unavailable',
|
|
127
|
+
'registry_unreadable',
|
|
128
|
+
'unverifiable_process_start',
|
|
129
|
+
'unverifiable_liveness_socket',
|
|
130
|
+
'probe_failed',
|
|
131
|
+
])
|
|
132
|
+
|
|
133
|
+
/** Can a turn refused for this reason be parked, or must it refuse now? */
|
|
134
|
+
export function queueableRefusal(reason: string | null | undefined): boolean {
|
|
135
|
+
return typeof reason === 'string' && QUEUEABLE_REFUSALS.has(reason)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** What the drainer observed about the thread this tick. */
|
|
139
|
+
export interface DrainObservation {
|
|
140
|
+
/** The full occupancy gate re-run. Delivery NEVER happens on a false. */
|
|
141
|
+
attachable: boolean
|
|
142
|
+
/**
|
|
143
|
+
* Did the holder's last transcript record end a turn (`result`, `turn_complete`)?
|
|
144
|
+
* The precise signal, when the provider writes one.
|
|
145
|
+
*/
|
|
146
|
+
turnEnded: boolean
|
|
147
|
+
/** The 30s transcript clock. `idle` is the backstop when no terminal record lands. */
|
|
148
|
+
activity: 'working' | 'idle' | 'unknown'
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export type DrainDecision = 'deliver' | 'hold' | 'expire' | 'give_up'
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Should this waiting turn go now?
|
|
155
|
+
*
|
|
156
|
+
* PURE, and every branch resolves. Order matters: expiry and the attempt ceiling are
|
|
157
|
+
* checked BEFORE readiness, so a turn that has run out of time or tries cannot be
|
|
158
|
+
* delivered by a lucky tick.
|
|
159
|
+
*/
|
|
160
|
+
export function drainDecision(
|
|
161
|
+
turn: Pick<QueuedThreadTurn, 'status' | 'queuedAt' | 'attempts'>,
|
|
162
|
+
seen: DrainObservation,
|
|
163
|
+
now: number,
|
|
164
|
+
ttlMs: number = QUEUED_TURN_TTL_MS,
|
|
165
|
+
): DrainDecision {
|
|
166
|
+
if (turn.status !== 'waiting') return 'hold'
|
|
167
|
+
if (!Number.isFinite(now) || !Number.isFinite(turn.queuedAt)) return 'hold'
|
|
168
|
+
if (now - turn.queuedAt >= ttlMs) return 'expire'
|
|
169
|
+
if (turn.attempts >= MAX_DELIVERY_ATTEMPTS) return 'give_up'
|
|
170
|
+
// THE GATE, unweakened. Everything below is about WHEN, never about whether.
|
|
171
|
+
if (!seen.attachable) return 'hold'
|
|
172
|
+
// Turn-ended is the precise signal; idle is the backstop for a holder that wrote no
|
|
173
|
+
// terminal record. `working` holds even when attachable, because attachable only says
|
|
174
|
+
// no one else owns it -- it does not say a turn is not mid-flight.
|
|
175
|
+
if (seen.turnEnded) return 'deliver'
|
|
176
|
+
return seen.activity === 'idle' ? 'deliver' : 'hold'
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Admit a turn to the queue, or say why not.
|
|
181
|
+
*
|
|
182
|
+
* Rejects a duplicate `clientTurnId` rather than parking a second copy: the id is an
|
|
183
|
+
* idempotency key, and a phone that retries after a dropped response must not put the
|
|
184
|
+
* same sentence into a real conversation twice.
|
|
185
|
+
*/
|
|
186
|
+
export function admitToQueue(
|
|
187
|
+
existing: readonly QueuedThreadTurn[],
|
|
188
|
+
turn: QueuedThreadTurn,
|
|
189
|
+
): { ok: true; queue: QueuedThreadTurn[] } | { ok: false; reason: string } {
|
|
190
|
+
if (!turn.clientTurnId || !turn.prompt.trim()) return { ok: false, reason: 'invalid_request' }
|
|
191
|
+
if (existing.some(t => t.clientTurnId === turn.clientTurnId)) {
|
|
192
|
+
return { ok: false, reason: 'duplicate_turn' }
|
|
193
|
+
}
|
|
194
|
+
const waiting = existing.filter(t => t.status === 'waiting')
|
|
195
|
+
if (waiting.length >= MAX_QUEUED_PER_THREAD) return { ok: false, reason: 'queue_full' }
|
|
196
|
+
return { ok: true, queue: [...existing, turn] }
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Drop rows nobody needs any more.
|
|
201
|
+
*
|
|
202
|
+
* Settled rows are kept briefly so the phone's poll can still see the outcome -- a row
|
|
203
|
+
* that vanishes reads as "lost", which is the one answer a pending ledger must never
|
|
204
|
+
* give. Waiting rows are never pruned here; only `drainDecision` retires those, so
|
|
205
|
+
* there is exactly one place a live turn can die.
|
|
206
|
+
*/
|
|
207
|
+
export function pruneQueue(
|
|
208
|
+
queue: readonly QueuedThreadTurn[],
|
|
209
|
+
now: number,
|
|
210
|
+
settledRetentionMs: number = 30 * 60 * 1000,
|
|
211
|
+
): QueuedThreadTurn[] {
|
|
212
|
+
return queue.filter(t => {
|
|
213
|
+
if (t.status === 'waiting' || t.status === 'delivering') return true
|
|
214
|
+
const settled = typeof t.settledAt === 'number' ? t.settledAt : t.queuedAt
|
|
215
|
+
return now - settled < settledRetentionMs
|
|
216
|
+
})
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Where a waiting turn sits, for the phone's row copy. `0` when it is next. */
|
|
220
|
+
export function queuePosition(queue: readonly QueuedThreadTurn[], clientTurnId: string): number {
|
|
221
|
+
return queue.filter(t => t.status === 'waiting').findIndex(t => t.clientTurnId === clientTurnId)
|
|
222
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
// Routes and drainer for turns spoken at a busy thread.
|
|
2
|
+
//
|
|
3
|
+
// WHY THE DRAINER CALLS THE SERVER'S OWN ROUTES OVER LOOPBACK, which looks odd until
|
|
4
|
+
// you look at what it would otherwise have to copy. Delivery is not "post a prompt": it
|
|
5
|
+
// is the occupancy gate, the target fence, the per-target claim, the divergence
|
|
6
|
+
// watermark, the idempotency ledger, and the child-pid accounting -- roughly 370 lines
|
|
7
|
+
// of ordering in agent-session-bindings.ts, where every branch exists because something
|
|
8
|
+
// once went wrong. A second copy of that in a background worker is a second place for
|
|
9
|
+
// the gate to drift, and the drift would be invisible until it mattered.
|
|
10
|
+
//
|
|
11
|
+
// So the drainer re-enters through the front door. The gate runs exactly once, in its
|
|
12
|
+
// existing home, and this file cannot weaken it even by accident -- the strongest form
|
|
13
|
+
// of "does not weaken the attach gate" available. It costs one loopback request per
|
|
14
|
+
// delivered turn, at a volume of a handful per day.
|
|
15
|
+
//
|
|
16
|
+
// EVERY DEPENDENCY IS INJECTED so the whole path is testable without a live server.
|
|
17
|
+
|
|
18
|
+
import { Router, type Request, type Response } from 'express'
|
|
19
|
+
import {
|
|
20
|
+
admitToQueue, drainDecision, queueableRefusal, queuePosition,
|
|
21
|
+
MAX_DELIVERY_ATTEMPTS, type DrainObservation, type QueuedThreadTurn,
|
|
22
|
+
} from '../lib/thread-turn-queue.js'
|
|
23
|
+
import { readQueue, writeQueue, queuedThreadKeys } from '../lib/thread-turn-queue-store.js'
|
|
24
|
+
|
|
25
|
+
export interface ThreadTurnQueueDeps {
|
|
26
|
+
/** Re-runs the FULL occupancy gate. The drainer never decides attachability itself. */
|
|
27
|
+
occupancy: (provider: string, threadId: string) => { attachable: boolean; reason: string | null }
|
|
28
|
+
/** Did the holder's last transcript record end a turn? */
|
|
29
|
+
turnEnded: (provider: string, threadId: string) => boolean
|
|
30
|
+
/** The 30s transcript clock, as a backstop. */
|
|
31
|
+
activity: (provider: string, threadId: string) => 'working' | 'idle' | 'unknown'
|
|
32
|
+
/**
|
|
33
|
+
* Deliver one turn. Production wires this to a loopback attach + turn.
|
|
34
|
+
*
|
|
35
|
+
* Resolves `{ ok: true }` only when the turn was provably admitted. Anything
|
|
36
|
+
* ambiguous must resolve `ok: false` with a reason: an unknown delivery that is
|
|
37
|
+
* retried puts the same sentence into a real conversation twice.
|
|
38
|
+
*/
|
|
39
|
+
deliver: (turn: QueuedThreadTurn) => Promise<{ ok: boolean; reason?: string }>
|
|
40
|
+
now: () => number
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Public shape of a queued turn. No prompt echo beyond a short preview. */
|
|
44
|
+
function publicRow(turn: QueuedThreadTurn, position: number): Record<string, unknown> {
|
|
45
|
+
return {
|
|
46
|
+
clientTurnId: turn.clientTurnId,
|
|
47
|
+
status: turn.status,
|
|
48
|
+
queuedAt: turn.queuedAt,
|
|
49
|
+
attempts: turn.attempts,
|
|
50
|
+
position: turn.status === 'waiting' ? position : -1,
|
|
51
|
+
preview: turn.prompt.length > 80 ? `${turn.prompt.slice(0, 79)}…` : turn.prompt,
|
|
52
|
+
...(turn.reason ? { reason: turn.reason } : {}),
|
|
53
|
+
...(turn.settledAt ? { settledAt: turn.settledAt } : {}),
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* One drain pass over one thread.
|
|
59
|
+
*
|
|
60
|
+
* Exported for the test and for the sweep. Sequential by design: two turns to the same
|
|
61
|
+
* thread must not race, and the second one's readiness is decided AFTER the first has
|
|
62
|
+
* landed, because the first makes the thread busy again.
|
|
63
|
+
*/
|
|
64
|
+
export async function drainThread(
|
|
65
|
+
provider: string,
|
|
66
|
+
threadId: string,
|
|
67
|
+
deps: ThreadTurnQueueDeps,
|
|
68
|
+
): Promise<{ delivered: number; held: number; retired: number }> {
|
|
69
|
+
const now = deps.now()
|
|
70
|
+
const queue = readQueue(provider, threadId, now)
|
|
71
|
+
if (queue.length === 0) return { delivered: 0, held: 0, retired: 0 }
|
|
72
|
+
|
|
73
|
+
let delivered = 0, held = 0, retired = 0
|
|
74
|
+
let dirty = false
|
|
75
|
+
|
|
76
|
+
for (const turn of queue) {
|
|
77
|
+
if (turn.status !== 'waiting') continue
|
|
78
|
+
|
|
79
|
+
// Observed FRESH for each turn: delivering one makes the thread busy again, so a
|
|
80
|
+
// verdict from the top of the loop would be stale by the second item.
|
|
81
|
+
const gate = deps.occupancy(provider, threadId)
|
|
82
|
+
const seen: DrainObservation = {
|
|
83
|
+
attachable: gate.attachable,
|
|
84
|
+
turnEnded: deps.turnEnded(provider, threadId),
|
|
85
|
+
activity: deps.activity(provider, threadId),
|
|
86
|
+
}
|
|
87
|
+
const decision = drainDecision(turn, seen, deps.now())
|
|
88
|
+
|
|
89
|
+
if (decision === 'hold') { held += 1; continue }
|
|
90
|
+
if (decision === 'expire' || decision === 'give_up') {
|
|
91
|
+
turn.status = decision === 'expire' ? 'expired' : 'refused'
|
|
92
|
+
turn.reason = decision === 'expire' ? 'queued_turn_expired' : 'delivery_attempts_exhausted'
|
|
93
|
+
turn.settledAt = deps.now()
|
|
94
|
+
retired += 1; dirty = true
|
|
95
|
+
continue
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ATTEMPT IS RECORDED BEFORE THE CALL, and persisted. A delivery that crashes the
|
|
99
|
+
// process mid-flight must not come back with its attempt count unchanged and try
|
|
100
|
+
// forever; the ceiling only bounds anything if it survives the crash it is bounding.
|
|
101
|
+
turn.attempts += 1
|
|
102
|
+
turn.status = 'delivering'
|
|
103
|
+
writeQueue(provider, threadId, queue)
|
|
104
|
+
dirty = true
|
|
105
|
+
|
|
106
|
+
let outcome: { ok: boolean; reason?: string }
|
|
107
|
+
try {
|
|
108
|
+
outcome = await deps.deliver(turn)
|
|
109
|
+
} catch (error) {
|
|
110
|
+
outcome = { ok: false, reason: error instanceof Error ? error.message : 'deliver_threw' }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (outcome.ok) {
|
|
114
|
+
turn.status = 'delivered'
|
|
115
|
+
turn.settledAt = deps.now()
|
|
116
|
+
delivered += 1
|
|
117
|
+
} else if (turn.attempts >= MAX_DELIVERY_ATTEMPTS) {
|
|
118
|
+
turn.status = 'refused'
|
|
119
|
+
turn.reason = outcome.reason ?? 'delivery_failed'
|
|
120
|
+
turn.settledAt = deps.now()
|
|
121
|
+
retired += 1
|
|
122
|
+
} else {
|
|
123
|
+
// Back to waiting for the next sweep. A failure that is not terminal is a
|
|
124
|
+
// failure to deliver NOW, not a failure of the turn.
|
|
125
|
+
turn.status = 'waiting'
|
|
126
|
+
turn.reason = outcome.reason
|
|
127
|
+
held += 1
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (dirty) writeQueue(provider, threadId, queue)
|
|
132
|
+
return { delivered, held, retired }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** One pass over every thread holding a queue. */
|
|
136
|
+
export async function drainAllThreads(deps: ThreadTurnQueueDeps): Promise<number> {
|
|
137
|
+
let delivered = 0
|
|
138
|
+
for (const { provider, threadId } of queuedThreadKeys()) {
|
|
139
|
+
try {
|
|
140
|
+
delivered += (await drainThread(provider, threadId, deps)).delivered
|
|
141
|
+
} catch (error) {
|
|
142
|
+
// One bad thread must not stop the sweep for every other thread.
|
|
143
|
+
console.error(`[thread-turn-queue] drain failed for ${provider}: ${error instanceof Error ? error.message : error}`)
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return delivered
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function createThreadTurnQueueRouter(deps: ThreadTurnQueueDeps): Router {
|
|
150
|
+
const router = Router()
|
|
151
|
+
|
|
152
|
+
// POST — park a turn for a thread that is busy right now.
|
|
153
|
+
router.post('/agent-sessions/:provider/:threadId/queued-turns', (req: Request, res: Response) => {
|
|
154
|
+
res.set('Cache-Control', 'private, no-store')
|
|
155
|
+
const provider = String(req.params.provider ?? '')
|
|
156
|
+
const threadId = String(req.params.threadId ?? '')
|
|
157
|
+
const body = (req.body ?? {}) as Record<string, unknown>
|
|
158
|
+
const clientTurnId = typeof body.clientTurnId === 'string' ? body.clientTurnId : ''
|
|
159
|
+
const cosSessionId = typeof body.cosSessionId === 'string' ? body.cosSessionId : ''
|
|
160
|
+
const prompt = typeof body.prompt === 'string' ? body.prompt : ''
|
|
161
|
+
if (!clientTurnId || !cosSessionId || !prompt.trim()) {
|
|
162
|
+
return res.status(400).json({ error: 'invalid_request' })
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// THE GATE DECIDES WHETHER PARKING IS EVEN HONEST. A structural refusal can never
|
|
166
|
+
// clear, and telling someone their turn is queued when it can never run is worse
|
|
167
|
+
// than refusing it. An attachable thread is not queued either -- it is sent now,
|
|
168
|
+
// through the ordinary route, which the client does on this 409.
|
|
169
|
+
const gate = deps.occupancy(provider, threadId)
|
|
170
|
+
if (gate.attachable) return res.status(409).json({ error: 'thread_free', hint: 'send_now' })
|
|
171
|
+
if (!queueableRefusal(gate.reason)) {
|
|
172
|
+
return res.status(423).json({ error: gate.reason ?? 'probe_failed', queueable: false })
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const now = deps.now()
|
|
176
|
+
const queue = readQueue(provider, threadId, now)
|
|
177
|
+
const admitted = admitToQueue(queue, {
|
|
178
|
+
clientTurnId, cosSessionId, provider, threadId, prompt,
|
|
179
|
+
queuedAt: now, status: 'waiting', attempts: 0,
|
|
180
|
+
})
|
|
181
|
+
if (!admitted.ok) return res.status(409).json({ error: admitted.reason })
|
|
182
|
+
|
|
183
|
+
writeQueue(provider, threadId, admitted.queue)
|
|
184
|
+
return res.status(202).json({
|
|
185
|
+
queued: true,
|
|
186
|
+
clientTurnId,
|
|
187
|
+
position: queuePosition(admitted.queue, clientTurnId),
|
|
188
|
+
waitingOn: gate.reason,
|
|
189
|
+
})
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
// GET — what is waiting, for the pending row on the lens.
|
|
193
|
+
router.get('/agent-sessions/:provider/:threadId/queued-turns', (req: Request, res: Response) => {
|
|
194
|
+
res.set('Cache-Control', 'private, no-store')
|
|
195
|
+
const provider = String(req.params.provider ?? '')
|
|
196
|
+
const threadId = String(req.params.threadId ?? '')
|
|
197
|
+
const queue = readQueue(provider, threadId, deps.now())
|
|
198
|
+
return res.json({ turns: queue.map(t => publicRow(t, queuePosition(queue, t.clientTurnId))) })
|
|
199
|
+
})
|
|
200
|
+
|
|
201
|
+
// DELETE — the cancel control, the same affordance the desktop queue offers.
|
|
202
|
+
router.delete('/agent-sessions/:provider/:threadId/queued-turns/:clientTurnId', (req: Request, res: Response) => {
|
|
203
|
+
res.set('Cache-Control', 'private, no-store')
|
|
204
|
+
const provider = String(req.params.provider ?? '')
|
|
205
|
+
const threadId = String(req.params.threadId ?? '')
|
|
206
|
+
const clientTurnId = String(req.params.clientTurnId ?? '')
|
|
207
|
+
const now = deps.now()
|
|
208
|
+
const queue = readQueue(provider, threadId, now)
|
|
209
|
+
const row = queue.find(t => t.clientTurnId === clientTurnId)
|
|
210
|
+
if (!row) return res.status(404).json({ error: 'unknown_turn' })
|
|
211
|
+
// A turn already handed to the adapter cannot be recalled, and saying otherwise
|
|
212
|
+
// would be the one lie this whole feature exists to avoid.
|
|
213
|
+
if (row.status === 'delivering') return res.status(409).json({ error: 'already_delivering' })
|
|
214
|
+
if (row.status === 'waiting') {
|
|
215
|
+
row.status = 'cancelled'
|
|
216
|
+
row.settledAt = now
|
|
217
|
+
writeQueue(provider, threadId, queue)
|
|
218
|
+
}
|
|
219
|
+
return res.json({ cancelled: true, clientTurnId, status: row.status })
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
return router
|
|
223
|
+
}
|