@gotcos/glasses-server 6.77.0 → 6.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +40 -0
- package/CHANGELOG.md +16 -0
- package/README.md +12 -1
- package/managed-runtime-contract.json +9 -1
- package/package.json +10 -1
- package/server/index.ts +92 -9
- package/server/lib/agent-session-binding-registry.ts +623 -9
- package/server/lib/boot-resume.ts +552 -0
- package/server/lib/claude-bridge.ts +152 -3
- package/server/lib/claude-cli-support.ts +39 -0
- package/server/lib/claude-steer.ts +259 -0
- package/server/lib/env-switch.ts +17 -0
- package/server/lib/managed-restart.ts +8 -0
- package/server/lib/provider-actions.ts +18 -0
- package/server/lib/query-job-coordinator.ts +300 -0
- package/server/lib/query-job-feature.ts +39 -0
- package/server/lib/query-job-resume.ts +269 -0
- package/server/lib/query-job-runtime.ts +23 -3
- package/server/lib/query-job-store.ts +516 -7
- package/server/lib/query-job-types.ts +210 -1
- package/server/lib/queue-turn-evidence.ts +213 -0
- package/server/lib/thread-turn-queue-deliver.ts +6 -1
- package/server/lib/thread-turn-queue-store.ts +13 -4
- package/server/lib/thread-turn-queue.ts +151 -1
- package/server/routes/agent-session-bindings.ts +169 -11
- package/server/routes/health.ts +26 -1
- package/server/routes/maintenance.ts +34 -0
- package/server/routes/meeting.ts +27 -9
- package/server/routes/query-jobs.ts +107 -13
- package/server/routes/recovery.ts +4 -0
- package/server/routes/thread-turn-queue.ts +371 -10
- package/server/scripts/live-cues-v2-notes-mutation-gate.mjs +0 -170
- package/server/scripts/transcript-cleanup-mutation-gate.mjs +0 -219
package/.env.example
CHANGED
|
@@ -61,6 +61,38 @@ BIND_HOST=0.0.0.0
|
|
|
61
61
|
# and drain safely.
|
|
62
62
|
# COS_DURABLE_QUERY_JOBS=0
|
|
63
63
|
|
|
64
|
+
# 6.78.0: crash-loop parking. Work the server resumes on boot (meeting finalization, a
|
|
65
|
+
# question that never started) is parked after it was resumed on 3 boots in a row that each
|
|
66
|
+
# followed a crash less than 30 minutes after the last, instead of restarting the server
|
|
67
|
+
# forever. Update Server and other clean restarts never count, and a clean exit starts the
|
|
68
|
+
# count over. Parked work stays saved and is listed under `parked` in
|
|
69
|
+
# GET /api/maintenance/status (the count is on /api/health). Retry it with
|
|
70
|
+
# POST /api/maintenance/parked/<kind>/<id>/retry with the API token (the phone and the lens
|
|
71
|
+
# can, not only this Mac), or Retry the question in the app. A parked meeting finalization
|
|
72
|
+
# also gets one automatic second chance on the first boot 30 minutes after it was parked.
|
|
73
|
+
# On by default; 0 / false / off turns it off (6.77.0 behaviour).
|
|
74
|
+
# COS_CRASH_LOOP_PARKING=0
|
|
75
|
+
|
|
76
|
+
# 6.78.0: restart truth. A question a restart cut off (a crash, or a graceful stop such as
|
|
77
|
+
# Update Server or a Mac restart) ends "Server restarted. Tap to retry." and its job record
|
|
78
|
+
# names the step it was cut off during and the steps that had already changed something
|
|
79
|
+
# (edits, commands), so you can choose Resume or Retry knowing what already happened. On by
|
|
80
|
+
# default; 0 / false / off turns it off (the 6.77.0 messages, and no `interruption` record).
|
|
81
|
+
# COS_RESTART_NOTICE=0
|
|
82
|
+
|
|
83
|
+
# 6.78.0: the compact question journal. Off by default in this release: this release only
|
|
84
|
+
# READS compact rows, so a rollback to it from the next one keeps every answer. 1 / true / on
|
|
85
|
+
# stores each streamed piece once instead of the whole answer so far (much smaller journals).
|
|
86
|
+
# Leave it off until the release after 6.78.0 turns it on.
|
|
87
|
+
# COS_QUERY_JOB_DELTA_JOURNAL=1
|
|
88
|
+
|
|
89
|
+
# 6.78.0: steer a running answer from the glasses (Claude only). Off by default in 6.78.0.
|
|
90
|
+
# 1 / true / on lets COS Glasses 6.10.645 or later add guidance to an Ask while Claude is still
|
|
91
|
+
# working on it: the guidance joins the running answer at its next step. Codex, Cursor and
|
|
92
|
+
# Ollama answers are never steered; the app sends the guidance as the next Ask instead.
|
|
93
|
+
# Needs a Claude Code CLI that lists --input-format and --replay-user-messages.
|
|
94
|
+
# COS_MESSAGES_STEER=1
|
|
95
|
+
|
|
64
96
|
# 6.52.0: the Messages trail (readable steps of a running Messages job, journaled and sent
|
|
65
97
|
# only to a client that asks with ?trail=1). On by default. 0 is the rollback that needs no
|
|
66
98
|
# downgrade: no trail is recorded and every job stream and snapshot is the 6.51.0 one.
|
|
@@ -377,3 +409,11 @@ COS_CONTINUE_FULL_PERMISSIONS=1
|
|
|
377
409
|
# Drop-in alternative: link a folder holding parakeet-ultra/ to
|
|
378
410
|
# <data>/stt/parakeet-sources and an install started with {} uses it, no download.
|
|
379
411
|
|
|
412
|
+
|
|
413
|
+
# ── Sessions: restart safety, Send again and steer (6.78.0) ─────────────────
|
|
414
|
+
# Read per request, like every 6.78.0 switch: 1 / true / on is on, 0 / false / off is off,
|
|
415
|
+
# anything else the default.
|
|
416
|
+
# COS_QUEUE_RECONCILE=1 # attempt journal, restart reconcile, receipts, reason copy (default on)
|
|
417
|
+
# COS_PIN_BOOT_REAP=1 # free a thread a crashed run held, 31 min after the restart and once its child is gone (default on; 0 = the 24 h reap)
|
|
418
|
+
# COS_QUEUE_RESEND=1 # Send again on a refused or expired queued turn (default on)
|
|
419
|
+
# COS_SESSION_STEER=0 # steer a busy Claude session through its peer inbox (default off)
|
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,19 @@
|
|
|
1
|
+
## 6.78.0 - Steer from the glasses, Retry and Send again, and crash safety for Asks and desk turns (Pi Durable parity)
|
|
2
|
+
|
|
3
|
+
Pairs with COS Glasses companion 6.10.645. Includes 6.77.0. No COS Control change is required (0.5.322 and 0.5.323 work; see the client matrix). Miles, 2026-10-18: steer from the glasses and retry, for Messages and Sessions; no regressions; better offline to online. Plan: Pi Durable parity build v3.1.
|
|
4
|
+
|
|
5
|
+
- **Messages steer (`COS_MESSAGES_STEER`, default off).** A running Claude Ask that was admitted `steerable: true` (outside the fingerprint) runs with `--input-format stream-json --replay-user-messages`: the prompt is one stream-json user line and stdin stays open. `POST /api/query-jobs/:jobId/steer {generation, steerId, text}` journals the steer (fsync) before it is written to stdin, then answers `{steerId, state, fallback, jobStatus}`; `GET /api/query-jobs/steers/:steerId?clientJobId=` answers the same from the newest generation. States: accepted, delivered (the CLI replayed the line), folded (a result covered it), included (carried into a Resume), too_late, unsupported, not_applied (computed when read). The run finishes only when the results cover every line written; a result that covers the prompt with a steer still open journals an answer checkpoint, and every terminal path but cancel completes with it. Same id and text replays; other text is 409 `steer_identity_conflict`; 8 steers a job (429 `steer_limit`). `fallback` is `queue_follow_up` for too_late, unsupported, and not_applied on a completed job only; a failed or interrupted job answers null because Retry or Resume carries the steer into generation + 1. A steer the CLI replayed (delivered) never falls back, even when a deadline, a restart or a second-turn error then completes the job from its checkpoint and the steer reads not_applied: it may already be in the answer or the session, so sending it again would apply it twice. A run that ends with a steer line written and no result covering it kills the CLI (instead of closing stdin, which let the CLI run the line unwatched) and drops its CLI session mapping, so the next Ask does not `--resume` a session that holds the line. The steer cut-off is measured on the monotonic clock the run's timers use, so a Mac that slept does not refuse every steer as too late. SSE `?steer=1` adds `event: steer` frames with no `id:` and no eventSeq. Capability `capabilities.durableQueryJobs.steer = {protocolVersion: 1, providers: ["claude"]}` only with the switch on and the CLI probe passed. Codex, Cursor and Ollama answer `unsupported`. With the switch off every argument, stdin byte and route body is 6.77.0's (goldens recorded from 6.77.0).
|
|
6
|
+
- **Session steer (`COS_SESSION_STEER`, default off).** `POST /api/agent-sessions/:provider/:threadId/queued-turns` takes `mode: "steer"` and `fallback: "queue"|"none"`. A busy Claude thread with a peer inbox, `COS_CONTINUE_LIVE` on and no pending permission prompt gets the steer through the peer inbox (200 `{state: delivered, reason: steered}`): the intent row is written durably, then the attempt, then the frame. Unverified, a write failure or a throw: 200 `refused`, `steer_unverified`, `deliveryState: unknown`, never resent. Otherwise it queues (202 with `steerFallback`). The attach refusal gains `steer: {available, mode | reason}` and `act.steer` per provider; both are absent with the switch off.
|
|
7
|
+
- **Restart truth (`COS_RESTART_NOTICE`, default on).** An Ask cut off by a crash (`server_restarted`) or a graceful shutdown (`server_shutdown`) ends with `error: {code: interrupted, cause, message: "Server restarted. Tap to retry.", retryable: true}` (31 characters: the whole message fits the lens's 32-column lead) and an `interruption` block built from the job's own trail: `cutOffDuring`, `alreadyDone` (side effects only, 5 entries of 60 characters at most), `incomplete`, `resumable`. A graceful shutdown first lets the steps a run already reported reach the journal (1 s for all runs). The raw text `server_shutdown` no longer reaches the lens.
|
|
8
|
+
- **Crash-loop parking (`COS_CRASH_LOOP_PARKING`, default on).** Boot-resumed work (a meeting finalization, a requeued Ask) that crashed the server on 3 consecutive boots within 30 minutes is parked instead of retried forever. Clean restarts never count; a missing or unreadable ledger reads as a clean boot; any I/O error fails open. A parked Ask ends `restart_limit`, `cause: crash_loop_parked`, "Stopped after 3 server crashes." `GET /api/maintenance/status` gains `parked[]` (contractVersion stays 2; a parked entry reads `attempts: 4`, the count past the limit); `POST /api/maintenance/parked/:kind/:id/retry` retries one (token auth, so the phone and the lens can, refused during a drain). `/api/health` adds `crash_loop_parking: {enabled, parked}`, a count only. A parked meeting finalization gets one automatic second chance on the first boot at least 30 minutes after it was parked (a kernel panic storm can park a meeting that was never the cause); a second park is permanent until Retry. A parked Ask never unparks by itself. The phone's Restart marks the exit clean before the kickstart, so a restart that ends in the 2 s `exit(1)` fallback is not counted as a crash.
|
|
9
|
+
- **Queued desk turns after a crash (`COS_QUEUE_RECONCILE`, default on).** A durable attempt journal (target-scoped, 48 h) is written before any frame or spawn. A row entering `delivering` is stamped with its boot. At the next boot a row another boot left `delivering` is settled on evidence, in order: a completed or queued ledger record or a sent attempt, an exact transcript match, a Codex queue row with the same text (delivered); an ambiguous record, an unstamped 6.77 row or a steer row without evidence (unknown); only a `not_sent` attempt or a stamped row with no attempt goes back to waiting. Unknown is settled `refused`, `delivery_unknown_after_restart`, kept 6 h, never resent. The turns POST consults receipts (delivered rows, unknown rows, the attempt journal) before the claim, so a phone retry after a restart replays instead of sending twice; the drainer's own `delivering` row is never a receipt. Rows gain `reasonCopy`, `deliveryState`, `mode`, `resendOf`, `resentAs`.
|
|
10
|
+
- **Orphan pins (`COS_PIN_BOOT_REAP`, default on).** A pin a dead boot left frees its thread 31 minutes after the later of the pin and the restart (the longest turn the attached adapter accepts, 30 minutes, plus one), not 24 h. A turn that spawned a resume child records the child's pid and measured start time on its pin, and that pin also waits until the child is gone (no such pid, or a reused one). Until then the attach refusal reads "An earlier run may still be running · frees h:mm", or, past the window with the child still running, "An earlier run is still running on this Mac".
|
|
11
|
+
- **Send again (`COS_QUEUE_RESEND`, default on).** `POST .../queued-turns/:clientTurnId/resend` copies a refused or expired row's words into one new row, once from any surface (a repeat answers the first copy); an unknown original needs `acknowledge: "may_duplicate"` (409 `acknowledge_required`); a pruned one is 404 `unknown_turn`. Rows the flag can act on carry `resendable: true`.
|
|
12
|
+
- **Ask journal size (MVP-D release 1).** The reader understands delta-only chunk rows and checks their length; `createQueryJobJournalStorage()` is exported. The writer stays behind `COS_QUERY_JOB_DELTA_JOURNAL` (default off) until the next release, so a rollback to 6.77.x never meets delta rows.
|
|
13
|
+
- **Flags.** On by default with kill switches: `COS_CRASH_LOOP_PARKING`, `COS_RESTART_NOTICE`, `COS_QUEUE_RECONCILE`, `COS_PIN_BOOT_REAP`, `COS_QUEUE_RESEND`. Off by default (on for Miles's candidate, flipped after a 2-week field window): `COS_MESSAGES_STEER`, `COS_SESSION_STEER`, `COS_QUERY_JOB_DELTA_JOURNAL`. All eight are in `managed-runtime-contract.json` `optionalEnvironment`, `.env.example` and the README. All eight read the same way (one parser): `1`, `true` or `on` is on, `0`, `false` or `off` is off, case and spaces ignored, anything else the default.
|
|
14
|
+
- **Client matrix.** Companion 6.10.645 shows the restart notice, Retry this ask, steer and Send again. Companion 6.10.644 shows an unknown row as "Refused" (its Edit parks the text as new and "Send as new" can duplicate), has no Send again, and still reads 409 `duplicate_turn` as parked. COS Control 0.5.322/0.5.323 shows "Refused · delivery unknown after restart", drops `reasonCopy` and has no Send again or parked-item row. A thread that had a stranded `delivering` row gets the busy-Codex hop back once the row is reconciled (intended).
|
|
15
|
+
- **Known limits.** A parked meeting finalization shows in the maintenance status, the `/api/health` count and the `boot_resume.parked` log line until Control's row ships; it has its one automatic second chance meanwhile. Pre-6.78 pins and stranded rows settle as unknown, with no ledger record, during the upgrade window. A loop already running at install parks after 4 crashes, not 3. A pin with no recorded child (a 6.77 pin, a live delivery, a spawn whose record failed) has only the 31-minute window, a bound and not a proof the orphan is gone. Session steer is Claude only (Codex has `turn/steer`, but COS does not hold its app-server channel); with hooks off a session held on a permission prompt is steered and lands when the prompt is answered. Live-verified on Claude Code 2.1.302: Messages steer in the bridge's exact argument shape (`--include-partial-messages`, `--effort`, `--system-prompt`, `--allowedTools`) on a fresh session and on a `--resume`d persisted one. Not live-verified: a steer during a permission prompt (approval) on a Messages run, and MCP tools with stream-json input. Dispatch auto-resume (the restart planner ships uncalled, marked internal), phone steer buttons and the delta writer are deferred.
|
|
16
|
+
|
|
1
17
|
## 6.77.0 - Transcript cleanup: the passages the engine was unsure of, resolved with the meeting's text and screen context (default off)
|
|
2
18
|
|
|
3
19
|
Includes 6.76.2. Off until Miles turns it on. No COS Control or companion change is required; Control can add the toggle, the screen feed and the Dictionary suggestions later against the contracts below.
|
package/README.md
CHANGED
|
@@ -464,7 +464,18 @@ No inline `#` on the EXTRA_ARGS line.),
|
|
|
464
464
|
`COS_SCRIPTS_DIR` (full pipeline), `COS_DURABLE_QUERY_JOBS=0` (optional
|
|
465
465
|
machine-wide rollback for build 204+ server-owned query recovery),
|
|
466
466
|
`COS_MESSAGES_TRAIL=0` (6.52.0: no Messages trail; every job stream and snapshot is the
|
|
467
|
-
6.51.0 one),
|
|
467
|
+
6.51.0 one), the 6.78.0 switches (each reads `1`, `true` or `on` as on and `0`, `false` or
|
|
468
|
+
`off` as off; anything else is its default): `COS_CRASH_LOOP_PARKING` (default on: boot-resumed
|
|
469
|
+
work that crashed the server on 3 boots in a row is parked; see `GET /api/maintenance/status`),
|
|
470
|
+
`COS_RESTART_NOTICE` (default on: an Ask a restart cut off says so and names the step it was
|
|
471
|
+
cut off during), `COS_QUEUE_RECONCILE` (default on: queued desk turns a crash cut off are
|
|
472
|
+
settled on evidence, never sent twice), `COS_PIN_BOOT_REAP` (default on: a thread a crashed
|
|
473
|
+
run held frees 31 minutes after the restart, once its child is gone), `COS_QUEUE_RESEND`
|
|
474
|
+
(default on: Send again on a refused or expired queued turn), `COS_MESSAGES_STEER` (default
|
|
475
|
+
off: steer a running Claude Ask from the glasses), `COS_SESSION_STEER` (default off: steer a
|
|
476
|
+
busy Claude desk session through its peer inbox) and `COS_QUERY_JOB_DELTA_JOURNAL` (default
|
|
477
|
+
off until the release after 6.78.0: the compact Ask journal writer),
|
|
478
|
+
`COS_PERMISSION_BROKER` (6.52.0: `0` answers every permission request with
|
|
468
479
|
the Mac's own dialog, `questions` holds questions but not tool approvals; unset holds
|
|
469
480
|
both), `COS_PERMISSION_BROKER_DESK_IDLE_S` (default 90, never below 30),
|
|
470
481
|
`COS_PERMISSION_BROKER_TIMEOUT_S` (default 110, clamped to 5 through 120),
|
|
@@ -14,7 +14,11 @@
|
|
|
14
14
|
"HTTPS_PORT",
|
|
15
15
|
"BIND_HOST",
|
|
16
16
|
"COS_DURABLE_QUERY_JOBS",
|
|
17
|
+
"COS_CRASH_LOOP_PARKING",
|
|
17
18
|
"COS_MESSAGES_TRAIL",
|
|
19
|
+
"COS_RESTART_NOTICE",
|
|
20
|
+
"COS_QUERY_JOB_DELTA_JOURNAL",
|
|
21
|
+
"COS_MESSAGES_STEER",
|
|
18
22
|
"COS_LIVE_CUES",
|
|
19
23
|
"COS_LIVE_CUES_MODEL",
|
|
20
24
|
"COS_LIVE_CUES_GRAPH",
|
|
@@ -60,7 +64,11 @@
|
|
|
60
64
|
"COS_LENS_GIST_DAILY_CAP",
|
|
61
65
|
"COS_DIARIZER",
|
|
62
66
|
"COS_NEMOTRON_CLI",
|
|
63
|
-
"COS_NEMOTRON_MODELS"
|
|
67
|
+
"COS_NEMOTRON_MODELS",
|
|
68
|
+
"COS_QUEUE_RECONCILE",
|
|
69
|
+
"COS_PIN_BOOT_REAP",
|
|
70
|
+
"COS_QUEUE_RESEND",
|
|
71
|
+
"COS_SESSION_STEER"
|
|
64
72
|
],
|
|
65
73
|
"maintenance": {
|
|
66
74
|
"scope": "cross_boot",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotcos/glasses-server",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.78.0",
|
|
4
4
|
"description": "COS Glasses: self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, Cursor Agent CLI, or local Ollama",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -62,7 +62,16 @@
|
|
|
62
62
|
"!server/scripts/merged-scribe-mutation-gate.mjs",
|
|
63
63
|
"!server/scripts/nemotron-replay.ts",
|
|
64
64
|
"!server/scripts/turn-request-mutation-gate.mjs",
|
|
65
|
+
"!server/scripts/crash-loop-mutation-gate.mjs",
|
|
66
|
+
"!server/scripts/restart-notice-mutation-gate.mjs",
|
|
67
|
+
"!server/scripts/gate-anchor-check.mjs",
|
|
68
|
+
"!server/scripts/messages-steer-mutation-gate.mjs",
|
|
65
69
|
"!server/scripts/mutation-gate.mjs",
|
|
70
|
+
"!server/scripts/turn-queue-durable-mutation-gate.mjs",
|
|
71
|
+
"!server/scripts/session-steer-mutation-gate.mjs",
|
|
72
|
+
"!server/scripts/check-gate-anchors.mjs",
|
|
73
|
+
"!server/scripts/transcript-cleanup-mutation-gate.mjs",
|
|
74
|
+
"!server/scripts/live-cues-v2-notes-mutation-gate.mjs",
|
|
66
75
|
"!server/lib/meeting-engine/__fixtures__/**",
|
|
67
76
|
"!server/lib/__fixtures__/**",
|
|
68
77
|
"!shared/**/*.test.ts",
|
package/server/index.ts
CHANGED
|
@@ -28,7 +28,7 @@ import { cachedHookStatus, isCosSpawnedPid, registryEntrypointSync, claudeDeskRu
|
|
|
28
28
|
import { clearingHaltOnSpawn, writeHaltMarker } from './lib/session-halt.js'
|
|
29
29
|
import { startProviderObserveRefresh } from './lib/provider-observe.js'
|
|
30
30
|
import { appendSessionCancelLedger, cancelHoldUntil as cancelHoldUntilFor, noteThreadCancelled, setDeskCancelReaders, threadCancel } from './lib/session-cancel.js'
|
|
31
|
-
import { makeQueueTurnEvidence } from './lib/queue-turn-evidence.js'
|
|
31
|
+
import { makeDeliveryEvidence, makeQueueTurnEvidence, makeTurnReceiptLookup } from './lib/queue-turn-evidence.js'
|
|
32
32
|
import {
|
|
33
33
|
PermissionBroker,
|
|
34
34
|
brokerSignalSink,
|
|
@@ -48,10 +48,10 @@ import {
|
|
|
48
48
|
|
|
49
49
|
threadAttachEnabled,
|
|
50
50
|
} from './routes/agent-session-bindings.js'
|
|
51
|
-
import { AgentSessionBindingRegistry } from './lib/agent-session-binding-registry.js'
|
|
51
|
+
import { AgentSessionBindingRegistry, makePinChildProbe } from './lib/agent-session-binding-registry.js'
|
|
52
52
|
import { targetKey } from './lib/agent-session-binding-store.js'
|
|
53
53
|
import { cosSpawnedPids } from './lib/agent-session-ownership-store.js'
|
|
54
|
-
import { buildOccupancyProbes, realOccupancyDirs, withCodexTurnClock, withCursorComposerTurn, withHookTurnClock } from './lib/occupancy-probes.js'
|
|
54
|
+
import { buildOccupancyProbes, processStartMs, realOccupancyDirs, withCodexTurnClock, withCursorComposerTurn, withHookTurnClock } from './lib/occupancy-probes.js'
|
|
55
55
|
import { readCodexTurnEndedAtMs } from './lib/codex-turn-clock.js'
|
|
56
56
|
import { codexLiveQueueEnabled } from './lib/codex-live-queue.js'
|
|
57
57
|
import { realAttachedWorkspaceDeps, resolveAttachedWorkspace } from './lib/attached-workspace.js'
|
|
@@ -64,7 +64,9 @@ import { readComposerFacts } from './lib/cursor-session-model.js'
|
|
|
64
64
|
import { MAX_PROMPT_CHARS } from './lib/attached-provider-adapter.js'
|
|
65
65
|
import { readRecentSessionTurns } from './lib/agent-session-turns.js'
|
|
66
66
|
import { parseAgentSession } from './lib/agent-session-store.js'
|
|
67
|
-
import { makeLiveTurnDeliverer } from './lib/session-peer-inbox-deps.js'
|
|
67
|
+
import { makeLiveTurnDeliverer, readPeerInboxRecords } from './lib/session-peer-inbox-deps.js'
|
|
68
|
+
import { selectPeerInbox } from './lib/session-peer-inbox.js'
|
|
69
|
+
import { nativeQueueControl, plainQueueText } from './lib/codex-queue-control.js'
|
|
68
70
|
import { makeCodexLiveDeliverer } from './lib/codex-live-queue-deps.js'
|
|
69
71
|
import { forkThread, realForkDeps } from './lib/fork-thread.js'
|
|
70
72
|
import { ForkJobLedger, fileForkJobPersistence } from './lib/fork-job-ledger.js'
|
|
@@ -158,6 +160,7 @@ import { createWorkSearchRouter } from './routes/work-search.js'
|
|
|
158
160
|
import { isReadOnlyApiPost } from './lib/work-search.js'
|
|
159
161
|
import { dataPath } from './lib/data-dir.js'
|
|
160
162
|
import { loadJsonOrQuarantine, durableAtomicWriteFileSync } from './lib/atomic-fs.js'
|
|
163
|
+
import { BOOT_RESUME_LEDGER_FILE, createBootResumeGate, installBootResumeGate } from './lib/boot-resume.js'
|
|
161
164
|
import { WorkIntakeStore, createOptionalWorkIntakeStore } from './lib/work-intake-store.js'
|
|
162
165
|
import { WorkHandoffRequestStore, createOptionalWorkHandoffRequestStore, handoffRequestCapabilities } from './lib/work-handoff-requests.js'
|
|
163
166
|
import { createWorkBoardReader } from './lib/work-board-reader.js'
|
|
@@ -192,16 +195,16 @@ import {
|
|
|
192
195
|
} from './lib/maintenance-lifecycle.js'
|
|
193
196
|
|
|
194
197
|
const app = express()
|
|
195
|
-
import { createThreadTurnQueueRouter, drainAllThreads } from './routes/thread-turn-queue.js'
|
|
198
|
+
import { createThreadTurnQueueRouter, drainAllThreads, steerAvailabilityFor } from './routes/thread-turn-queue.js'
|
|
196
199
|
import { createCursorStopFollowupRouter } from './routes/cursor-stop-followup.js'
|
|
197
|
-
import { queuedThreadKeys, readQueue, transcriptTurnVerdict, writeQueue } from './lib/thread-turn-queue-store.js'
|
|
200
|
+
import { queuedThreadKeys, readQueue, readTranscriptTailLines, transcriptTurnVerdict, writeQueue } from './lib/thread-turn-queue-store.js'
|
|
198
201
|
import { cursorDeskHaltReady, hookHaltReady, readHookToken } from './lib/claude-hooks-installer.js'
|
|
199
202
|
import { OPEN_TURN_CEILING_MS } from './lib/session-state-derive.js'
|
|
200
203
|
import { createDrainKick, kickPlanFor } from './lib/thread-drain-kick.js'
|
|
201
204
|
import { transcriptPathFor } from './lib/native-head.js'
|
|
202
205
|
import { deliverQueuedTurnOverLoopback } from './lib/thread-turn-queue-deliver.js'
|
|
203
206
|
import { readFences, writeFences } from './lib/thread-fence-store.js'
|
|
204
|
-
import type
|
|
207
|
+
import { sessionSteerEnabled, type QueuedThreadTurn } from './lib/thread-turn-queue.js'
|
|
205
208
|
|
|
206
209
|
const PORT = parseInt(process.env.PORT ?? '3141', 10)
|
|
207
210
|
|
|
@@ -489,7 +492,9 @@ app.use((_req, _res, next) => {
|
|
|
489
492
|
|
|
490
493
|
// Hydrated once at boot, before any route can read it. Never throws; a corrupt
|
|
491
494
|
// or unreadable store yields a degraded registry rather than a silent empty one.
|
|
492
|
-
|
|
495
|
+
// 6.78.0 (P7 S6): a crash's orphan pin that names its resume child waits for that pid, with
|
|
496
|
+
// its MEASURED start time, so a reused pid reads as gone.
|
|
497
|
+
const agentSessionBindingRegistry = AgentSessionBindingRegistry.open({ pinChildProbe: makePinChildProbe(processStartMs) })
|
|
493
498
|
// Boot reap, and it is load-bearing rather than housekeeping. A turn pins its
|
|
494
499
|
// binding and unpins in a `finally` that never runs if the process dies during the
|
|
495
500
|
// provider run — a window of up to the 21-minute attached timeout, and exactly what
|
|
@@ -507,10 +512,23 @@ try {
|
|
|
507
512
|
} catch (error) {
|
|
508
513
|
console.warn('[binding-registry] boot reap failed', error)
|
|
509
514
|
}
|
|
515
|
+
// 6.78.0 (COS_PIN_BOOT_REAP): a pin a dead boot left frees its thread 21 minutes after the
|
|
516
|
+
// later of its pin and this restart (the query jobs' orphan fence), not 24 h. Checked on the
|
|
517
|
+
// tick below and lazily on every attach and occupancy check (`attachBlocked`, `create`).
|
|
518
|
+
// Not at boot: nothing can be due until 21 minutes after this registry opened.
|
|
519
|
+
const releaseOrphanPins = (when: string): void => {
|
|
520
|
+
try {
|
|
521
|
+
const orphans = agentSessionBindingRegistry.reapOrphanedPins(Date.now())
|
|
522
|
+
if (orphans.released > 0) console.log(`[binding-registry] ${when}: released ${orphans.released} pin(s) a dead boot left, ${orphans.recorded} recorded as delivery unknown`)
|
|
523
|
+
} catch (error) {
|
|
524
|
+
console.warn('[binding-registry] orphan pin release failed', error)
|
|
525
|
+
}
|
|
526
|
+
}
|
|
510
527
|
// And periodically, so a strand created mid-run clears without waiting for the next
|
|
511
528
|
// restart. Unref'd so it never holds the process open during shutdown.
|
|
512
529
|
const bindingReapTimer = setInterval(() => {
|
|
513
530
|
try { agentSessionBindingRegistry.reap(Date.now()) } catch { /* next tick retries */ }
|
|
531
|
+
releaseOrphanPins('tick')
|
|
514
532
|
}, 10 * 60_000)
|
|
515
533
|
bindingReapTimer.unref()
|
|
516
534
|
|
|
@@ -865,6 +883,26 @@ const sharedTargetGuard = new TargetGuard(
|
|
|
865
883
|
: null,
|
|
866
884
|
)
|
|
867
885
|
|
|
886
|
+
// 6.78.0 (P3): one live-turn transport for the turns route's live hop and Sessions steer,
|
|
887
|
+
// so the health counters see both. And what a steer needs to know about a Claude session:
|
|
888
|
+
// a peer inbox it can be reached through, and whether it is held on a dialog (hooks).
|
|
889
|
+
const liveTurnDeliverer = makeLiveTurnDeliverer(nativeHeadDeps)
|
|
890
|
+
const steerDeps = {
|
|
891
|
+
deliverLiveTurn: liveTurnDeliverer,
|
|
892
|
+
steerProbe: (_provider: string, threadId: string) => ({
|
|
893
|
+
peerInbox: selectPeerInbox(threadId, readPeerInboxRecords(), pid => cosSpawnedPids().has(pid)) !== null,
|
|
894
|
+
waitingOnPrompt: sessionHooksEnabled() ? (signalFor(threadId)?.waiting ?? null) !== null : false,
|
|
895
|
+
}),
|
|
896
|
+
}
|
|
897
|
+
/** The bounded transcript tail the reconcile and the receipts read evidence from. */
|
|
898
|
+
const QUEUE_EVIDENCE_TAIL_BYTES = 4 * 1024 * 1024
|
|
899
|
+
const queueEvidenceLines = (provider: string, threadId: string): readonly string[] | null =>
|
|
900
|
+
provider === 'claude' || provider === 'codex'
|
|
901
|
+
? readTranscriptTailLines(transcriptPathFor(provider, threadId, nativeHeadDeps), QUEUE_EVIDENCE_TAIL_BYTES)
|
|
902
|
+
: null
|
|
903
|
+
const queueAttempt = (provider: string, threadId: string, clientTurnId: string) =>
|
|
904
|
+
agentSessionBindingRegistry.attemptFor(provider, threadId, clientTurnId)
|
|
905
|
+
|
|
868
906
|
const queueTurnEvidence = makeQueueTurnEvidence({
|
|
869
907
|
signalFor: threadId => signalFor(threadId),
|
|
870
908
|
registryIdleAfterStop: (threadId, stopAt) => registryIdleAfterStop(threadId, stopAt),
|
|
@@ -942,6 +980,28 @@ if (threadAttachEnabled()) {
|
|
|
942
980
|
}
|
|
943
981
|
},
|
|
944
982
|
deliver: (turn: QueuedThreadTurn) => deliverQueuedTurnOverLoopback(turn, PORT, API_TOKEN),
|
|
983
|
+
// 6.78.0 (P3): the restart reconcile, the attempt journal and Sessions steer.
|
|
984
|
+
bootId: serverMetrics.bootId,
|
|
985
|
+
deliveryEvidence: makeDeliveryEvidence({
|
|
986
|
+
ledgerOutcome: (provider, threadId, clientTurnId) => {
|
|
987
|
+
const result = agentSessionBindingRegistry.findTurnForTarget(provider, threadId, clientTurnId)?.result
|
|
988
|
+
return result && typeof result === 'object' && typeof (result as { outcome?: unknown }).outcome === 'string'
|
|
989
|
+
? (result as { outcome: string }).outcome
|
|
990
|
+
: null
|
|
991
|
+
},
|
|
992
|
+
attempt: queueAttempt,
|
|
993
|
+
transcriptLines: queueEvidenceLines,
|
|
994
|
+
codexQueueTexts: async threadId => (await nativeQueueControl.list(threadId))
|
|
995
|
+
.map(row => plainQueueText(row))
|
|
996
|
+
.filter((text): text is string => typeof text === 'string'),
|
|
997
|
+
}),
|
|
998
|
+
...steerDeps,
|
|
999
|
+
attempts: {
|
|
1000
|
+
begin: (provider: string, threadId: string, clientTurnId: string, mode: 'turn' | 'steer', now: number) =>
|
|
1001
|
+
agentSessionBindingRegistry.beginAttempt(provider, threadId, clientTurnId, mode, now),
|
|
1002
|
+
settle: (provider: string, threadId: string, clientTurnId: string, exit: 'sent' | 'not_sent' | 'unknown', now: number) =>
|
|
1003
|
+
agentSessionBindingRegistry.settleAttempt(provider, threadId, clientTurnId, exit, now),
|
|
1004
|
+
},
|
|
945
1005
|
// 6.53.0: parked turns hold for two minutes after a cancel on their thread.
|
|
946
1006
|
cancelHoldUntil: (provider: string, threadId: string) => cancelHoldUntilFor(provider, threadId),
|
|
947
1007
|
// 6.62.0 (W14): a busy Agent CLI chat's parked turn drains here (no Stop hook fires for `-p`).
|
|
@@ -1021,7 +1081,7 @@ app.use('/api', createAgentSessionBindingsRouter({
|
|
|
1021
1081
|
// 6.49.0: the running session first (Desktop tab or iTerm, same registry record),
|
|
1022
1082
|
// the resume child second. Reads `COS_CONTINUE_LIVE` per call, so the flag reports
|
|
1023
1083
|
// on /api/health without a restart and every refusal falls back to the 6.48.2 path.
|
|
1024
|
-
deliverLiveTurn:
|
|
1084
|
+
deliverLiveTurn: liveTurnDeliverer,
|
|
1025
1085
|
// 6.51.0: a Codex thread the Codex app holds goes into that app's own queue
|
|
1026
1086
|
// (`codex queue`), never into a resume child that cannot open a held thread.
|
|
1027
1087
|
// `COS_CODEX_LIVE_QUEUE=0` restores the 6.50 path.
|
|
@@ -1067,6 +1127,14 @@ app.use('/api', createAgentSessionBindingsRouter({
|
|
|
1067
1127
|
// empty list — "nothing is bound" and "the store could not be read" must not
|
|
1068
1128
|
// look the same.
|
|
1069
1129
|
bindings: agentSessionBindingRegistry,
|
|
1130
|
+
// 6.78.0 (P3): receipts before the claim, and the `steer` field on an attach refusal.
|
|
1131
|
+
turnReceipt: makeTurnReceiptLookup({
|
|
1132
|
+
queueRows: (provider, threadId) => readQueue(provider, threadId, Date.now()),
|
|
1133
|
+
attempt: queueAttempt,
|
|
1134
|
+
transcriptLines: queueEvidenceLines,
|
|
1135
|
+
bootId: serverMetrics.bootId,
|
|
1136
|
+
}),
|
|
1137
|
+
sessionSteer: (provider, threadId, gateReason) => (sessionSteerEnabled() ? steerAvailabilityFor(steerDeps, provider, threadId, gateReason) : null),
|
|
1070
1138
|
// 6.53.0: cancel a session run from the lens. The route's own in-flight map answers for
|
|
1071
1139
|
// a COS turn; these answer for a run at the desk. Read per request, so Install hooks in
|
|
1072
1140
|
// Control flips `hooks_outdated` to `desk_run` within the status cache's five seconds.
|
|
@@ -1151,6 +1219,15 @@ app.get('/', (_req, res) => {
|
|
|
1151
1219
|
)
|
|
1152
1220
|
})
|
|
1153
1221
|
|
|
1222
|
+
// 6.78.0: crash-loop parking (lib/boot-resume.ts). One gate per process. It opens its boot
|
|
1223
|
+
// in startAdmittedRuntime (after the listeners bind, so exits 74 and 75 never count), before
|
|
1224
|
+
// any boot resume, and gracefulShutdown marks a clean exit first thing, so only boots that
|
|
1225
|
+
// follow a crash count. A Retry of a parked meeting resumes that one job.
|
|
1226
|
+
// COS_CRASH_LOOP_PARKING=0 turns it off (default on).
|
|
1227
|
+
const bootResume = installBootResumeGate(createBootResumeGate({ path: dataPath(BOOT_RESUME_LEDGER_FILE) }), {
|
|
1228
|
+
resume: { meeting_finalization: sessionId => resumeMeetingFinalizationJobs({ sessionId }) },
|
|
1229
|
+
})
|
|
1230
|
+
|
|
1154
1231
|
// Graceful shutdown persists an interrupted terminal before provider abort.
|
|
1155
1232
|
// The bounded force-exit keeps service managers from hanging forever on a
|
|
1156
1233
|
// broken disk while retaining the previous session/catalog/Whisper cleanup.
|
|
@@ -1158,6 +1235,10 @@ let gracefulShutdownStarted = false
|
|
|
1158
1235
|
async function gracefulShutdown(): Promise<void> {
|
|
1159
1236
|
if (gracefulShutdownStarted) return
|
|
1160
1237
|
gracefulShutdownStarted = true
|
|
1238
|
+
// 6.78.0: first, before the 8 s force exit can fire. A SIGTERM/SIGINT shutdown (Update
|
|
1239
|
+
// Server, kickstart -k, bootout) is not a crash loop even if a later step hangs into exit(1),
|
|
1240
|
+
// so the next boot does not count toward crash-loop parking.
|
|
1241
|
+
bootResume.markCleanShutdown()
|
|
1161
1242
|
const forceExit = setTimeout(() => process.exit(1), 8_000)
|
|
1162
1243
|
forceExit.unref?.()
|
|
1163
1244
|
await meetingAskRuntime.shutdown()
|
|
@@ -1373,6 +1454,8 @@ listenRequiredServers(listeners).then(() => {
|
|
|
1373
1454
|
if (admittedRuntimeStarted) return
|
|
1374
1455
|
admittedRuntimeStarted = true
|
|
1375
1456
|
|
|
1457
|
+
// 6.78.0: before any boot resume. Was the last exit clean? Only boots after a crash count.
|
|
1458
|
+
bootResume.openBoot()
|
|
1376
1459
|
// Resume already-saved meetings whose post-response HQ/operations work
|
|
1377
1460
|
// was interrupted by a prior server update or process exit.
|
|
1378
1461
|
resumeMeetingFinalizationJobs()
|