@agentdeck/bridge 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/fm-helper/agentdeck-fm-helper +0 -0
- package/dist/adapters/claude-code.d.ts.map +1 -1
- package/dist/adapters/claude-code.js +3 -0
- package/dist/adapters/claude-code.js.map +1 -1
- package/dist/adb-reverse.d.ts.map +1 -1
- package/dist/adb-reverse.js +4 -2
- package/dist/adb-reverse.js.map +1 -1
- package/dist/apme/collector.js +2 -2
- package/dist/apme/collector.js.map +1 -1
- package/dist/apme/outcome.js +2 -2
- package/dist/apme/outcome.js.map +1 -1
- package/dist/apme/runner.js +3 -3
- package/dist/apme/runner.js.map +1 -1
- package/dist/ble-sync-spawn.js +1 -1
- package/dist/ble-sync-spawn.js.map +1 -1
- package/dist/calendar.d.ts +82 -0
- package/dist/calendar.d.ts.map +1 -0
- package/dist/calendar.js +324 -0
- package/dist/calendar.js.map +1 -0
- package/dist/card-feed.d.ts +63 -5
- package/dist/card-feed.d.ts.map +1 -1
- package/dist/card-feed.js +218 -14
- package/dist/card-feed.js.map +1 -1
- package/dist/card-modules.d.ts +4 -0
- package/dist/card-modules.d.ts.map +1 -1
- package/dist/card-modules.js +5 -3
- package/dist/card-modules.js.map +1 -1
- package/dist/check-deps.js +4 -4
- package/dist/check-deps.js.map +1 -1
- package/dist/cli.d.ts +35 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +269 -9
- package/dist/cli.js.map +1 -1
- package/dist/codex-otel.d.ts +89 -0
- package/dist/codex-otel.d.ts.map +1 -0
- package/dist/codex-otel.js +487 -0
- package/dist/codex-otel.js.map +1 -0
- package/dist/codex-rate-limits.d.ts.map +1 -1
- package/dist/codex-rate-limits.js +15 -3
- package/dist/codex-rate-limits.js.map +1 -1
- package/dist/daemon-server.d.ts +4 -0
- package/dist/daemon-server.d.ts.map +1 -1
- package/dist/daemon-server.js +880 -63
- package/dist/daemon-server.js.map +1 -1
- package/dist/daemon-ws-client.d.ts +108 -24
- package/dist/daemon-ws-client.d.ts.map +1 -1
- package/dist/daemon-ws-client.js +175 -39
- package/dist/daemon-ws-client.js.map +1 -1
- package/dist/daemon.js +1 -1
- package/dist/device-voice-reply.d.ts +1 -10
- package/dist/device-voice-reply.d.ts.map +1 -1
- package/dist/device-voice-reply.js +19 -36
- package/dist/device-voice-reply.js.map +1 -1
- package/dist/device-voice.d.ts +37 -0
- package/dist/device-voice.d.ts.map +1 -1
- package/dist/device-voice.js +130 -1
- package/dist/device-voice.js.map +1 -1
- package/dist/diag-analyzer.d.ts.map +1 -1
- package/dist/diag-analyzer.js +1 -0
- package/dist/diag-analyzer.js.map +1 -1
- package/dist/display-monitor.d.ts.map +1 -1
- package/dist/display-monitor.js +1 -0
- package/dist/display-monitor.js.map +1 -1
- package/dist/esp32-serial.d.ts +4 -0
- package/dist/esp32-serial.d.ts.map +1 -1
- package/dist/esp32-serial.js +6 -3
- package/dist/esp32-serial.js.map +1 -1
- package/dist/foundation-models-helper.d.ts +23 -0
- package/dist/foundation-models-helper.d.ts.map +1 -1
- package/dist/foundation-models-helper.js +90 -21
- package/dist/foundation-models-helper.js.map +1 -1
- package/dist/gateway-probe.js +1 -1
- package/dist/gateway-probe.js.map +1 -1
- package/dist/glance-frame.d.ts +64 -0
- package/dist/glance-frame.d.ts.map +1 -0
- package/dist/glance-frame.js +364 -0
- package/dist/glance-frame.js.map +1 -0
- package/dist/hook-codex-sessions.d.ts +40 -0
- package/dist/hook-codex-sessions.d.ts.map +1 -0
- package/dist/hook-codex-sessions.js +174 -0
- package/dist/hook-codex-sessions.js.map +1 -0
- package/dist/idotmatrix/idotmatrix-discover.d.ts.map +1 -1
- package/dist/idotmatrix/idotmatrix-discover.js +7 -4
- package/dist/idotmatrix/idotmatrix-discover.js.map +1 -1
- package/dist/idotmatrix/idotmatrix-settings.d.ts +15 -0
- package/dist/idotmatrix/idotmatrix-settings.d.ts.map +1 -1
- package/dist/idotmatrix/idotmatrix-settings.js +21 -0
- package/dist/idotmatrix/idotmatrix-settings.js.map +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +88 -7
- package/dist/index.js.map +1 -1
- package/dist/linux-service.d.ts +52 -0
- package/dist/linux-service.d.ts.map +1 -0
- package/dist/linux-service.js +206 -0
- package/dist/linux-service.js.map +1 -0
- package/dist/logger.d.ts +6 -0
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +15 -0
- package/dist/logger.js.map +1 -1
- package/dist/mdns-discover.d.ts +19 -0
- package/dist/mdns-discover.d.ts.map +1 -0
- package/dist/mdns-discover.js +109 -0
- package/dist/mdns-discover.js.map +1 -0
- package/dist/model-catalog.d.ts.map +1 -1
- package/dist/model-catalog.js +1 -0
- package/dist/model-catalog.js.map +1 -1
- package/dist/modules/adb-module.js +1 -1
- package/dist/modules/adb-module.js.map +1 -1
- package/dist/openclaw-session.d.ts +6 -0
- package/dist/openclaw-session.d.ts.map +1 -1
- package/dist/openclaw-session.js +7 -1
- package/dist/openclaw-session.js.map +1 -1
- package/dist/output-parser.d.ts +9 -0
- package/dist/output-parser.d.ts.map +1 -1
- package/dist/output-parser.js +67 -16
- package/dist/output-parser.js.map +1 -1
- package/dist/passive-observer.d.ts +57 -8
- package/dist/passive-observer.d.ts.map +1 -1
- package/dist/passive-observer.js +210 -74
- package/dist/passive-observer.js.map +1 -1
- package/dist/pixoo/pixoo-bridge.d.ts.map +1 -1
- package/dist/pixoo/pixoo-bridge.js +47 -6
- package/dist/pixoo/pixoo-bridge.js.map +1 -1
- package/dist/pixoo/pixoo-renderer.d.ts +2 -1
- package/dist/pixoo/pixoo-renderer.d.ts.map +1 -1
- package/dist/pixoo/pixoo-renderer.js +137 -8
- package/dist/pixoo/pixoo-renderer.js.map +1 -1
- package/dist/pocket-autonomy.d.ts +78 -0
- package/dist/pocket-autonomy.d.ts.map +1 -0
- package/dist/pocket-autonomy.js +529 -0
- package/dist/pocket-autonomy.js.map +1 -0
- package/dist/pty-manager.d.ts.map +1 -1
- package/dist/pty-manager.js +3 -2
- package/dist/pty-manager.js.map +1 -1
- package/dist/python-ble-runtime.d.ts +2 -0
- package/dist/python-ble-runtime.d.ts.map +1 -1
- package/dist/python-ble-runtime.js +2 -1
- package/dist/python-ble-runtime.js.map +1 -1
- package/dist/remote-sessions.d.ts +64 -0
- package/dist/remote-sessions.d.ts.map +1 -0
- package/dist/remote-sessions.js +81 -0
- package/dist/remote-sessions.js.map +1 -0
- package/dist/review-runner.js +2 -2
- package/dist/review-runner.js.map +1 -1
- package/dist/session-aggregator.d.ts +2 -0
- package/dist/session-aggregator.d.ts.map +1 -1
- package/dist/session-aggregator.js +6 -1
- package/dist/session-aggregator.js.map +1 -1
- package/dist/session-focus-relay.d.ts +47 -1
- package/dist/session-focus-relay.d.ts.map +1 -1
- package/dist/session-focus-relay.js +130 -15
- package/dist/session-focus-relay.js.map +1 -1
- package/dist/session-push-channel.d.ts +31 -0
- package/dist/session-push-channel.d.ts.map +1 -0
- package/dist/session-push-channel.js +94 -0
- package/dist/session-push-channel.js.map +1 -0
- package/dist/session-registry.d.ts +126 -0
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +180 -3
- package/dist/session-registry.js.map +1 -1
- package/dist/subagent-timeline.d.ts +34 -0
- package/dist/subagent-timeline.d.ts.map +1 -0
- package/dist/subagent-timeline.js +156 -0
- package/dist/subagent-timeline.js.map +1 -0
- package/dist/timebox/timebox-discover.js +1 -1
- package/dist/timebox/timebox-discover.js.map +1 -1
- package/dist/tui/gauge.d.ts +4 -2
- package/dist/tui/gauge.d.ts.map +1 -1
- package/dist/tui/gauge.js +8 -4
- package/dist/tui/gauge.js.map +1 -1
- package/dist/tui/renderer.d.ts.map +1 -1
- package/dist/tui/renderer.js +28 -0
- package/dist/tui/renderer.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/usage-api.d.ts +43 -0
- package/dist/usage-api.d.ts.map +1 -1
- package/dist/usage-api.js +114 -9
- package/dist/usage-api.js.map +1 -1
- package/dist/usage-event.d.ts +19 -0
- package/dist/usage-event.d.ts.map +1 -1
- package/dist/usage-event.js +35 -0
- package/dist/usage-event.js.map +1 -1
- package/dist/utils/project-name.d.ts.map +1 -1
- package/dist/utils/project-name.js +1 -0
- package/dist/utils/project-name.js.map +1 -1
- package/dist/version-check.d.ts.map +1 -1
- package/dist/version-check.js +3 -2
- package/dist/version-check.js.map +1 -1
- package/dist/voice-assistant.d.ts.map +1 -1
- package/dist/voice-assistant.js +2 -1
- package/dist/voice-assistant.js.map +1 -1
- package/dist/voice.d.ts.map +1 -1
- package/dist/voice.js +1 -0
- package/dist/voice.js.map +1 -1
- package/dist/weather.d.ts +72 -0
- package/dist/weather.d.ts.map +1 -0
- package/dist/weather.js +206 -0
- package/dist/weather.js.map +1 -0
- package/dist/windows-service.js +5 -5
- package/dist/windows-service.js.map +1 -1
- package/fm-helper/AgentDeckFMHelper.swift +200 -0
- package/fm-helper/Info.plist +27 -0
- package/package.json +15 -13
- package/scripts/build-fm-helper.mjs +35 -3
- package/src/idotmatrix/identity_generated.py +50 -0
- package/src/idotmatrix/scan.py +33 -13
package/dist/daemon-server.js
CHANGED
|
@@ -22,7 +22,9 @@ import { BridgeLogStream } from './log-stream.js';
|
|
|
22
22
|
import { PassiveSessionObserver } from './passive-observer.js';
|
|
23
23
|
import { SessionTimelineRelay } from './session-timeline-relay.js';
|
|
24
24
|
import { SessionFocusRelay } from './session-focus-relay.js';
|
|
25
|
-
import {
|
|
25
|
+
import { SubagentTimelineTracker } from './subagent-timeline.js';
|
|
26
|
+
import { getRemoteSession, getRemoteSender, listRemoteEnriched, } from './remote-sessions.js';
|
|
27
|
+
import { createPushChannelHandler } from './session-push-channel.js';
|
|
26
28
|
import { setAwaitingOverlay, setPermissionNotificationOverlay, setAskUserQuestionOverlay, clearAwaitingOverlay, clearAskUserQuestionOverlay, getAwaitingOverlay, isPermissionNotification, applyAwaitingOverlayToObserved, } from './awaiting-overlay.js';
|
|
27
29
|
import { registerPending, resolvePending, sweepStalePending, drainAllPending, isPendingRequest } from './permission-resolver.js';
|
|
28
30
|
import { shouldHoldPreToolUse, gateReleased, buildGateQuestion, consumeStop, requestStop, clearStop, STOP_DENY_REASON, queueDirective, takeDirective, clearOnUserPrompt, clearSession as clearSteeringSession, } from './observed-steering.js';
|
|
@@ -33,7 +35,7 @@ import { setSerialCommandSink, setSerialVoiceSink, setSerialQuiesceCheck, sendSe
|
|
|
33
35
|
import { parsePeripheralMappings, resolvePeripheralAction, commandForAction } from './peripheral-mapping.js';
|
|
34
36
|
import { DeviceVoiceCollector } from './device-voice.js';
|
|
35
37
|
import { DevicePhotoCollector } from './device-photo.js';
|
|
36
|
-
import { transcribeWithHelper, synthesizeWavWithHelper } from './foundation-models-helper.js';
|
|
38
|
+
import { transcribeWithHelper, synthesizeWavWithHelper, recordWithHelper, stopHelperRecording, speakWithHelper, } from './foundation-models-helper.js';
|
|
37
39
|
import { enqueueOpenCodeCommand, pollOpenCodeCommands } from './opencode-steering.js';
|
|
38
40
|
import { runSessionReview, reviewSnapshot } from './review-runner.js';
|
|
39
41
|
/** Observed-session device approval gate (PreToolUse hold). Default ON — the
|
|
@@ -54,13 +56,15 @@ import { loadIDotMatrixDevices } from './idotmatrix/idotmatrix-settings.js';
|
|
|
54
56
|
import { handlePixooWake } from './pixoo/pixoo-client.js';
|
|
55
57
|
import { triggerMdnsRecovery } from './mdns.js';
|
|
56
58
|
import { rgbToBmp, pixooLiveHtml } from './hook-server.js';
|
|
57
|
-
import { enableDebugLog, debug } from './logger.js';
|
|
59
|
+
import { enableDebugLog, debug, debugThrottled } from './logger.js';
|
|
60
|
+
import { CodexOtelTracker, CODEX_OTEL_TRACES_PATH, spanNameSummary } from './codex-otel.js';
|
|
61
|
+
import { HookCodexSessions } from './hook-codex-sessions.js';
|
|
58
62
|
import { initApme, isTimelineProjectionEnabled, loadApmeConfig } from './apme/index.js';
|
|
59
63
|
import { FallbackTaskTimeline } from './fallback-task-timeline.js';
|
|
60
64
|
import { handleApmeRequest } from './apme/http.js';
|
|
61
65
|
import { readModelFromTranscript } from './apme/claude-transcript-reader.js';
|
|
62
66
|
import { transcriptTimelineForSession, lastAssistantTextFromTranscript, lastAssistantTextForSession, } from './session-transcript-timeline.js';
|
|
63
|
-
import { DeviceVoiceReplyRouter, speakableReply, } from './device-voice-reply.js';
|
|
67
|
+
import { DeviceVoiceReplyRouter, speakableReply, spokenDigest, pcmFromWav, } from './device-voice-reply.js';
|
|
64
68
|
import { rawSessionId } from '@agentdeck/shared';
|
|
65
69
|
import { lastAgentMessageFromCodexRollout } from './codex-rollout-response.js';
|
|
66
70
|
import { callFoundationModelsHelper } from './foundation-models-helper.js';
|
|
@@ -71,10 +75,16 @@ import { getConnectedAdbDevices, hasAdb, getAdbDeviceCount } from './adb-reverse
|
|
|
71
75
|
import { getPixooDeviceDetails, pixooDeviceCount } from './pixoo/pixoo-bridge.js';
|
|
72
76
|
import { loadTimeboxDevices } from './timebox/timebox-settings.js';
|
|
73
77
|
import { getLanIp, stripUnsafeText, cleanRawText, prepareMarkdownDetail, normalizeCommandPrompt, formatDurationSec } from '@agentdeck/shared';
|
|
74
|
-
import { injectOpenClawSession } from './openclaw-session.js';
|
|
75
|
-
import { buildCardFeed, applyOutboxDecisions, FeedPullTracker, formatFeedPull, normalizeClientIp, } from './card-feed.js';
|
|
76
|
-
import {
|
|
77
|
-
import {
|
|
78
|
+
import { injectOpenClawSession, OPENCLAW_SESSION_ID } from './openclaw-session.js';
|
|
79
|
+
import { buildCardFeed, buildGlance, applyOutboxDecisions, FeedPullTracker, formatFeedPull, normalizeClientIp, parsePullTelemetry, } from './card-feed.js';
|
|
80
|
+
import { threadModule } from './card-modules.js';
|
|
81
|
+
import { AutonomousPocketEngine, createAutonomousPocketModules, parsePocketAutonomyConfig, } from './pocket-autonomy.js';
|
|
82
|
+
import { WeatherProvider, parseWeatherSettings } from './weather.js';
|
|
83
|
+
import { CalendarProvider, parseCalendarSettings } from './calendar.js';
|
|
84
|
+
import { renderGlanceFrame, GLANCE_FRAME_BOARDS } from './glance-frame.js';
|
|
85
|
+
import { mergeRelayedSessionUsage } from './usage-event.js';
|
|
86
|
+
import { CARD_FEED_PATH, CARD_OUTBOX_PATH, GLANCE_FRAME_PATH } from '@agentdeck/shared';
|
|
87
|
+
import { readFileSync, statSync, writeFileSync, appendFileSync } from 'fs';
|
|
78
88
|
import { readFile, rm } from 'fs/promises';
|
|
79
89
|
import { tmpdir } from 'os';
|
|
80
90
|
import { join } from 'path';
|
|
@@ -121,6 +131,76 @@ function listWifiEsp32Devices() {
|
|
|
121
131
|
* other observable — a battery client with an empty outbox never identifies
|
|
122
132
|
* itself and never opens a socket. */
|
|
123
133
|
const feedPulls = new FeedPullTracker();
|
|
134
|
+
/** Daemon-owned personal feed. It persists aggregate bandit state only — no
|
|
135
|
+
* card copy, session names, calendar titles or weather locations. */
|
|
136
|
+
const pocketAutonomy = new AutonomousPocketEngine({ onError: (message) => log(`[agentdeck] ${message}`) });
|
|
137
|
+
const pocketCardModules = [threadModule, ...createAutonomousPocketModules(pocketAutonomy)];
|
|
138
|
+
// THREAD is a live AgentDeck session digest. It remains available to legacy
|
|
139
|
+
// feed consumers, while the dedicated Pocket surface receives only portable,
|
|
140
|
+
// daemon-authored reading cards.
|
|
141
|
+
const pocketReaderModules = pocketCardModules.filter((module) => module.id !== 'thread');
|
|
142
|
+
/** Glance weather for the card-feed sleep dashboard. Config is read from
|
|
143
|
+
* settings.json per pull (cheap; honors edits without a restart); the
|
|
144
|
+
* provider caches the upstream fetch itself. */
|
|
145
|
+
const glanceWeather = new WeatherProvider();
|
|
146
|
+
/** Glance schedule (today's ICS events) — same config/caching contract as
|
|
147
|
+
* weather: settings.json `calendar: {ics}`, 30 min cache, never throws. */
|
|
148
|
+
const glanceCalendar = new CalendarProvider();
|
|
149
|
+
const stagedFwByBoard = new Map();
|
|
150
|
+
const stagedFwStatePath = () => join(homedir(), '.agentdeck', 'staged-fw.json');
|
|
151
|
+
function loadStagedFw() {
|
|
152
|
+
try {
|
|
153
|
+
const raw = JSON.parse(readFileSync(stagedFwStatePath(), 'utf8'));
|
|
154
|
+
for (const [board, s] of Object.entries(raw)) {
|
|
155
|
+
if (s && typeof s.firmwarePath === 'string' && typeof s.md5 === 'string' && typeof s.size === 'number') {
|
|
156
|
+
stagedFwByBoard.set(board, s);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
catch { /* no staging file — nothing staged */ }
|
|
161
|
+
}
|
|
162
|
+
loadStagedFw();
|
|
163
|
+
function saveStagedFw() {
|
|
164
|
+
try {
|
|
165
|
+
writeFileSync(stagedFwStatePath(), JSON.stringify(Object.fromEntries(stagedFwByBoard), null, 2));
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
log(`[agentdeck] staged-fw persist failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/** Stage a build for a board. Reads the file once to fingerprint it; the file
|
|
172
|
+
* itself is re-read at device download time (a rebuild at the same path is
|
|
173
|
+
* re-fingerprinted by re-staging). */
|
|
174
|
+
function stageEsp32Fw(board, firmwarePath) {
|
|
175
|
+
const firmware = readFileSync(firmwarePath);
|
|
176
|
+
const md5 = createHash('md5').update(firmware).digest('hex');
|
|
177
|
+
stagedFwByBoard.set(board, { firmwarePath, md5, size: firmware.length, stagedAt: Date.now() });
|
|
178
|
+
saveStagedFw();
|
|
179
|
+
log(`[agentdeck] staged firmware for ${board}: ${firmware.length} bytes md5=${md5} — installs on its next feed pull`);
|
|
180
|
+
return { board, bytes: firmware.length, md5 };
|
|
181
|
+
}
|
|
182
|
+
/** The staged advert for a feed response, re-validated against the file on
|
|
183
|
+
* disk so a rebuilt-in-place binary never adverts a stale md5. */
|
|
184
|
+
function stagedFwAdvert(board) {
|
|
185
|
+
if (!board)
|
|
186
|
+
return undefined;
|
|
187
|
+
const staged = stagedFwByBoard.get(board);
|
|
188
|
+
if (!staged)
|
|
189
|
+
return undefined;
|
|
190
|
+
try {
|
|
191
|
+
const firmware = readFileSync(staged.firmwarePath);
|
|
192
|
+
const md5 = createHash('md5').update(firmware).digest('hex');
|
|
193
|
+
if (md5 !== staged.md5 || firmware.length !== staged.size) {
|
|
194
|
+
stagedFwByBoard.set(board, { ...staged, md5, size: firmware.length });
|
|
195
|
+
saveStagedFw();
|
|
196
|
+
return { size: firmware.length, md5 };
|
|
197
|
+
}
|
|
198
|
+
return { size: staged.size, md5: staged.md5 };
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
return undefined; // file gone — advert nothing rather than a dead download
|
|
202
|
+
}
|
|
203
|
+
}
|
|
124
204
|
/** Name a pulling client from the WiFi WS roster. A board that has been
|
|
125
205
|
* deep-sleeping may have aged out of that roster, which is why the tracker
|
|
126
206
|
* keeps its own IP→board memory once it has learned one. */
|
|
@@ -219,6 +299,12 @@ function registerWifiEsp32(d, ws) {
|
|
|
219
299
|
if (previous?.uptimeSec != null && uptimeSec != null && uptimeSec + 10 < previous.uptimeSec) {
|
|
220
300
|
log(`[agentdeck] ESP32 reboot observed: ${key} uptime ${previous.uptimeSec}s -> ${uptimeSec}s reset=${resetReason ?? 'unknown'} code=${resetReasonCode ?? 'unknown'}`);
|
|
221
301
|
}
|
|
302
|
+
// Plain-log a board APPEARING on WS (absent or stale before) — a sleeping
|
|
303
|
+
// battery board's brief interactive window is exactly when a WS OTA can
|
|
304
|
+
// land, and this line is the only wake signal an operator/script can tail.
|
|
305
|
+
if (!previous || Date.now() - previous.lastSeenMs > 90_000) {
|
|
306
|
+
log(`[agentdeck] ESP32 WiFi registered: ${key}`);
|
|
307
|
+
}
|
|
222
308
|
debug('daemon', `WiFi ESP32 registered: ${key} v${String(d.version ?? '')} build=${String(d.buildHash ?? '')} uptime=${String(uptimeSec ?? '')} reset=${String(resetReason ?? '')}`);
|
|
223
309
|
}
|
|
224
310
|
function touchWifiEsp32Socket(ws) {
|
|
@@ -563,6 +649,7 @@ async function fetchUsageViaWs(siblings) {
|
|
|
563
649
|
extraUsageMonthlyLimit: evt.extraUsageMonthlyLimit ?? null,
|
|
564
650
|
extraUsageUsedCredits: evt.extraUsageUsedCredits ?? null,
|
|
565
651
|
extraUsageUtilization: evt.extraUsageUtilization ?? null,
|
|
652
|
+
scopedLimits: Array.isArray(evt.scopedLimits) ? evt.scopedLimits : [],
|
|
566
653
|
inferredBillingType: null,
|
|
567
654
|
});
|
|
568
655
|
}
|
|
@@ -579,7 +666,8 @@ async function fetchUsageViaWs(siblings) {
|
|
|
579
666
|
}
|
|
580
667
|
return null;
|
|
581
668
|
}
|
|
582
|
-
|
|
669
|
+
/** @internal Exported for the relay-fallback regression test. */
|
|
670
|
+
export async function fetchUsageRelayed(selfPort) {
|
|
583
671
|
const sessions = listActiveSessions();
|
|
584
672
|
const siblings = sessions.filter(s => s.port !== selfPort && s.agentType !== 'daemon');
|
|
585
673
|
if (siblings.length > 0) {
|
|
@@ -589,8 +677,15 @@ async function fetchUsageRelayed(selfPort) {
|
|
|
589
677
|
const wsResult = await fetchUsageViaWs(siblings);
|
|
590
678
|
if (wsResult)
|
|
591
679
|
return wsResult;
|
|
592
|
-
|
|
593
|
-
|
|
680
|
+
// Relay is only an optimization to avoid multiple API pollers. A PTY session
|
|
681
|
+
// bridge only refreshes usage when it has *direct* inbound WS clients, but
|
|
682
|
+
// dashboards/Stream Deck connect to the daemon (sole entry point) — so a
|
|
683
|
+
// sibling with no direct clients never populates its /usage and the relay
|
|
684
|
+
// yields null forever, freezing the daemon's usage at its startup value.
|
|
685
|
+
// Fall back to the direct API; the shared usage-cache.json (120s TTL) dedupes
|
|
686
|
+
// so this can't cause a 429 storm even if a sibling is also actively fetching.
|
|
687
|
+
debug('daemon', 'Siblings exist but relay yielded nothing fresh — falling back to direct API (shared file cache dedupes)');
|
|
688
|
+
return fetchUsageFromApi();
|
|
594
689
|
}
|
|
595
690
|
debug('daemon', 'No siblings, using direct API');
|
|
596
691
|
return fetchUsageFromApi();
|
|
@@ -610,6 +705,11 @@ export function enrichGatewayTimelineEntry(entry) {
|
|
|
610
705
|
...entry,
|
|
611
706
|
agentType: entry.agentType ?? 'openclaw',
|
|
612
707
|
projectName: entry.projectName ?? 'OpenClaw',
|
|
708
|
+
// Devices and the voice-reply router address the gateway session as
|
|
709
|
+
// 'openclaw-gateway'; without this stamp its chat_response rows carry no
|
|
710
|
+
// sessionId, so a board armed for a spoken reply never matches the turn
|
|
711
|
+
// completion and the reply silently never plays (observed 2026-07-31).
|
|
712
|
+
sessionId: entry.sessionId ?? 'openclaw-gateway',
|
|
613
713
|
};
|
|
614
714
|
}
|
|
615
715
|
/**
|
|
@@ -638,6 +738,9 @@ export function enrichGatewayTimelineEntry(entry) {
|
|
|
638
738
|
* daemon change.
|
|
639
739
|
*/
|
|
640
740
|
export function classifyObservedHookEvent(eventName, mapped) {
|
|
741
|
+
if (eventName === 'codex_subagent_start' || eventName === 'codex_subagent_stop') {
|
|
742
|
+
return { boundary: eventName, agentType: 'codex-cli' };
|
|
743
|
+
}
|
|
641
744
|
const prefixed = /^(codex|opencode|antigravity)_(session_start|session_end|user_prompt_submit|tool_start|tool_end|stop|turn_complete|notification|permission_asked|permission_replied)$/
|
|
642
745
|
.exec(eventName);
|
|
643
746
|
if (!prefixed)
|
|
@@ -858,6 +961,13 @@ export async function startDaemon(opts) {
|
|
|
858
961
|
// that is provably alive (agentic turns regularly outlive any fixed age
|
|
859
962
|
// threshold). Declared before the HTTP handlers that populate it.
|
|
860
963
|
const hookSessionsSeen = new Set();
|
|
964
|
+
// Codex OTel span state. Declared before the HTTP server because the
|
|
965
|
+
// `/otel/v1/traces` handler closes over it — Codex's exporter can POST before
|
|
966
|
+
// the rest of daemon startup finishes.
|
|
967
|
+
const codexOtel = new CodexOtelTracker();
|
|
968
|
+
// Codex sessions known only from `codex_*` hooks — the backstop for when the
|
|
969
|
+
// process scan can't see one (lsof timeout, no rollout held open).
|
|
970
|
+
const hookCodexSessions = new HookCodexSessions();
|
|
861
971
|
// Declare early — HTTP /health handler references this in its closure.
|
|
862
972
|
// Must be declared before the HTTP server so it's initialized (not in TDZ)
|
|
863
973
|
// when the first /health request arrives.
|
|
@@ -873,6 +983,22 @@ export async function startDaemon(opts) {
|
|
|
873
983
|
// `gatewaySessionState` / `gatewayModelName` dedicated fields.
|
|
874
984
|
let gatewaySessionState = 'idle';
|
|
875
985
|
let gatewayModelName;
|
|
986
|
+
/**
|
|
987
|
+
* Snapshot for an `openclaw` state event: the global machine with the
|
|
988
|
+
* Gateway-local model pinned back on.
|
|
989
|
+
*
|
|
990
|
+
* The Gateway's `model_info` no longer writes `core.stateMachine` (see the
|
|
991
|
+
* parser branch): that machine is fed by every observed Claude/Codex/OpenCode
|
|
992
|
+
* hook, and the hub stamps its broadcasts with the user's focused session —
|
|
993
|
+
* so a model written there by a *different* agent is read as the focused
|
|
994
|
+
* session's own. That is how `GLM-5.2 (1M)` appeared as the model of a Claude
|
|
995
|
+
* session on both decks. Keeping the model out of the shared slot means the
|
|
996
|
+
* Gateway's own events have to carry it explicitly, which is what this does.
|
|
997
|
+
*/
|
|
998
|
+
const gatewaySnapshot = (base) => ({
|
|
999
|
+
...(base ?? core.stateMachine.getSnapshot()),
|
|
1000
|
+
modelName: gatewayModelName ?? null,
|
|
1001
|
+
});
|
|
876
1002
|
// Wi-Fi WebSocket e-ink panels (XTeink X3/X4 CrossPoint fork) self-register
|
|
877
1003
|
// their device roster via `client_register{clientType:"eink-device"}` — the
|
|
878
1004
|
// same volunteer model as the Stream Deck plugin. This handler lived ONLY in
|
|
@@ -937,6 +1063,9 @@ export async function startDaemon(opts) {
|
|
|
937
1063
|
* read this binding at request time rather than close over one in its TDZ.
|
|
938
1064
|
*/
|
|
939
1065
|
let speakToBoard = null;
|
|
1066
|
+
// Assigned after BridgeCore exists. The HTTP handler reads it only after
|
|
1067
|
+
// startup; nullable keeps the tiny listen→core initialization window safe.
|
|
1068
|
+
let subagentTimeline = null;
|
|
940
1069
|
// ===== HTTP server =====
|
|
941
1070
|
const httpServer = createServer((req, res) => {
|
|
942
1071
|
// APME routes: auth-gated (task prompts, project paths, hook payloads are sensitive).
|
|
@@ -971,6 +1100,11 @@ export async function startDaemon(opts) {
|
|
|
971
1100
|
gateway: gatewayAdapter?.isAlive() ? 'connected' : 'disconnected',
|
|
972
1101
|
uptime: process.uptime(), port, pid: process.pid,
|
|
973
1102
|
pairingToken: core.authToken,
|
|
1103
|
+
// Capability: this daemon drives remote sessions down their own push
|
|
1104
|
+
// socket (session_focus_down / session_command_down / session_event_up).
|
|
1105
|
+
// Workers require this flag before remote-attaching; the Swift daemon
|
|
1106
|
+
// does not advertise it and is therefore never selected as a remote hub.
|
|
1107
|
+
sameSocketControl: true,
|
|
974
1108
|
modules: moduleHealthProvider(),
|
|
975
1109
|
apme: apme
|
|
976
1110
|
? {
|
|
@@ -1067,7 +1201,7 @@ export async function startDaemon(opts) {
|
|
|
1067
1201
|
// Reply on the board's own transport (serial when it is cabled), the
|
|
1068
1202
|
// same way the chunked path did — the HTTP response only confirms
|
|
1069
1203
|
// receipt of the bytes.
|
|
1070
|
-
await finishPhotoCapture(saved,
|
|
1204
|
+
await finishPhotoCapture(saved, boardResultSinkFor(board));
|
|
1071
1205
|
return { ok: true, bytes: saved.bytes, path: saved.path };
|
|
1072
1206
|
})().then((result) => {
|
|
1073
1207
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
@@ -1078,6 +1212,113 @@ export async function startDaemon(opts) {
|
|
|
1078
1212
|
});
|
|
1079
1213
|
return;
|
|
1080
1214
|
}
|
|
1215
|
+
// Device voice upload — a whole utterance of raw PCM16LE as one request
|
|
1216
|
+
// body. Live streaming lost audio on both of the ips10's transports
|
|
1217
|
+
// (115200-baud serial ceiling; hosted-C6 WS stalls overflowing the DRAM
|
|
1218
|
+
// ring), and transcription only starts once the utterance is complete, so
|
|
1219
|
+
// buffering on-device and letting TCP handle delivery loses nothing.
|
|
1220
|
+
// Mirrors /esp32/photo, including replying on the board's own transport.
|
|
1221
|
+
if (req.method === 'POST' && pathname === '/esp32/voice') {
|
|
1222
|
+
const ip = req.socket.remoteAddress ?? '';
|
|
1223
|
+
if (!isLocalConnection(ip)) {
|
|
1224
|
+
const token = parsedUrl.searchParams.get('token') ?? '';
|
|
1225
|
+
if (!validateToken(token)) {
|
|
1226
|
+
res.writeHead(401, { 'Content-Type': 'application/json' });
|
|
1227
|
+
res.end(JSON.stringify({ error: 'Unauthorized' }));
|
|
1228
|
+
return;
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
(async () => {
|
|
1232
|
+
const chunks = [];
|
|
1233
|
+
let total = 0;
|
|
1234
|
+
for await (const chunk of req) {
|
|
1235
|
+
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
1236
|
+
total += buf.length;
|
|
1237
|
+
// 16 kHz mono PCM16 for the firmware's 30 s runaway cap is 960 KB.
|
|
1238
|
+
if (total > 2 * 1024 * 1024)
|
|
1239
|
+
throw new Error('utterance_too_large');
|
|
1240
|
+
chunks.push(buf);
|
|
1241
|
+
}
|
|
1242
|
+
if (total === 0)
|
|
1243
|
+
throw new Error('empty_body');
|
|
1244
|
+
const board = parsedUrl.searchParams.get('board') ?? 'esp32';
|
|
1245
|
+
const saved = deviceVoice.saveDirect(Buffer.concat(chunks), {
|
|
1246
|
+
board,
|
|
1247
|
+
sessionId: parsedUrl.searchParams.get('sessionId') ?? '',
|
|
1248
|
+
sampleRate: Number(parsedUrl.searchParams.get('rate') ?? 0) || 16000,
|
|
1249
|
+
durationMs: Number(parsedUrl.searchParams.get('ms') ?? 0) || 0,
|
|
1250
|
+
});
|
|
1251
|
+
// The HTTP response only confirms receipt of the bytes; the
|
|
1252
|
+
// transcript/outcome goes to the board as a voice_result frame.
|
|
1253
|
+
await finishVoiceCapture(saved, boardResultSinkFor(board));
|
|
1254
|
+
return { ok: true, bytes: total };
|
|
1255
|
+
})().then((result) => {
|
|
1256
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1257
|
+
res.end(JSON.stringify(result));
|
|
1258
|
+
}).catch((err) => {
|
|
1259
|
+
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
1260
|
+
res.end(JSON.stringify({ ok: false, error: err instanceof Error ? err.message : String(err) }));
|
|
1261
|
+
});
|
|
1262
|
+
return;
|
|
1263
|
+
}
|
|
1264
|
+
// Staged spoken reply for an `audio_http_pull` board — raw PCM16LE body.
|
|
1265
|
+
// The board fetches at its own pace (TCP flow control), which is what
|
|
1266
|
+
// keeps the hosted-C6 RX path from being burst-fed and asserting.
|
|
1267
|
+
if (req.method === 'GET' && pathname === '/esp32/voice-reply') {
|
|
1268
|
+
const ip = req.socket.remoteAddress ?? '';
|
|
1269
|
+
if (!isLocalConnection(ip)) {
|
|
1270
|
+
const token = parsedUrl.searchParams.get('token') ?? '';
|
|
1271
|
+
if (!validateToken(token)) {
|
|
1272
|
+
res.writeHead(401, { 'Content-Type': 'application/json' });
|
|
1273
|
+
res.end(JSON.stringify({ error: 'Unauthorized' }));
|
|
1274
|
+
return;
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
const board = parsedUrl.searchParams.get('board') ?? '';
|
|
1278
|
+
const staged = board ? stagedVoiceReplies.get(board) : undefined;
|
|
1279
|
+
if (!staged) {
|
|
1280
|
+
res.writeHead(404, { 'Content-Type': 'application/json' });
|
|
1281
|
+
res.end(JSON.stringify({ error: 'no_staged_reply' }));
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
// Kept until TTL rather than deleted here: an aborted download may retry.
|
|
1285
|
+
res.writeHead(200, {
|
|
1286
|
+
'Content-Type': 'application/octet-stream',
|
|
1287
|
+
'Content-Length': staged.pcm.length,
|
|
1288
|
+
'X-Sample-Rate': String(staged.sampleRate),
|
|
1289
|
+
});
|
|
1290
|
+
// Paced writes, NOT res.end(pcm): a single end() hands the whole body
|
|
1291
|
+
// to the kernel, which blasts a TCP-initial-window-sized segment burst.
|
|
1292
|
+
// TCP flow control lives in lwIP — it cannot protect the hosted C6→P4
|
|
1293
|
+
// handoff underneath it, whose RX pool exhausted and ASSERTED the board
|
|
1294
|
+
// one second into the first pull (`sdio_rx_get_buffer (*buf)`,
|
|
1295
|
+
// 2026-07-31 12:06). 2 KB per 15 ms ≈ 133 KB/s keeps at most a couple
|
|
1296
|
+
// of segments in flight while still beating playback rate 4x.
|
|
1297
|
+
{
|
|
1298
|
+
// 8 KB / 10 ms ≈ 800 KB/s — a 2 MB answer lands in ~2.5 s. The
|
|
1299
|
+
// original 2 KB / 15 ms (~133 KB/s) predated the PSRAM-mempool core
|
|
1300
|
+
// fix and made long replies take 15+ s to start, which read as "no
|
|
1301
|
+
// playback"; with the hosted mempool in PSRAM the burst constraint is
|
|
1302
|
+
// gone and only gentle pacing is kept as hygiene.
|
|
1303
|
+
const CHUNK = 8192;
|
|
1304
|
+
const GAP_MS = 10;
|
|
1305
|
+
let off = 0;
|
|
1306
|
+
const writeNext = () => {
|
|
1307
|
+
if (res.destroyed)
|
|
1308
|
+
return;
|
|
1309
|
+
if (off >= staged.pcm.length) {
|
|
1310
|
+
res.end();
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
const end = Math.min(off + CHUNK, staged.pcm.length);
|
|
1314
|
+
res.write(staged.pcm.subarray(off, end));
|
|
1315
|
+
off = end;
|
|
1316
|
+
setTimeout(writeNext, GAP_MS).unref?.();
|
|
1317
|
+
};
|
|
1318
|
+
writeNext();
|
|
1319
|
+
}
|
|
1320
|
+
return;
|
|
1321
|
+
}
|
|
1081
1322
|
if (req.method === 'POST' && pathname === '/esp32/ota') {
|
|
1082
1323
|
(async () => {
|
|
1083
1324
|
const body = await readJsonBody(req);
|
|
@@ -1086,6 +1327,11 @@ export async function startDaemon(opts) {
|
|
|
1086
1327
|
if (!target || !firmwarePath) {
|
|
1087
1328
|
throw new Error('target and firmwarePath are required');
|
|
1088
1329
|
}
|
|
1330
|
+
// Pull-OTA staging: remember the build; the board installs it on its
|
|
1331
|
+
// next feed pull (no live WS needed). See stagedFwByBoard.
|
|
1332
|
+
if (body.stage === true) {
|
|
1333
|
+
return { ok: true, staged: true, ...stageEsp32Fw(target, firmwarePath) };
|
|
1334
|
+
}
|
|
1089
1335
|
return performWifiEsp32Ota(core, target, firmwarePath);
|
|
1090
1336
|
})().then((result) => {
|
|
1091
1337
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
@@ -1096,10 +1342,118 @@ export async function startDaemon(opts) {
|
|
|
1096
1342
|
});
|
|
1097
1343
|
return;
|
|
1098
1344
|
}
|
|
1345
|
+
// ===== Pull OTA (feed-carried) — staged firmware download =====
|
|
1346
|
+
// The board saw `fw: {size, md5}` on its feed pull and fetches the image
|
|
1347
|
+
// here. Auth mirrors the feed routes (LAN needs the pairing token).
|
|
1348
|
+
if (req.method === 'GET' && pathname === '/esp32/fw') {
|
|
1349
|
+
const ip = req.socket.remoteAddress ?? '';
|
|
1350
|
+
if (!isLocalConnection(ip)) {
|
|
1351
|
+
const token = parsedUrl.searchParams.get('token') ?? '';
|
|
1352
|
+
if (!validateToken(token)) {
|
|
1353
|
+
res.writeHead(401, { 'Content-Type': 'application/json' });
|
|
1354
|
+
res.end(JSON.stringify({ error: 'Unauthorized — token required for firmware download' }));
|
|
1355
|
+
return;
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
const board = parsedUrl.searchParams.get('board') ?? '';
|
|
1359
|
+
const staged = stagedFwByBoard.get(board);
|
|
1360
|
+
if (!staged) {
|
|
1361
|
+
res.writeHead(404, { 'Content-Type': 'application/json' });
|
|
1362
|
+
res.end(JSON.stringify({ error: `nothing staged for board "${board}"` }));
|
|
1363
|
+
return;
|
|
1364
|
+
}
|
|
1365
|
+
try {
|
|
1366
|
+
const firmware = readFileSync(staged.firmwarePath);
|
|
1367
|
+
// Resumable download: `?from=<offset>` serves the remainder. A 5 MB
|
|
1368
|
+
// image over a marginal link never survives one connection (this
|
|
1369
|
+
// bridge host's two-NICs-on-one-subnet setup black-holes full-MTU
|
|
1370
|
+
// segments in one direction, and a restart-from-zero download can
|
|
1371
|
+
// never converge there). The device appends whatever arrives and
|
|
1372
|
+
// re-asks from where it stopped, so partial transfers accumulate.
|
|
1373
|
+
const fromRaw = Number(parsedUrl.searchParams.get('from') ?? 0);
|
|
1374
|
+
const from = Number.isFinite(fromRaw) && fromRaw > 0 ? Math.min(Math.floor(fromRaw), firmware.length) : 0;
|
|
1375
|
+
const body = from > 0 ? firmware.subarray(from) : firmware;
|
|
1376
|
+
log(`[agentdeck] pull-OTA download: ${board} (${ip}) ← ${body.length} bytes`
|
|
1377
|
+
+ (from > 0 ? ` (resume from ${from}/${firmware.length})` : ''));
|
|
1378
|
+
res.writeHead(from > 0 ? 206 : 200, {
|
|
1379
|
+
'Content-Type': 'application/octet-stream',
|
|
1380
|
+
'Content-Length': String(body.length),
|
|
1381
|
+
...(from > 0
|
|
1382
|
+
? { 'Content-Range': `bytes ${from}-${Math.max(from, firmware.length - 1)}/${firmware.length}` }
|
|
1383
|
+
: {}),
|
|
1384
|
+
'X-FW-MD5': staged.md5,
|
|
1385
|
+
'Cache-Control': 'no-store',
|
|
1386
|
+
});
|
|
1387
|
+
res.end(body);
|
|
1388
|
+
}
|
|
1389
|
+
catch (err) {
|
|
1390
|
+
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
1391
|
+
res.end(JSON.stringify({ error: err instanceof Error ? err.message : String(err) }));
|
|
1392
|
+
}
|
|
1393
|
+
return;
|
|
1394
|
+
}
|
|
1099
1395
|
// ===== Card Feed pull sync (M6) — wake-sync-sleep battery clients =====
|
|
1100
1396
|
// Auth mirrors /apme: local connections are free; anything else needs the
|
|
1101
1397
|
// pairing token (?token=). Devices hold it from provisioning; /health
|
|
1102
1398
|
// exposes it for LAN pairing.
|
|
1399
|
+
// ===== Glance Frame (M8) — daemon-rendered pixels of the glance =====
|
|
1400
|
+
// The device-side glance renderer is the offline fallback; this is the
|
|
1401
|
+
// rich face. `?format=png` returns the exact dithered panel pixels, so a
|
|
1402
|
+
// browser preview IS the production frame (protocol.ts § Glance Frame).
|
|
1403
|
+
if (req.method === 'GET' && pathname === GLANCE_FRAME_PATH) {
|
|
1404
|
+
const ip = req.socket.remoteAddress ?? '';
|
|
1405
|
+
if (!isLocalConnection(ip)) {
|
|
1406
|
+
const token = parsedUrl.searchParams.get('token') ?? '';
|
|
1407
|
+
if (!validateToken(token)) {
|
|
1408
|
+
res.writeHead(401, { 'Content-Type': 'application/json' });
|
|
1409
|
+
res.end(JSON.stringify({ error: 'Unauthorized — token required for glance frame' }));
|
|
1410
|
+
return;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
(async () => {
|
|
1414
|
+
const board = parsedUrl.searchParams.get('board') ?? 'xteink_x3';
|
|
1415
|
+
const preset = GLANCE_FRAME_BOARDS[board];
|
|
1416
|
+
const w = Number(parsedUrl.searchParams.get('w'));
|
|
1417
|
+
const h = Number(parsedUrl.searchParams.get('h'));
|
|
1418
|
+
const geometry = Number.isFinite(w) && Number.isFinite(h) && w >= 128 && h >= 128 && w <= 1600 && h <= 1600
|
|
1419
|
+
? { width: Math.round(w), height: Math.round(h), landscape: w > h }
|
|
1420
|
+
: preset ?? GLANCE_FRAME_BOARDS.xteink_x3;
|
|
1421
|
+
const sessions = await core.buildSessionsSnapshot();
|
|
1422
|
+
const glance = buildGlance({
|
|
1423
|
+
sessions: sessions,
|
|
1424
|
+
usage: core.buildUsage(),
|
|
1425
|
+
weather: await glanceWeather.get(parseWeatherSettings(loadDaemonSettings())),
|
|
1426
|
+
events: await glanceCalendar.get(parseCalendarSettings(loadDaemonSettings())),
|
|
1427
|
+
});
|
|
1428
|
+
const d = new Date();
|
|
1429
|
+
const serverHm = `${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}`;
|
|
1430
|
+
const frame = await renderGlanceFrame({ glance, serverHm, geometry });
|
|
1431
|
+
const headers = {
|
|
1432
|
+
'X-Frame-Width': String(frame.width),
|
|
1433
|
+
'X-Frame-Height': String(frame.height),
|
|
1434
|
+
'X-Frame-Sig': frame.sig,
|
|
1435
|
+
'Cache-Control': 'no-store',
|
|
1436
|
+
};
|
|
1437
|
+
// Conditional fetch, one layer down from the feed's deckSig: matching
|
|
1438
|
+
// sig → 304, no body, the panel keeps holding the frame it has.
|
|
1439
|
+
if (parsedUrl.searchParams.get('sig') === frame.sig) {
|
|
1440
|
+
res.writeHead(304, headers);
|
|
1441
|
+
res.end();
|
|
1442
|
+
return;
|
|
1443
|
+
}
|
|
1444
|
+
if (parsedUrl.searchParams.get('format') === 'png') {
|
|
1445
|
+
res.writeHead(200, { ...headers, 'Content-Type': 'image/png' });
|
|
1446
|
+
res.end(await frame.png());
|
|
1447
|
+
return;
|
|
1448
|
+
}
|
|
1449
|
+
res.writeHead(200, { ...headers, 'Content-Type': 'application/octet-stream' });
|
|
1450
|
+
res.end(frame.packed);
|
|
1451
|
+
})().catch((err) => {
|
|
1452
|
+
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
1453
|
+
res.end(JSON.stringify({ error: err instanceof Error ? err.message : String(err) }));
|
|
1454
|
+
});
|
|
1455
|
+
return;
|
|
1456
|
+
}
|
|
1103
1457
|
if ((req.method === 'GET' && pathname === CARD_FEED_PATH)
|
|
1104
1458
|
|| (req.method === 'POST' && pathname === CARD_OUTBOX_PATH)) {
|
|
1105
1459
|
const ip = req.socket.remoteAddress ?? '';
|
|
@@ -1114,7 +1468,33 @@ export async function startDaemon(opts) {
|
|
|
1114
1468
|
(async () => {
|
|
1115
1469
|
if (req.method === 'GET') {
|
|
1116
1470
|
const sessions = await core.buildSessionsSnapshot();
|
|
1117
|
-
const
|
|
1471
|
+
const settings = loadDaemonSettings();
|
|
1472
|
+
pocketAutonomy.configure(parsePocketAutonomyConfig(settings));
|
|
1473
|
+
// Sleep-dashboard glance: provider quota + work wrap-up + weather.
|
|
1474
|
+
// Weather resolves from cache almost always (30 min window) and is
|
|
1475
|
+
// bounded by its own fetch timeout, so the pull stays fast.
|
|
1476
|
+
const glance = buildGlance({
|
|
1477
|
+
sessions: sessions,
|
|
1478
|
+
usage: core.buildUsage(),
|
|
1479
|
+
weather: await glanceWeather.get(parseWeatherSettings(settings)),
|
|
1480
|
+
events: await glanceCalendar.get(parseCalendarSettings(settings)),
|
|
1481
|
+
});
|
|
1482
|
+
const now = Date.now();
|
|
1483
|
+
const pocketReader = parsedUrl.searchParams.get('surface') === 'pocket-reader';
|
|
1484
|
+
const feed = buildCardFeed(sessions, now, pocketReader ? pocketReaderModules : pocketCardModules, {
|
|
1485
|
+
glance,
|
|
1486
|
+
echoSig: parsedUrl.searchParams.get('sig') ?? undefined,
|
|
1487
|
+
includeSessions: !pocketReader,
|
|
1488
|
+
});
|
|
1489
|
+
// Pull-OTA advert: rides full AND `unchanged` responses, so a
|
|
1490
|
+
// sleeping board learns about a staged build on any pull. Board
|
|
1491
|
+
// identity from the query (newer firmware) or the IP memory.
|
|
1492
|
+
const pullBoard = parsedUrl.searchParams.get('board') ?? boardForClientIp(ip);
|
|
1493
|
+
if (!feed.unchanged)
|
|
1494
|
+
pocketAutonomy.observeDelivery(feed.cards, now, pullBoard ?? undefined);
|
|
1495
|
+
const fwAdvert = stagedFwAdvert(pullBoard ?? undefined);
|
|
1496
|
+
if (fwAdvert)
|
|
1497
|
+
feed.fw = fwAdvert;
|
|
1118
1498
|
// A sleeping client's whole visit is this one request: no body, no
|
|
1119
1499
|
// board id, nothing pushed. Record it, because the gap between two
|
|
1120
1500
|
// pulls is the only evidence that the timer wake fired at all.
|
|
@@ -1122,6 +1502,8 @@ export async function startDaemon(opts) {
|
|
|
1122
1502
|
log(`[agentdeck] ${formatFeedPull(feedPulls.record(ip, {
|
|
1123
1503
|
cards: feed.cards.length,
|
|
1124
1504
|
nextPullSec: feed.nextPullSec,
|
|
1505
|
+
unchanged: feed.unchanged === true,
|
|
1506
|
+
telemetry: parsePullTelemetry(parsedUrl.searchParams),
|
|
1125
1507
|
}))}`);
|
|
1126
1508
|
return feed;
|
|
1127
1509
|
}
|
|
@@ -1136,6 +1518,7 @@ export async function startDaemon(opts) {
|
|
|
1136
1518
|
sessions: sessions,
|
|
1137
1519
|
isPendingRequest,
|
|
1138
1520
|
dispatch: (cmd) => handleDeviceCommand(cmd),
|
|
1521
|
+
modules: pocketCardModules,
|
|
1139
1522
|
});
|
|
1140
1523
|
const applied = result.results.filter((r) => r.status === 'applied').length;
|
|
1141
1524
|
if (result.results.length > 0) {
|
|
@@ -1175,6 +1558,12 @@ export async function startDaemon(opts) {
|
|
|
1175
1558
|
billingType: snap.billingType,
|
|
1176
1559
|
},
|
|
1177
1560
|
wsClients: core.wsServer.getClientCount(),
|
|
1561
|
+
// Codex observation backstops — which sessions each source believes in.
|
|
1562
|
+
// A session present here but absent from sessions_list means the ps
|
|
1563
|
+
// observer found it too and won, which is the expected steady state.
|
|
1564
|
+
hookCodexSessions: hookCodexSessions.snapshot(),
|
|
1565
|
+
codexOtelThreads: codexOtel.snapshot(),
|
|
1566
|
+
pocketAutonomy: pocketAutonomy.diagnostics(),
|
|
1178
1567
|
recentJournal: [],
|
|
1179
1568
|
ptyTail: '',
|
|
1180
1569
|
journalDir: join(homedir(), '.agentdeck'),
|
|
@@ -1272,6 +1661,41 @@ export async function startDaemon(opts) {
|
|
|
1272
1661
|
// codex/opencode *state* is owned by the passive observer's turn
|
|
1273
1662
|
// semantics, not these hooks.
|
|
1274
1663
|
const { boundary, agentType: hookAgentType } = classifyObservedHookEvent(eventName, mapped);
|
|
1664
|
+
const earlyHookSid = typeof json.session_id === 'string' && json.session_id
|
|
1665
|
+
? json.session_id : 'daemon-hook';
|
|
1666
|
+
const earlyHookCwd = (typeof json.cwd === 'string' ? json.cwd
|
|
1667
|
+
: (typeof json.project_path === 'string' ? json.project_path : '')) || '';
|
|
1668
|
+
const earlyHookProject = (typeof json.project_name === 'string' && json.project_name)
|
|
1669
|
+
? json.project_name
|
|
1670
|
+
: (earlyHookCwd ? earlyHookCwd.split('/').filter(Boolean).pop() : undefined);
|
|
1671
|
+
const childHook = subagentTimeline?.handle({
|
|
1672
|
+
eventName,
|
|
1673
|
+
payload: json,
|
|
1674
|
+
sessionId: earlyHookSid,
|
|
1675
|
+
agentType: hookAgentType,
|
|
1676
|
+
projectName: earlyHookProject,
|
|
1677
|
+
}).childOnly === true;
|
|
1678
|
+
if (childHook) {
|
|
1679
|
+
// Child activity is observation-only. Never let child PreToolUse
|
|
1680
|
+
// reach the parent's approval, STOP, state, or APME paths.
|
|
1681
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1682
|
+
res.end(eventName === 'PreToolUse' || eventName === 'Stop'
|
|
1683
|
+
? ''
|
|
1684
|
+
: JSON.stringify({ received: true }));
|
|
1685
|
+
return;
|
|
1686
|
+
}
|
|
1687
|
+
// Hook-derived Codex session rows. Placed after the child-hook return so
|
|
1688
|
+
// subagent lifecycle never drives the parent row, and kept independent
|
|
1689
|
+
// of the state machine: this only decides whether a Codex the process
|
|
1690
|
+
// scan can't see still has a session. See bridge/src/hook-codex-sessions.ts.
|
|
1691
|
+
if (eventName.startsWith('codex_')) {
|
|
1692
|
+
hookCodexSessions.note(eventName, {
|
|
1693
|
+
sessionId: typeof json.session_id === 'string' ? json.session_id : undefined,
|
|
1694
|
+
cwd: earlyHookCwd || undefined,
|
|
1695
|
+
projectName: earlyHookProject,
|
|
1696
|
+
toolName: typeof json.tool_name === 'string' ? json.tool_name : undefined,
|
|
1697
|
+
});
|
|
1698
|
+
}
|
|
1275
1699
|
// State machine
|
|
1276
1700
|
if (mapped === 'session_start')
|
|
1277
1701
|
core.stateMachine.handleHookEvent('SessionStart', json);
|
|
@@ -1724,6 +2148,42 @@ export async function startDaemon(opts) {
|
|
|
1724
2148
|
});
|
|
1725
2149
|
return;
|
|
1726
2150
|
}
|
|
2151
|
+
// Codex OTel exporter target. Codex (CLI and the ChatGPT.app desktop app)
|
|
2152
|
+
// POSTs OTLP/HTTP JSON spans here when `[otel.trace_exporter.otlp-http]`
|
|
2153
|
+
// points at the daemon. Only the Swift daemon used to implement this, so a
|
|
2154
|
+
// Node-daemon host answered 404 and dropped every span. Always 200: an OTel
|
|
2155
|
+
// exporter that sees errors backs off, and nothing here is worth telling
|
|
2156
|
+
// Codex about. See bridge/src/codex-otel.ts.
|
|
2157
|
+
if (req.method === 'POST' && (pathname === CODEX_OTEL_TRACES_PATH || pathname === '/v1/traces')) {
|
|
2158
|
+
let body = '';
|
|
2159
|
+
req.on('data', (c) => { body += c; if (body.length > 8_000_000)
|
|
2160
|
+
req.destroy(); });
|
|
2161
|
+
req.on('end', () => {
|
|
2162
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
2163
|
+
res.end('{}');
|
|
2164
|
+
try {
|
|
2165
|
+
const json = body ? JSON.parse(body) : {};
|
|
2166
|
+
// Dev capture. Codex's span vocabulary is not a stable API, so when it
|
|
2167
|
+
// shifts the only way to re-derive the dispatch table is to look at
|
|
2168
|
+
// real batches: `AGENTDECK_CODEX_OTEL_DUMP=/tmp/otel.jsonl agentdeck
|
|
2169
|
+
// daemon start`. Bodies are pretty-printed and large — point it at a
|
|
2170
|
+
// scratch path and delete it afterwards.
|
|
2171
|
+
if (process.env.AGENTDECK_CODEX_OTEL_DUMP) {
|
|
2172
|
+
try {
|
|
2173
|
+
appendFileSync(process.env.AGENTDECK_CODEX_OTEL_DUMP, `${body}\n`);
|
|
2174
|
+
}
|
|
2175
|
+
catch { /* dev only */ }
|
|
2176
|
+
}
|
|
2177
|
+
if (codexOtel.ingest(json) === 0) {
|
|
2178
|
+
debugThrottled('codex-otel', 'no-recognized-spans', 60_000, `no recognized spans; len=${body.length} names=${spanNameSummary(json)}`);
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
2181
|
+
catch {
|
|
2182
|
+
debugThrottled('codex-otel', 'unparsable-body', 60_000, `unparsable body len=${body.length}`);
|
|
2183
|
+
}
|
|
2184
|
+
});
|
|
2185
|
+
return;
|
|
2186
|
+
}
|
|
1727
2187
|
// OpenCode observer-plugin steering: the plugin long-polls here and
|
|
1728
2188
|
// executes returned commands via its in-process SDK client (abort /
|
|
1729
2189
|
// prompt). See bridge/src/opencode-steering.ts.
|
|
@@ -1955,6 +2415,9 @@ export async function startDaemon(opts) {
|
|
|
1955
2415
|
httpServer,
|
|
1956
2416
|
isDaemon: true,
|
|
1957
2417
|
});
|
|
2418
|
+
subagentTimeline = new SubagentTimelineTracker((entry) => {
|
|
2419
|
+
core.bridgeTimeline.addEntry(entry, { bypassSuppression: true });
|
|
2420
|
+
});
|
|
1958
2421
|
// Hooks resolve a fallback daemon port through daemon.json. Another process
|
|
1959
2422
|
// can delete that file while this daemon remains healthy, which makes every
|
|
1960
2423
|
// standalone Claude/Codex hook silently fall back to an empty :9120. Mirror
|
|
@@ -2028,6 +2491,24 @@ export async function startDaemon(opts) {
|
|
|
2028
2491
|
core.wireDisplayMonitor();
|
|
2029
2492
|
let lastStateEvent = null;
|
|
2030
2493
|
let userFocusedSessionId = null;
|
|
2494
|
+
/**
|
|
2495
|
+
* Stamp the focused session onto a hub `state_update`.
|
|
2496
|
+
*
|
|
2497
|
+
* This field carries two jobs, and both are load-bearing, which is why it
|
|
2498
|
+
* cannot simply be narrowed:
|
|
2499
|
+
* - attribution: `buildStateEvent` sets no `sessionId` at all, so a
|
|
2500
|
+
* hook-observed session's AWAITING_PERMISSION options reach the decks
|
|
2501
|
+
* attributed only by this field (see session-deck-awaiting tests).
|
|
2502
|
+
* - focus ack: the Apple app toggles creature focus against it and needs it
|
|
2503
|
+
* even for a session that emits nothing of its own.
|
|
2504
|
+
*
|
|
2505
|
+
* The consequence is a standing invariant on the payload: whatever rides along
|
|
2506
|
+
* here is read as the focused session's own. So nothing may write a *different*
|
|
2507
|
+
* agent's per-session field into `core.stateMachine` — the Gateway's model went
|
|
2508
|
+
* in and came out as a Claude session's model. Gateway-specific state lives in
|
|
2509
|
+
* `gatewaySessionState` / `gatewayModelName` and is re-attached by
|
|
2510
|
+
* `gatewaySnapshot()`.
|
|
2511
|
+
*/
|
|
2031
2512
|
const attachFocusedSessionId = (event) => {
|
|
2032
2513
|
if (event.type !== 'state_update')
|
|
2033
2514
|
return event;
|
|
@@ -2036,6 +2517,28 @@ export async function startDaemon(opts) {
|
|
|
2036
2517
|
focusedSessionId: userFocusedSessionId ?? '',
|
|
2037
2518
|
};
|
|
2038
2519
|
};
|
|
2520
|
+
/**
|
|
2521
|
+
* Stamp a *Gateway-specific* state event — one whose payload really is the
|
|
2522
|
+
* Gateway's (its parser events, its connection transitions), as opposed to the
|
|
2523
|
+
* hub broadcasts above that merely get labelled `openclaw` while the Gateway is
|
|
2524
|
+
* alive.
|
|
2525
|
+
*
|
|
2526
|
+
* These name their own row (`sessionId`), which is the honest attribution and
|
|
2527
|
+
* the reason a client never has to guess: an event carrying the Gateway's model
|
|
2528
|
+
* now says so. The focus ack rides only when the Gateway IS the focused row —
|
|
2529
|
+
* otherwise this event would claim to describe a Claude session, which is
|
|
2530
|
+
* precisely how `GLM-5.2 (1M)` became that session's model on both decks. The
|
|
2531
|
+
* ack still reaches the Apple app through every hub broadcast.
|
|
2532
|
+
*/
|
|
2533
|
+
const attachGatewayEvent = (event) => {
|
|
2534
|
+
if (event.type !== 'state_update')
|
|
2535
|
+
return event;
|
|
2536
|
+
return {
|
|
2537
|
+
...event,
|
|
2538
|
+
sessionId: OPENCLAW_SESSION_ID,
|
|
2539
|
+
focusedSessionId: userFocusedSessionId === OPENCLAW_SESSION_ID ? OPENCLAW_SESSION_ID : '',
|
|
2540
|
+
};
|
|
2541
|
+
};
|
|
2039
2542
|
const broadcastFocusedState = () => {
|
|
2040
2543
|
const gwAlive = gatewayAdapter?.isAlive() ?? false;
|
|
2041
2544
|
const stateEvent = attachFocusedSessionId(core.buildStateEvent({
|
|
@@ -2097,6 +2600,12 @@ export async function startDaemon(opts) {
|
|
|
2097
2600
|
timelineRelay.start();
|
|
2098
2601
|
// Session focus relay — allows SD plugin to interact with a specific session via daemon
|
|
2099
2602
|
const focusRelay = new SessionFocusRelay();
|
|
2603
|
+
// Remote (cross-machine) sessions aren't in sessions.json. Their only
|
|
2604
|
+
// reverse path is same-socket: drive the session down the live push socket
|
|
2605
|
+
// it opened — no inbound dial, so NAT'd / SSH-only workers just work.
|
|
2606
|
+
focusRelay.setSameSocketResolver((sessionId) => getRemoteSender(sessionId) ?? null);
|
|
2607
|
+
// Push-channel frames from session bridges (register / state / reverse events).
|
|
2608
|
+
const handlePushChannelMessage = createPushChannelHandler({ core, focusRelay });
|
|
2100
2609
|
focusRelay.setEventHandler((evt) => {
|
|
2101
2610
|
if (evt.type === 'state_update') {
|
|
2102
2611
|
const focusedId = focusRelay.getFocusedSessionId();
|
|
@@ -2121,14 +2630,43 @@ export async function startDaemon(opts) {
|
|
|
2121
2630
|
else if (evt.type === 'usage_update') {
|
|
2122
2631
|
// Sync daemon cache with relay's already-adjusted values (prevents oscillation)
|
|
2123
2632
|
const u = evt;
|
|
2633
|
+
const hasClaudeData = u.fiveHourPercent != null || u.sevenDayPercent != null;
|
|
2124
2634
|
if (core.cachedApiUsage && u.fiveHourPercent != null) {
|
|
2125
2635
|
core.cachedApiUsage.fiveHourPercent = u.fiveHourPercent;
|
|
2126
2636
|
core.cachedApiUsage.fiveHourResetsAt = u.fiveHourResetsAt ?? null;
|
|
2127
2637
|
core.cachedApiUsage.sevenDayPercent = u.sevenDayPercent ?? core.cachedApiUsage.sevenDayPercent;
|
|
2128
2638
|
core.cachedApiUsage.sevenDayResetsAt = u.sevenDayResetsAt ?? core.cachedApiUsage.sevenDayResetsAt ?? null;
|
|
2639
|
+
// Scoped caps ride the same relay snapshot. The Swift daemon syncs them on
|
|
2640
|
+
// its equivalent live-update path; leaving them out here made the daemon's
|
|
2641
|
+
// own next broadcastUsage re-emit a stale scoped set until the next full
|
|
2642
|
+
// fetch — the exact drift DaemonServer.swift documents.
|
|
2643
|
+
core.cachedApiUsage.scopedLimits = Array.isArray(u.scopedLimits) ? u.scopedLimits : [];
|
|
2129
2644
|
core.apiUsagePreAdjusted = true;
|
|
2130
2645
|
}
|
|
2131
|
-
|
|
2646
|
+
// A session bridge broadcasts usage on EVERY state change. In daemon-first
|
|
2647
|
+
// mode its snapshot usually has no model context, so the Claude quota model
|
|
2648
|
+
// gate (isClaudeSubscriptionModel) yields no percents and the event carries
|
|
2649
|
+
// usageStale=true. Forwarding that bare event verbatim clobbered the daemon's
|
|
2650
|
+
// own authoritative aggregate on the dashboard: the Claude gauge blanked to
|
|
2651
|
+
// "No usage data" on every state tick (the Codex gauge, which ignores global
|
|
2652
|
+
// staleness, stayed put — hence the Claude-only flicker).
|
|
2653
|
+
//
|
|
2654
|
+
// Suppressing the relay instead is NOT an option: usage_update is the ONLY
|
|
2655
|
+
// carrier of per-session tokens / cost / duration (state_update has none of
|
|
2656
|
+
// those fields), and the daemon's own buildUsage() reads a UsageTracker that
|
|
2657
|
+
// no PTY ever feeds, so its session half is zeros. Dropping the event would
|
|
2658
|
+
// blank the session numbers for every focus that legitimately carries no
|
|
2659
|
+
// Claude quota — Codex, OpenCode, API billing.
|
|
2660
|
+
//
|
|
2661
|
+
// So split the event by half instead of by event: the ACCOUNT half (quota,
|
|
2662
|
+
// scoped caps, extra usage, subscriptions, Codex/Ollama, usageStale) comes
|
|
2663
|
+
// from the daemon's authoritative aggregate, the SESSION half from the relay.
|
|
2664
|
+
if (hasClaudeData) {
|
|
2665
|
+
core.wsServer.broadcast(evt);
|
|
2666
|
+
}
|
|
2667
|
+
else {
|
|
2668
|
+
core.wsServer.broadcast(mergeRelayedSessionUsage(core.buildUsage(), u));
|
|
2669
|
+
}
|
|
2132
2670
|
}
|
|
2133
2671
|
else {
|
|
2134
2672
|
// prompt_options — relay as-is
|
|
@@ -2333,6 +2871,11 @@ export async function startDaemon(opts) {
|
|
|
2333
2871
|
// immediately). When a scan lands fresh observations, push them out via
|
|
2334
2872
|
// the debounced broadcast so clients don't wait for the next 10 s poll.
|
|
2335
2873
|
passiveSessionObserver.onRefreshed = () => core.maybeBroadcastSessionsList();
|
|
2874
|
+
// OTel turn boundaries arrive in milliseconds where the observer only re-reads
|
|
2875
|
+
// the rollout tail every scan interval, so a span that changed thread state is
|
|
2876
|
+
// worth a broadcast of its own.
|
|
2877
|
+
codexOtel.onChanged = () => core.maybeBroadcastSessionsList();
|
|
2878
|
+
hookCodexSessions.onChanged = () => core.maybeBroadcastSessionsList();
|
|
2336
2879
|
// ===== Gateway adapter lifecycle =====
|
|
2337
2880
|
// (gatewayAdapter + gatewayConnecting declared earlier, before HTTP server)
|
|
2338
2881
|
// Inject OpenClaw virtual session only after Gateway authentication succeeds.
|
|
@@ -2344,7 +2887,11 @@ export async function startDaemon(opts) {
|
|
|
2344
2887
|
// the 5s-throttled observer, so a Notification arriving mid-window still
|
|
2345
2888
|
// surfaces within one frame. Key = the Claude session UUID embedded in
|
|
2346
2889
|
// `observed:claude:<uuid>`.
|
|
2347
|
-
|
|
2890
|
+
// Codex OTel spans overlay state/tool onto the matching ps-observed row and
|
|
2891
|
+
// synthesize a `codex-app` row for any thread the process scan missed —
|
|
2892
|
+
// Swift's only Codex-app source, here a second opinion on top of the
|
|
2893
|
+
// observer. See bridge/src/codex-otel.ts.
|
|
2894
|
+
const observed = applyAwaitingOverlayToObserved(hookCodexSessions.applyTo(codexOtel.applyTo(passiveSessionObserver.collect(sessions))))
|
|
2348
2895
|
.map((s) => {
|
|
2349
2896
|
// Steering feedback for observed Claude sessions: devices render
|
|
2350
2897
|
// "stopping at next tool" / queued-directive badges from these.
|
|
@@ -2363,10 +2910,15 @@ export async function startDaemon(opts) {
|
|
|
2363
2910
|
...(snap.queuedDirectives > 0 ? { queuedDirectives: snap.queuedDirectives } : {}),
|
|
2364
2911
|
};
|
|
2365
2912
|
});
|
|
2913
|
+
// Sessions that attached from another machine over the push channel. They
|
|
2914
|
+
// aren't in this daemon's sessions.json, so inject them here (dedup against
|
|
2915
|
+
// any id already present from the local registry).
|
|
2916
|
+
const localIds = new Set(sessions.map((s) => s.id));
|
|
2917
|
+
const remote = listRemoteEnriched().filter((s) => !localIds.has(s.id));
|
|
2366
2918
|
// Derive per-session elapsed seconds from startedAt so NTP-less devices
|
|
2367
2919
|
// (ESP32 IPS10 mosaic) render an elapsed value per cell without a wall clock.
|
|
2368
2920
|
const now = Date.now();
|
|
2369
|
-
const enrichedSessions = [...sessions, ...observed].map((s) => {
|
|
2921
|
+
const enrichedSessions = [...sessions, ...observed, ...remote].map((s) => {
|
|
2370
2922
|
// On-demand review badge (REVIEW tile verdict / REVIEWING state) —
|
|
2371
2923
|
// applies to every session type, managed included.
|
|
2372
2924
|
const review = reviewSnapshot(s.id);
|
|
@@ -2436,7 +2988,14 @@ export async function startDaemon(opts) {
|
|
|
2436
2988
|
core.stateMachine.handleHookEvent('SessionEnd', {});
|
|
2437
2989
|
break;
|
|
2438
2990
|
case 'parser':
|
|
2439
|
-
|
|
2991
|
+
// `model_info` is deliberately NOT forwarded: the global machine holds
|
|
2992
|
+
// one modelName for the whole hub, fed by every observed hook session,
|
|
2993
|
+
// and its broadcasts are stamped with the user's focused session — so a
|
|
2994
|
+
// model written here by the Gateway is read as that session's own
|
|
2995
|
+
// (GLM on a Claude row). It lives in `gatewayModelName` instead, and
|
|
2996
|
+
// `gatewaySnapshot()` puts it back on the Gateway's own events.
|
|
2997
|
+
if (evt.event !== 'model_info')
|
|
2998
|
+
core.stateMachine.handleParserEvent(evt.event, evt.data);
|
|
2440
2999
|
// Gateway-local activity state for the virtual session row (mirror of
|
|
2441
3000
|
// Swift `gatewaySessionState`). Kept separate from the global
|
|
2442
3001
|
// `core.stateMachine` above so the OpenClaw row reflects only the
|
|
@@ -2465,10 +3024,10 @@ export async function startDaemon(opts) {
|
|
|
2465
3024
|
if (models) {
|
|
2466
3025
|
core.cachedModelCatalog = models;
|
|
2467
3026
|
const snap = core.stateMachine.getSnapshot();
|
|
2468
|
-
const stateEvent =
|
|
3027
|
+
const stateEvent = attachGatewayEvent(core.buildStateEvent({
|
|
2469
3028
|
agentType: 'openclaw',
|
|
2470
3029
|
agentCapabilities: OPENCLAW_CAPABILITIES,
|
|
2471
|
-
snapshot: snap,
|
|
3030
|
+
snapshot: gatewaySnapshot(snap),
|
|
2472
3031
|
}));
|
|
2473
3032
|
lastStateEvent = stateEvent;
|
|
2474
3033
|
core.broadcast(stateEvent);
|
|
@@ -2521,10 +3080,10 @@ export async function startDaemon(opts) {
|
|
|
2521
3080
|
}
|
|
2522
3081
|
// Force full state broadcast
|
|
2523
3082
|
const snap = core.stateMachine.getSnapshot();
|
|
2524
|
-
const gwStateEvent =
|
|
3083
|
+
const gwStateEvent = attachGatewayEvent(core.buildStateEvent({
|
|
2525
3084
|
agentType: 'openclaw',
|
|
2526
3085
|
agentCapabilities: OPENCLAW_CAPABILITIES,
|
|
2527
|
-
snapshot: snap,
|
|
3086
|
+
snapshot: gatewaySnapshot(snap),
|
|
2528
3087
|
}));
|
|
2529
3088
|
lastStateEvent = gwStateEvent;
|
|
2530
3089
|
core.wsServer.broadcast(gwStateEvent);
|
|
@@ -2736,24 +3295,10 @@ export async function startDaemon(opts) {
|
|
|
2736
3295
|
if (handleEsp32OtaReply(msg)) {
|
|
2737
3296
|
return true;
|
|
2738
3297
|
}
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
try {
|
|
2744
|
-
sender.send(JSON.stringify({ type: 'session_push_ack', sessionId }));
|
|
2745
|
-
}
|
|
2746
|
-
catch { /* client disconnecting */ }
|
|
2747
|
-
return true; // consumed
|
|
2748
|
-
}
|
|
2749
|
-
if (msg.type === 'session_push_state') {
|
|
2750
|
-
const { sessionId, state, modelName, effortLevel, permissionMode } = msg;
|
|
2751
|
-
if (sessionId && state) {
|
|
2752
|
-
updatePushState(sessionId, state, modelName, effortLevel, permissionMode);
|
|
2753
|
-
// Trigger sessions list broadcast so clients get fresh state
|
|
2754
|
-
core.maybeBroadcastSessionsList();
|
|
2755
|
-
}
|
|
2756
|
-
return true; // consumed
|
|
3298
|
+
// Push-channel frames (session_push_register / session_push_state /
|
|
3299
|
+
// session_event_up) — extracted handler, integration-tested directly.
|
|
3300
|
+
if (handlePushChannelMessage(msg, sender)) {
|
|
3301
|
+
return true;
|
|
2757
3302
|
}
|
|
2758
3303
|
if (msg.type === 'deck_slot_map') {
|
|
2759
3304
|
// Plugin pushed its keypad layout. Forward to other viewers (extra
|
|
@@ -2770,8 +3315,24 @@ export async function startDaemon(opts) {
|
|
|
2770
3315
|
}
|
|
2771
3316
|
if (msg.type === 'voice_end') {
|
|
2772
3317
|
const captured = deviceVoice.end(sender, msg);
|
|
2773
|
-
if (captured)
|
|
3318
|
+
if (captured) {
|
|
2774
3319
|
void finishVoiceCapture(captured, stableSink(sender));
|
|
3320
|
+
}
|
|
3321
|
+
else if (msg.cancel !== true) {
|
|
3322
|
+
// Nothing was captured but the user did hold the button — answer
|
|
3323
|
+
// anyway, or the board's "Sending…" state has nothing to resolve it.
|
|
3324
|
+
try {
|
|
3325
|
+
sender.send(JSON.stringify({ type: 'voice_result', text: '', error: 'empty_capture' }));
|
|
3326
|
+
}
|
|
3327
|
+
catch { /* client disconnecting */ }
|
|
3328
|
+
}
|
|
3329
|
+
return true; // consumed
|
|
3330
|
+
}
|
|
3331
|
+
if (msg.type === 'voice_abort') {
|
|
3332
|
+
// The board tried to upload an utterance over HTTP and failed; the
|
|
3333
|
+
// reason would otherwise only exist on its serial console.
|
|
3334
|
+
log(`[agentdeck] voice: device upload aborted (${String(msg.reason ?? 'unknown')}, `
|
|
3335
|
+
+ `${String(msg.total ?? '?')} bytes)`);
|
|
2775
3336
|
return true; // consumed
|
|
2776
3337
|
}
|
|
2777
3338
|
if (msg.type === 'photo_begin') {
|
|
@@ -2895,6 +3456,13 @@ export async function startDaemon(opts) {
|
|
|
2895
3456
|
}
|
|
2896
3457
|
const handleDeviceCommand = (cmd) => {
|
|
2897
3458
|
debug('daemon', `cmd: ${cmd.type}`);
|
|
3459
|
+
// Host push-to-talk is the daemon's own capability (host mic + speakers),
|
|
3460
|
+
// so claim it ahead of the gateway-first-dibs routing below — it must not
|
|
3461
|
+
// depend on what any adapter chooses to do with the legacy `voice` shape.
|
|
3462
|
+
if (cmd.type === 'voice') {
|
|
3463
|
+
handleHostVoicePtt(cmd);
|
|
3464
|
+
return;
|
|
3465
|
+
}
|
|
2898
3466
|
// Session-scoped commands are NEVER OpenClaw's to consume: a device
|
|
2899
3467
|
// answering a named session (select_option{sessionId}) must reach the
|
|
2900
3468
|
// session routing below — the gateway swallowing it as an exec-approval
|
|
@@ -2929,10 +3497,10 @@ export async function startDaemon(opts) {
|
|
|
2929
3497
|
if (target === 'openclaw' && gatewayAdapter?.isAlive()) {
|
|
2930
3498
|
// Force broadcast OpenClaw state to all clients
|
|
2931
3499
|
const snap = core.stateMachine.getSnapshot();
|
|
2932
|
-
const gwStateEvent =
|
|
3500
|
+
const gwStateEvent = attachGatewayEvent(core.buildStateEvent({
|
|
2933
3501
|
agentType: 'openclaw',
|
|
2934
3502
|
agentCapabilities: OPENCLAW_CAPABILITIES,
|
|
2935
|
-
snapshot: snap,
|
|
3503
|
+
snapshot: gatewaySnapshot(snap),
|
|
2936
3504
|
}));
|
|
2937
3505
|
lastStateEvent = gwStateEvent;
|
|
2938
3506
|
core.wsServer.broadcast(gwStateEvent);
|
|
@@ -3049,6 +3617,16 @@ export async function startDaemon(opts) {
|
|
|
3049
3617
|
if (!rawSessionId || !command)
|
|
3050
3618
|
return;
|
|
3051
3619
|
const sessionId = resolveDeviceSessionId(rawSessionId);
|
|
3620
|
+
// The gateway session has no bridge port — the focus-relay path below
|
|
3621
|
+
// would connect to nothing and silently drop the command. Hand it to
|
|
3622
|
+
// the adapter, the same way the wake-word assistant does.
|
|
3623
|
+
if (sessionId === 'openclaw-gateway') {
|
|
3624
|
+
if (gatewayAdapter?.isAlive())
|
|
3625
|
+
gatewayAdapter.handleCommand(command);
|
|
3626
|
+
else
|
|
3627
|
+
debug('daemon', 'session_command for openclaw-gateway but gateway not alive');
|
|
3628
|
+
return;
|
|
3629
|
+
}
|
|
3052
3630
|
// Observed Claude sessions have no bridge/PTY — route to the hook-based
|
|
3053
3631
|
// steering primitives instead (soft STOP, turn-end directive queue,
|
|
3054
3632
|
// gate approval). This replaces the old silent drop.
|
|
@@ -3070,7 +3648,7 @@ export async function startDaemon(opts) {
|
|
|
3070
3648
|
return;
|
|
3071
3649
|
}
|
|
3072
3650
|
const sessions = listActiveSessions();
|
|
3073
|
-
const target = sessions.find(s => s.id === sessionId);
|
|
3651
|
+
const target = sessions.find(s => s.id === sessionId) ?? getRemoteSession(sessionId);
|
|
3074
3652
|
if (!target) {
|
|
3075
3653
|
debug('daemon', `session_command: session ${sessionId} not found`);
|
|
3076
3654
|
return;
|
|
@@ -3096,7 +3674,7 @@ export async function startDaemon(opts) {
|
|
|
3096
3674
|
handleObservedClaudeCommand(sessionId.slice('observed:claude:'.length), cmd);
|
|
3097
3675
|
return;
|
|
3098
3676
|
}
|
|
3099
|
-
const target = listActiveSessions().find(s => s.id === sessionId);
|
|
3677
|
+
const target = listActiveSessions().find(s => s.id === sessionId) ?? getRemoteSession(sessionId);
|
|
3100
3678
|
if (target) {
|
|
3101
3679
|
userFocusedSessionId = sessionId;
|
|
3102
3680
|
broadcastFocusedState();
|
|
@@ -3180,20 +3758,46 @@ export async function startDaemon(opts) {
|
|
|
3180
3758
|
* drops the WebSocket it dictated over, and the answer must still arrive.
|
|
3181
3759
|
* Serial first, mirroring the single-path dedup policy used elsewhere.
|
|
3182
3760
|
*/
|
|
3761
|
+
/**
|
|
3762
|
+
* Replies staged for `audio_http_pull` boards, keyed by device (board id) —
|
|
3763
|
+
* the board fetches via GET /esp32/voice-reply at its own pace. Swept by
|
|
3764
|
+
* TTL: a board that reboots before fetching must not hear a stale answer
|
|
3765
|
+
* minutes later.
|
|
3766
|
+
*/
|
|
3767
|
+
const stagedVoiceReplies = new Map();
|
|
3768
|
+
const STAGED_REPLY_TTL_MS = 2 * 60_000;
|
|
3769
|
+
const stagedReplySweep = setInterval(() => {
|
|
3770
|
+
const cutoff = Date.now() - STAGED_REPLY_TTL_MS;
|
|
3771
|
+
for (const [key, entry] of stagedVoiceReplies) {
|
|
3772
|
+
if (entry.ts < cutoff)
|
|
3773
|
+
stagedVoiceReplies.delete(key);
|
|
3774
|
+
}
|
|
3775
|
+
}, 30_000);
|
|
3776
|
+
stagedReplySweep.unref?.();
|
|
3183
3777
|
const voiceReply = new DeviceVoiceReplyRouter(() => Date.now(), (ms) => new Promise((r) => setTimeout(r, ms)), (deviceKey) => {
|
|
3778
|
+
// This resolver serves the spoken-reply router only, so unlike the
|
|
3779
|
+
// general serial-first dedup it prefers whichever live transport can
|
|
3780
|
+
// actually PLAY audio (advertises audio_out): a CH340 serial link
|
|
3781
|
+
// physically cannot carry PCM even when it is the primary state path.
|
|
3782
|
+
const candidates = [];
|
|
3184
3783
|
const serialPort = liveSerialPortForBoard(deviceKey);
|
|
3185
|
-
if (serialPort)
|
|
3186
|
-
|
|
3187
|
-
return serialSinkFor(serialPort);
|
|
3188
|
-
}
|
|
3784
|
+
if (serialPort)
|
|
3785
|
+
candidates.push(serialSinkFor(serialPort));
|
|
3189
3786
|
for (const [key, sock] of wifiEsp32Sockets) {
|
|
3190
3787
|
if (sock.readyState !== WebSocket.OPEN)
|
|
3191
3788
|
continue;
|
|
3192
3789
|
const board = wifiEsp32Devices.get(key)?.board;
|
|
3193
|
-
if (board === deviceKey || key === deviceKey)
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3790
|
+
if (board === deviceKey || key === deviceKey)
|
|
3791
|
+
candidates.push(stableSink(sock));
|
|
3792
|
+
}
|
|
3793
|
+
const pick = candidates.find((s) => {
|
|
3794
|
+
const caps = s.capabilities();
|
|
3795
|
+
return caps.includes('audio_out') || caps.includes('audio_http_pull');
|
|
3796
|
+
}) ?? candidates[0];
|
|
3797
|
+
if (pick) {
|
|
3798
|
+
log(`[agentdeck] voice: routing reply for ${deviceKey} via ${pick.describe()}`
|
|
3799
|
+
+ ` (caps=${pick.capabilities().join('|') || 'none'})`);
|
|
3800
|
+
return pick;
|
|
3197
3801
|
}
|
|
3198
3802
|
log(`[agentdeck] voice: no live transport for ${deviceKey}`
|
|
3199
3803
|
+ ` (serial boards: ${serialBoardsAttached().join(',') || 'none'})`);
|
|
@@ -3258,7 +3862,18 @@ export async function startDaemon(opts) {
|
|
|
3258
3862
|
const voiceCfg = loadDaemonSettings().voice;
|
|
3259
3863
|
if (voiceCfg?.speakReplies === false)
|
|
3260
3864
|
return; // opt-out, default on
|
|
3261
|
-
|
|
3865
|
+
// Speech gets the lead, not the transcript: an explicit summary line if the
|
|
3866
|
+
// answer has one, else its first sentence. The full answer is already on the
|
|
3867
|
+
// screen the user is sitting at, where it can be skimmed — read aloud it is
|
|
3868
|
+
// a minute of monologue nobody can skip. `voice.speakFullReply` opts back in.
|
|
3869
|
+
const full = voiceCfg?.speakFullReply === true;
|
|
3870
|
+
const spoken = spokenDigest(text, { full });
|
|
3871
|
+
if (!full) {
|
|
3872
|
+
const readable = speakableReply(text, Number.MAX_SAFE_INTEGER);
|
|
3873
|
+
if (spoken && readable.length > spoken.length) {
|
|
3874
|
+
log(`[agentdeck] voice: reply digested ${readable.length} -> ${spoken.length} chars`);
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3262
3877
|
if (!spoken) {
|
|
3263
3878
|
// A code-only answer has nothing worth reading aloud; say that instead of
|
|
3264
3879
|
// spelling out punctuation, so the user still knows the turn finished.
|
|
@@ -3276,12 +3891,59 @@ export async function startDaemon(opts) {
|
|
|
3276
3891
|
}
|
|
3277
3892
|
const locale = typeof voiceCfg?.locale === 'string' && voiceCfg.locale
|
|
3278
3893
|
? voiceCfg.locale : undefined;
|
|
3894
|
+
// Host-armed dictations (deck push-to-talk) play through the host's own
|
|
3895
|
+
// speakers — AVSpeechSynthesizer speaks directly, no WAV/PCM streaming
|
|
3896
|
+
// detour. Split them out before synthesis so a host-only reply skips the
|
|
3897
|
+
// file round trip entirely.
|
|
3898
|
+
const hostTargets = targets.filter((s) => s.capabilities().includes('audio_host'));
|
|
3899
|
+
const boardTargets = targets.filter((s) => !s.capabilities().includes('audio_host'));
|
|
3900
|
+
if (hostTargets.length > 0) {
|
|
3901
|
+
for (const sink of hostTargets)
|
|
3902
|
+
voiceReply.disarm(sink);
|
|
3903
|
+
void (async () => {
|
|
3904
|
+
try {
|
|
3905
|
+
await speakWithHelper(spoken, locale ? { locale } : {});
|
|
3906
|
+
log(`[agentdeck] voice: spoke reply on host speakers (${spoken.length} chars)`);
|
|
3907
|
+
}
|
|
3908
|
+
catch (err) {
|
|
3909
|
+
log(`[agentdeck] voice: host speak failed: ${String(err).slice(0, 140)}`);
|
|
3910
|
+
}
|
|
3911
|
+
})();
|
|
3912
|
+
}
|
|
3913
|
+
if (boardTargets.length === 0)
|
|
3914
|
+
return;
|
|
3279
3915
|
void (async () => {
|
|
3280
3916
|
const out = join(tmpdir(), `agentdeck-reply-${process.pid}-${Date.now()}.wav`);
|
|
3281
3917
|
try {
|
|
3282
3918
|
await synthesizeWavWithHelper(spoken, out, locale ? { locale } : {});
|
|
3283
3919
|
const wav = await readFile(out);
|
|
3284
|
-
for (const sink of
|
|
3920
|
+
for (const sink of boardTargets) {
|
|
3921
|
+
// Pull-capable boards fetch the audio themselves over HTTP — the WS
|
|
3922
|
+
// push stream both stuttered and could crash the hosted-link RX
|
|
3923
|
+
// path (pkt_rxbuff assert). Stage the PCM, send a tiny notify
|
|
3924
|
+
// frame, and let TCP flow control pace the transfer.
|
|
3925
|
+
if (sink.capabilities().includes('audio_http_pull')) {
|
|
3926
|
+
const parsed = pcmFromWav(wav);
|
|
3927
|
+
if (parsed) {
|
|
3928
|
+
stagedVoiceReplies.set(sink.deviceKey(), {
|
|
3929
|
+
pcm: parsed.pcm, sampleRate: parsed.sampleRate, ts: Date.now(),
|
|
3930
|
+
});
|
|
3931
|
+
try {
|
|
3932
|
+
sink.send(JSON.stringify({
|
|
3933
|
+
type: 'audio_reply_ready',
|
|
3934
|
+
bytes: parsed.pcm.length,
|
|
3935
|
+
sampleRate: parsed.sampleRate,
|
|
3936
|
+
durationMs: Math.round((parsed.pcm.length / 2) / parsed.sampleRate * 1000),
|
|
3937
|
+
text: spoken.slice(0, 120),
|
|
3938
|
+
}));
|
|
3939
|
+
log(`[agentdeck] voice: staged reply for ${sink.deviceKey()}`
|
|
3940
|
+
+ ` (${parsed.pcm.length}B pull, ${spoken.length} chars)`);
|
|
3941
|
+
}
|
|
3942
|
+
catch { /* sink closing — TTL sweep reclaims the staging */ }
|
|
3943
|
+
voiceReply.disarm(sink);
|
|
3944
|
+
continue;
|
|
3945
|
+
}
|
|
3946
|
+
}
|
|
3285
3947
|
const ok = await voiceReply.stream(sink, wav, spoken);
|
|
3286
3948
|
log(`[agentdeck] voice: ${ok ? 'spoke' : 'FAILED to speak'} reply`
|
|
3287
3949
|
+ ` (${wav.length}B, ${spoken.length} chars)`);
|
|
@@ -3451,7 +4113,7 @@ export async function startDaemon(opts) {
|
|
|
3451
4113
|
* no socket to answer on, so resolve the board's live transport the same
|
|
3452
4114
|
* way a queued voice reply does (serial first, then its WiFi socket).
|
|
3453
4115
|
*/
|
|
3454
|
-
const
|
|
4116
|
+
const boardResultSinkFor = (board) => {
|
|
3455
4117
|
const port = liveSerialPortForBoard(board);
|
|
3456
4118
|
if (port)
|
|
3457
4119
|
return serialSinkFor(port);
|
|
@@ -3482,8 +4144,19 @@ export async function startDaemon(opts) {
|
|
|
3482
4144
|
}
|
|
3483
4145
|
else if (msg.type === 'voice_end') {
|
|
3484
4146
|
const captured = deviceVoice.end(port, msg);
|
|
3485
|
-
if (captured)
|
|
4147
|
+
if (captured) {
|
|
3486
4148
|
void finishVoiceCapture(captured, serialSinkFor(port));
|
|
4149
|
+
}
|
|
4150
|
+
else if (msg.cancel !== true) {
|
|
4151
|
+
try {
|
|
4152
|
+
serialSinkFor(port).send(JSON.stringify({ type: 'voice_result', text: '', error: 'empty_capture' }));
|
|
4153
|
+
}
|
|
4154
|
+
catch { /* port closing */ }
|
|
4155
|
+
}
|
|
4156
|
+
}
|
|
4157
|
+
else if (msg.type === 'voice_abort') {
|
|
4158
|
+
log(`[agentdeck] voice: device upload aborted (${String(msg.reason ?? 'unknown')}, `
|
|
4159
|
+
+ `${String(msg.total ?? '?')} bytes) [serial ${port}]`);
|
|
3487
4160
|
}
|
|
3488
4161
|
else if (msg.type === 'photo_begin') {
|
|
3489
4162
|
// Photo over USB: the strip is a desk fixture and usually powered from
|
|
@@ -3519,8 +4192,38 @@ export async function startDaemon(opts) {
|
|
|
3519
4192
|
* like the same board on WiFi, which is only true if there is one
|
|
3520
4193
|
* implementation.
|
|
3521
4194
|
*/
|
|
4195
|
+
/**
|
|
4196
|
+
* The sink to ARM for the spoken reply, which is not always the sink the
|
|
4197
|
+
* result JSON goes to: an HTTP-uploaded utterance resolves its result sink
|
|
4198
|
+
* serial-first (guaranteed delivery), but the ips10 advertises `audio_out`
|
|
4199
|
+
* on its WiFi device_info only — its 115200-baud serial physically cannot
|
|
4200
|
+
* carry PCM, so arming the serial sink would make voiceReply.arm() decline
|
|
4201
|
+
* and the reply silently never play. Prefer the sink itself when it can
|
|
4202
|
+
* play audio; otherwise find the board's live WS socket that can.
|
|
4203
|
+
*/
|
|
4204
|
+
const canPlayReply = (s) => {
|
|
4205
|
+
const caps = s.capabilities();
|
|
4206
|
+
return caps.includes('audio_out') || caps.includes('audio_http_pull');
|
|
4207
|
+
};
|
|
4208
|
+
const audioArmSinkFor = (sink, board) => {
|
|
4209
|
+
if (canPlayReply(sink))
|
|
4210
|
+
return sink;
|
|
4211
|
+
for (const [key, sock] of wifiEsp32Sockets) {
|
|
4212
|
+
if (sock.readyState !== WebSocket.OPEN)
|
|
4213
|
+
continue;
|
|
4214
|
+
if (wifiEsp32Devices.get(key)?.board === board || key === board) {
|
|
4215
|
+
const s = stableSink(sock);
|
|
4216
|
+
if (canPlayReply(s))
|
|
4217
|
+
return s;
|
|
4218
|
+
}
|
|
4219
|
+
}
|
|
4220
|
+
return sink;
|
|
4221
|
+
};
|
|
3522
4222
|
const finishVoiceCapture = async (captured, sink) => {
|
|
3523
4223
|
try {
|
|
4224
|
+
if (captured.integrityError) {
|
|
4225
|
+
throw new Error(captured.integrityError);
|
|
4226
|
+
}
|
|
3524
4227
|
// Locale matters: the recognizer falls back to the system locale,
|
|
3525
4228
|
// which mangles speech in another language. settings.json
|
|
3526
4229
|
// `voice.locale` (BCP-47, e.g. "en-US") overrides it.
|
|
@@ -3537,7 +4240,20 @@ export async function startDaemon(opts) {
|
|
|
3537
4240
|
let via;
|
|
3538
4241
|
let deliverReason;
|
|
3539
4242
|
if (text && sessionId) {
|
|
3540
|
-
if (sessionId
|
|
4243
|
+
if (sessionId === 'openclaw-gateway') {
|
|
4244
|
+
// The gateway session has no bridge port: the generic
|
|
4245
|
+
// session_command path focuses a relay that can never connect
|
|
4246
|
+
// and silently drops the prompt while this function reported
|
|
4247
|
+
// delivered=true (observed 2026-07-31 — "transcript showed,
|
|
4248
|
+
// nothing answered"). Route straight to the adapter and report
|
|
4249
|
+
// what actually happened.
|
|
4250
|
+
delivered = !!(gatewayAdapter?.isAlive()
|
|
4251
|
+
&& gatewayAdapter.handleCommand({ type: 'send_prompt', text }));
|
|
4252
|
+
via = 'gateway';
|
|
4253
|
+
if (!delivered)
|
|
4254
|
+
deliverReason = 'gateway_unavailable';
|
|
4255
|
+
}
|
|
4256
|
+
else if (sessionId.startsWith('observed:')) {
|
|
3541
4257
|
// Any observed agent (claude, codex, …): type the dictated line
|
|
3542
4258
|
// into the terminal that owns the session. session_command only
|
|
3543
4259
|
// knew how to route observed *Claude*, so a Codex target was
|
|
@@ -3560,15 +4276,17 @@ export async function startDaemon(opts) {
|
|
|
3560
4276
|
}
|
|
3561
4277
|
// Only a delivered prompt earns a spoken reply: arming on a
|
|
3562
4278
|
// dropped one would read out whatever that session happened to
|
|
3563
|
-
// answer someone else.
|
|
3564
|
-
|
|
4279
|
+
// answer someone else. Armed on an audio-capable sink, which may
|
|
4280
|
+
// differ from the result sink — see audioArmSinkFor.
|
|
4281
|
+
const armSink = audioArmSinkFor(sink, captured.board);
|
|
4282
|
+
const armed = delivered ? voiceReply.arm(armSink, sessionId) : false;
|
|
3565
4283
|
// Logged at info level rather than debug: these are rare,
|
|
3566
4284
|
// user-initiated events, and the silence made "the board said sent
|
|
3567
4285
|
// but nothing happened" undiagnosable without a rebuild.
|
|
3568
4286
|
log(`[agentdeck] voice: "${text.slice(0, 40)}" -> ${sessionId.slice(0, 32)}`
|
|
3569
4287
|
+ ` delivered=${delivered}${via ? ` via ${via}` : ''}`
|
|
3570
4288
|
+ `${deliverReason ? ` (${deliverReason})` : ''}`
|
|
3571
|
-
+ ` spokenReply=${armed ? `armed(${
|
|
4289
|
+
+ ` spokenReply=${armed ? `armed(${armSink.describe()})` : 'no'}`
|
|
3572
4290
|
+ ` [${sink.describe()} key=${sink.deviceKey()}`
|
|
3573
4291
|
+ ` caps=${sink.capabilities().join('|') || 'none'}]`);
|
|
3574
4292
|
}
|
|
@@ -3582,10 +4300,24 @@ export async function startDaemon(opts) {
|
|
|
3582
4300
|
catch { /* client disconnecting */ }
|
|
3583
4301
|
}
|
|
3584
4302
|
catch (err) {
|
|
3585
|
-
const reason = String(err).slice(0,
|
|
3586
|
-
|
|
4303
|
+
const reason = String(err).slice(0, 300);
|
|
4304
|
+
// Voice turns are rare and user-initiated. Keep failures at info
|
|
4305
|
+
// level so a board's short "Voice error" banner can always be
|
|
4306
|
+
// diagnosed after the temporary WAV has been cleaned up.
|
|
4307
|
+
log(`[agentdeck] voice: transcription failed on ${sink.describe()}: ${reason}`);
|
|
4308
|
+
// The board renders `error` on a one-line banner at glance distance —
|
|
4309
|
+
// a short stable code it can translate beats 160 chars of
|
|
4310
|
+
// NSError soup. The full reason still ships as `detail` and lives in
|
|
4311
|
+
// the log above.
|
|
4312
|
+
const code = /No speech detected|kAFAssistantErrorDomain Code=1110/.test(reason)
|
|
4313
|
+
? 'no_speech'
|
|
4314
|
+
: /audio_transport_overflow/.test(reason) ? 'audio_lost'
|
|
4315
|
+
: /audio_transport_incomplete/.test(reason) ? 'audio_incomplete'
|
|
4316
|
+
: 'voice_failed';
|
|
3587
4317
|
try {
|
|
3588
|
-
sink.send(JSON.stringify({
|
|
4318
|
+
sink.send(JSON.stringify({
|
|
4319
|
+
type: 'voice_result', text: '', error: code, detail: reason.slice(0, 160),
|
|
4320
|
+
}));
|
|
3589
4321
|
}
|
|
3590
4322
|
catch { /* client disconnecting */ }
|
|
3591
4323
|
}
|
|
@@ -3593,6 +4325,91 @@ export async function startDaemon(opts) {
|
|
|
3593
4325
|
captured.cleanup();
|
|
3594
4326
|
}
|
|
3595
4327
|
};
|
|
4328
|
+
// ── Host push-to-talk (deck surfaces: Stream Deck, D200H) ────────────────
|
|
4329
|
+
// A deck key contributes only a button; the host contributes the mic, the
|
|
4330
|
+
// recognizer and the speakers. This restores the capability of the retired
|
|
4331
|
+
// Voice dial (pulled pre-Marketplace because capture meant borrowing
|
|
4332
|
+
// iTerm2's mic grant via AppleScript + Homebrew sox + a whisper model) on a
|
|
4333
|
+
// dependency-free stack: capture, STT and TTS all live in the bundled Swift
|
|
4334
|
+
// helper, and delivery/reply reuse the device-voice pipeline unchanged.
|
|
4335
|
+
const HOST_SPEAKER_KEY = 'host-speaker';
|
|
4336
|
+
const broadcastVoiceState = (state, extra = {}) => {
|
|
4337
|
+
core.broadcast({ type: 'voice_state', state, ...extra });
|
|
4338
|
+
};
|
|
4339
|
+
/**
|
|
4340
|
+
* ReplySink whose "board" is the host itself. `voice_result` frames become
|
|
4341
|
+
* broadcast `voice_state` events so every deck surface can render the
|
|
4342
|
+
* outcome; the spoken reply is played by the audio_host branch in
|
|
4343
|
+
* speakReplyTo rather than streamed as PCM frames.
|
|
4344
|
+
*/
|
|
4345
|
+
const hostSpeakerSink = {
|
|
4346
|
+
send: (data) => {
|
|
4347
|
+
try {
|
|
4348
|
+
const msg = JSON.parse(data);
|
|
4349
|
+
if (msg.type === 'voice_result') {
|
|
4350
|
+
broadcastVoiceState(msg.error ? 'error' : 'idle', {
|
|
4351
|
+
...(msg.text ? { text: msg.text } : {}),
|
|
4352
|
+
...(msg.error ? { error: msg.error } : {}),
|
|
4353
|
+
...(msg.sessionId ? { sessionId: msg.sessionId } : {}),
|
|
4354
|
+
});
|
|
4355
|
+
}
|
|
4356
|
+
}
|
|
4357
|
+
catch { /* nothing renderable */ }
|
|
4358
|
+
},
|
|
4359
|
+
sendBinary: () => { },
|
|
4360
|
+
isOpen: () => true,
|
|
4361
|
+
capabilities: () => ['audio_out', 'audio_host'],
|
|
4362
|
+
describe: () => 'host:speaker',
|
|
4363
|
+
deviceKey: () => HOST_SPEAKER_KEY,
|
|
4364
|
+
};
|
|
4365
|
+
/** One capture at a time — PTT models a single held key. */
|
|
4366
|
+
let hostPttBusy = false;
|
|
4367
|
+
function handleHostVoicePtt(cmd) {
|
|
4368
|
+
if (process.platform !== 'darwin') {
|
|
4369
|
+
broadcastVoiceState('error', { error: 'host voice capture requires macOS' });
|
|
4370
|
+
return;
|
|
4371
|
+
}
|
|
4372
|
+
if (cmd.action === 'stop' || cmd.action === 'cancel') {
|
|
4373
|
+
if (!hostPttBusy)
|
|
4374
|
+
return;
|
|
4375
|
+
// Resolves the in-flight recordWithHelper below; a release that races
|
|
4376
|
+
// the max-duration stop is a harmless no-op.
|
|
4377
|
+
void stopHelperRecording({ cancel: cmd.action === 'cancel' }).catch(() => { });
|
|
4378
|
+
return;
|
|
4379
|
+
}
|
|
4380
|
+
if (hostPttBusy)
|
|
4381
|
+
return;
|
|
4382
|
+
hostPttBusy = true;
|
|
4383
|
+
const sessionId = resolveDeviceSessionId(cmd.sessionId) || userFocusedSessionId || '';
|
|
4384
|
+
const sessionExtra = sessionId ? { sessionId } : {};
|
|
4385
|
+
const wavPath = join(tmpdir(), `agentdeck-ptt-${process.pid}-${Date.now()}.wav`);
|
|
4386
|
+
broadcastVoiceState('recording', sessionExtra);
|
|
4387
|
+
void (async () => {
|
|
4388
|
+
try {
|
|
4389
|
+
const rec = await recordWithHelper(wavPath);
|
|
4390
|
+
broadcastVoiceState('transcribing', sessionExtra);
|
|
4391
|
+
await finishVoiceCapture({
|
|
4392
|
+
wavPath: rec.wav,
|
|
4393
|
+
sessionId,
|
|
4394
|
+
board: HOST_SPEAKER_KEY,
|
|
4395
|
+
cleanup: () => { void rm(rec.wav, { force: true }).catch(() => { }); },
|
|
4396
|
+
}, hostSpeakerSink);
|
|
4397
|
+
}
|
|
4398
|
+
catch (err) {
|
|
4399
|
+
const reason = String(err);
|
|
4400
|
+
if (/record_cancelled/.test(reason)) {
|
|
4401
|
+
broadcastVoiceState('idle', sessionExtra);
|
|
4402
|
+
}
|
|
4403
|
+
else {
|
|
4404
|
+
log(`[agentdeck] voice: host PTT failed: ${reason.slice(0, 200)}`);
|
|
4405
|
+
broadcastVoiceState('error', { error: reason.slice(0, 160), ...sessionExtra });
|
|
4406
|
+
}
|
|
4407
|
+
}
|
|
4408
|
+
finally {
|
|
4409
|
+
hostPttBusy = false;
|
|
4410
|
+
}
|
|
4411
|
+
})();
|
|
4412
|
+
}
|
|
3596
4413
|
/**
|
|
3597
4414
|
* Finish one captured photo: build the prompt that points the target session
|
|
3598
4415
|
* at the saved file, deliver it through the same ladder as dictated text
|