@agentdeck/bridge 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/assets/fm-helper/agentdeck-fm-helper +0 -0
  2. package/dist/apme/collector.d.ts.map +1 -1
  3. package/dist/apme/collector.js +5 -2
  4. package/dist/apme/collector.js.map +1 -1
  5. package/dist/apme/runner.d.ts +11 -0
  6. package/dist/apme/runner.d.ts.map +1 -1
  7. package/dist/apme/runner.js +60 -4
  8. package/dist/apme/runner.js.map +1 -1
  9. package/dist/apme/settings.d.ts +10 -0
  10. package/dist/apme/settings.d.ts.map +1 -1
  11. package/dist/apme/settings.js +14 -0
  12. package/dist/apme/settings.js.map +1 -1
  13. package/dist/awaiting-overlay.d.ts +44 -10
  14. package/dist/awaiting-overlay.d.ts.map +1 -1
  15. package/dist/awaiting-overlay.js +96 -28
  16. package/dist/awaiting-overlay.js.map +1 -1
  17. package/dist/bridge-core.d.ts +10 -0
  18. package/dist/bridge-core.d.ts.map +1 -1
  19. package/dist/bridge-core.js +47 -12
  20. package/dist/bridge-core.js.map +1 -1
  21. package/dist/card-feed.d.ts +107 -0
  22. package/dist/card-feed.d.ts.map +1 -0
  23. package/dist/card-feed.js +277 -0
  24. package/dist/card-feed.js.map +1 -0
  25. package/dist/card-modules.d.ts +87 -0
  26. package/dist/card-modules.d.ts.map +1 -0
  27. package/dist/card-modules.js +179 -0
  28. package/dist/card-modules.js.map +1 -0
  29. package/dist/cli.d.ts.map +1 -1
  30. package/dist/cli.js +93 -0
  31. package/dist/cli.js.map +1 -1
  32. package/dist/daemon-server.d.ts.map +1 -1
  33. package/dist/daemon-server.js +1008 -47
  34. package/dist/daemon-server.js.map +1 -1
  35. package/dist/daemon-ws-client.d.ts +2 -1
  36. package/dist/daemon-ws-client.d.ts.map +1 -1
  37. package/dist/daemon-ws-client.js +2 -1
  38. package/dist/daemon-ws-client.js.map +1 -1
  39. package/dist/daemon.js +1 -1
  40. package/dist/device-photo.d.ts +86 -0
  41. package/dist/device-photo.d.ts.map +1 -0
  42. package/dist/device-photo.js +200 -0
  43. package/dist/device-photo.js.map +1 -0
  44. package/dist/device-voice-reply.d.ts +124 -0
  45. package/dist/device-voice-reply.d.ts.map +1 -0
  46. package/dist/device-voice-reply.js +236 -0
  47. package/dist/device-voice-reply.js.map +1 -0
  48. package/dist/device-voice.d.ts +56 -0
  49. package/dist/device-voice.d.ts.map +1 -0
  50. package/dist/device-voice.js +134 -0
  51. package/dist/device-voice.js.map +1 -0
  52. package/dist/esp32-serial.d.ts +40 -0
  53. package/dist/esp32-serial.d.ts.map +1 -1
  54. package/dist/esp32-serial.js +109 -13
  55. package/dist/esp32-serial.js.map +1 -1
  56. package/dist/foundation-models-helper.d.ts +30 -0
  57. package/dist/foundation-models-helper.d.ts.map +1 -1
  58. package/dist/foundation-models-helper.js +45 -0
  59. package/dist/foundation-models-helper.js.map +1 -1
  60. package/dist/index.js +2 -2
  61. package/dist/index.js.map +1 -1
  62. package/dist/mdns.d.ts +11 -0
  63. package/dist/mdns.d.ts.map +1 -1
  64. package/dist/mdns.js +13 -0
  65. package/dist/mdns.js.map +1 -1
  66. package/dist/observed-inject.d.ts +119 -0
  67. package/dist/observed-inject.d.ts.map +1 -0
  68. package/dist/observed-inject.js +405 -0
  69. package/dist/observed-inject.js.map +1 -0
  70. package/dist/passive-observer.d.ts +14 -0
  71. package/dist/passive-observer.d.ts.map +1 -1
  72. package/dist/passive-observer.js +35 -5
  73. package/dist/passive-observer.js.map +1 -1
  74. package/dist/peripheral-mapping.d.ts +65 -0
  75. package/dist/peripheral-mapping.d.ts.map +1 -0
  76. package/dist/peripheral-mapping.js +115 -0
  77. package/dist/peripheral-mapping.js.map +1 -0
  78. package/dist/session-aggregator.d.ts +4 -1
  79. package/dist/session-aggregator.d.ts.map +1 -1
  80. package/dist/session-aggregator.js +5 -5
  81. package/dist/session-aggregator.js.map +1 -1
  82. package/dist/session-id-resolve.d.ts +17 -0
  83. package/dist/session-id-resolve.d.ts.map +1 -0
  84. package/dist/session-id-resolve.js +35 -0
  85. package/dist/session-id-resolve.js.map +1 -0
  86. package/dist/session-transcript-timeline.d.ts +7 -0
  87. package/dist/session-transcript-timeline.d.ts.map +1 -1
  88. package/dist/session-transcript-timeline.js +16 -6
  89. package/dist/session-transcript-timeline.js.map +1 -1
  90. package/dist/timeline-store.js +2 -2
  91. package/dist/timeline-store.js.map +1 -1
  92. package/dist/usage-event.d.ts.map +1 -1
  93. package/dist/usage-event.js +14 -7
  94. package/dist/usage-event.js.map +1 -1
  95. package/dist/voice.d.ts +13 -10
  96. package/dist/voice.d.ts.map +1 -1
  97. package/dist/voice.js +26 -272
  98. package/dist/voice.js.map +1 -1
  99. package/dist/ws-server.d.ts +14 -0
  100. package/dist/ws-server.d.ts.map +1 -1
  101. package/dist/ws-server.js +66 -1
  102. package/dist/ws-server.js.map +1 -1
  103. package/fm-helper/AgentDeckFMHelper.swift +303 -0
  104. package/package.json +3 -3
  105. package/dist/d200h/hid-protocol.d.ts +0 -62
  106. package/dist/d200h/hid-protocol.d.ts.map +0 -1
  107. package/dist/d200h/hid-protocol.js +0 -124
  108. package/dist/d200h/hid-protocol.js.map +0 -1
  109. package/dist/d200h/image-renderer.d.ts +0 -24
  110. package/dist/d200h/image-renderer.d.ts.map +0 -1
  111. package/dist/d200h/image-renderer.js +0 -321
  112. package/dist/d200h/image-renderer.js.map +0 -1
  113. package/dist/hook-migration.d.ts +0 -2
  114. package/dist/hook-migration.d.ts.map +0 -1
  115. package/dist/hook-migration.js +0 -104
  116. package/dist/hook-migration.js.map +0 -1
  117. package/dist/modules/d200h-module.d.ts +0 -65
  118. package/dist/modules/d200h-module.d.ts.map +0 -1
  119. package/dist/modules/d200h-module.js +0 -495
  120. package/dist/modules/d200h-module.js.map +0 -1
  121. package/dist/modules/d200h-renderer.d.ts +0 -7
  122. package/dist/modules/d200h-renderer.d.ts.map +0 -1
  123. package/dist/modules/d200h-renderer.js +0 -321
  124. package/dist/modules/d200h-renderer.js.map +0 -1
  125. package/dist/modules/trmnl-module.d.ts +0 -38
  126. package/dist/modules/trmnl-module.d.ts.map +0 -1
  127. package/dist/modules/trmnl-module.js +0 -111
  128. package/dist/modules/trmnl-module.js.map +0 -1
  129. package/dist/trmnl/byos-server.d.ts +0 -39
  130. package/dist/trmnl/byos-server.d.ts.map +0 -1
  131. package/dist/trmnl/byos-server.js +0 -301
  132. package/dist/trmnl/byos-server.js.map +0 -1
  133. package/dist/trmnl/frame-cache.d.ts +0 -37
  134. package/dist/trmnl/frame-cache.d.ts.map +0 -1
  135. package/dist/trmnl/frame-cache.js +0 -157
  136. package/dist/trmnl/frame-cache.js.map +0 -1
  137. package/dist/trmnl/image-renderer.d.ts +0 -27
  138. package/dist/trmnl/image-renderer.d.ts.map +0 -1
  139. package/dist/trmnl/image-renderer.js +0 -235
  140. package/dist/trmnl/image-renderer.js.map +0 -1
  141. package/dist/trmnl/trmnl-settings.d.ts +0 -96
  142. package/dist/trmnl/trmnl-settings.d.ts.map +0 -1
  143. package/dist/trmnl/trmnl-settings.js +0 -177
  144. package/dist/trmnl/trmnl-settings.js.map +0 -1
  145. package/dist/trmnl/trmnl-telemetry.d.ts +0 -47
  146. package/dist/trmnl/trmnl-telemetry.d.ts.map +0 -1
  147. package/dist/trmnl/trmnl-telemetry.js +0 -56
  148. package/dist/trmnl/trmnl-telemetry.js.map +0 -1
  149. package/dist/whisper-server-manager.d.ts +0 -19
  150. package/dist/whisper-server-manager.d.ts.map +0 -1
  151. package/dist/whisper-server-manager.js +0 -171
  152. package/dist/whisper-server-manager.js.map +0 -1
@@ -5,10 +5,16 @@ export declare class WsServer {
5
5
  private wss;
6
6
  private commandCallback;
7
7
  private rawMessageCallback;
8
+ private binaryCallback;
9
+ private onPongCallback;
8
10
  private onConnectCallback;
9
11
  private onDisconnectCallback;
10
12
  private clientAlive;
11
13
  private esp32Clients;
14
+ private recentConnectsByIp;
15
+ private flappingClients;
16
+ private knownBoardIps;
17
+ private socketRemoteIp;
12
18
  private eventTransformer;
13
19
  private ulanziClients;
14
20
  private tuiClients;
@@ -19,10 +25,18 @@ export declare class WsServer {
19
25
  onBroadcast(hook: (event: BridgeEvent) => void): void;
20
26
  setEventTransformer(transformer: ((event: BridgeEvent, client: WebSocket) => BridgeEvent | null) | null): void;
21
27
  isEsp32Client(ws: WebSocket): boolean;
28
+ /** True when this socket's remote IP has been reconnecting fast enough to
29
+ * count as flapping (see the connection handler). Callers use it to skip
30
+ * the heavy parts of the initial burst so a marginal client can't resonate
31
+ * the daemon into saturation. */
32
+ isFlappingClient(ws: WebSocket): boolean;
22
33
  markEsp32Client(ws: WebSocket): void;
23
34
  private payloadFor;
24
35
  broadcast(event: BridgeEvent): void;
25
36
  onCommand(callback: (cmd: PluginCommand) => void): void;
37
+ /** Register a callback for binary frames (device audio). */
38
+ onPong(callback: (ws: WebSocket) => void): void;
39
+ onBinary(callback: (data: Buffer, sender: WebSocket) => void): void;
26
40
  /** Register a callback for raw messages before PluginCommand dispatch. Return true to consume. */
27
41
  onRawMessage(callback: (msg: Record<string, unknown>, sender: WebSocket) => boolean): void;
28
42
  /** Broadcast to all clients except the sender */
@@ -1 +1 @@
1
- {"version":3,"file":"ws-server.d.ts","sourceRoot":"","sources":["../src/ws-server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,SAAS,EAAE,MAAM,IAAI,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAmB,MAAM,MAAM,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAK7D,qBAAa,QAAQ;IACnB,OAAO,CAAC,GAAG,CAAkB;IAC7B,OAAO,CAAC,eAAe,CAA+C;IACtE,OAAO,CAAC,kBAAkB,CAA+E;IACzG,OAAO,CAAC,iBAAiB,CAA0C;IACnE,OAAO,CAAC,oBAAoB,CAA0C;IACtE,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,gBAAgB,CAAgF;IAGxG,OAAO,CAAC,aAAa,CAAwB;IAK7C,OAAO,CAAC,UAAU,CAAsD;IACxE,OAAO,CAAC,SAAS,CAAiC;gBAEtC,MAAM,EAAE,MAAM;IAiH1B,OAAO,CAAC,cAAc,CAA2C;IAEjE,sFAAsF;IACtF,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAIrD,mBAAmB,CAAC,WAAW,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,KAAK,WAAW,GAAG,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI;IAI9G,aAAa,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO;IAIrC,eAAe,CAAC,EAAE,EAAE,SAAS,GAAG,IAAI;IAIpC,OAAO,CAAC,UAAU;IAOlB,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAiBnC,SAAS,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI;IAIvD,kGAAkG;IAClG,YAAY,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,SAAS,KAAK,OAAO,GAAG,IAAI;IAI1F,iDAAiD;IACjD,eAAe,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,GAAG,IAAI;IAW5D,eAAe,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,KAAK,IAAI,GAAG,IAAI;IAIxD,kBAAkB,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,KAAK,IAAI,GAAG,IAAI;IAI3D,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,GAAG,IAAI;IAQ/C,cAAc,IAAI,MAAM;IAIxB,oBAAoB,IAAI,MAAM;IAI9B;;oDAEgD;IAChD,aAAa,IAAI,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAMpD,KAAK,IAAI,IAAI;CASd"}
1
+ {"version":3,"file":"ws-server.d.ts","sourceRoot":"","sources":["../src/ws-server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,SAAS,EAAE,MAAM,IAAI,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAmB,MAAM,MAAM,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAK7D,qBAAa,QAAQ;IACnB,OAAO,CAAC,GAAG,CAAkB;IAC7B,OAAO,CAAC,eAAe,CAA+C;IACtE,OAAO,CAAC,kBAAkB,CAA+E;IACzG,OAAO,CAAC,cAAc,CAA4D;IAClF,OAAO,CAAC,cAAc,CAA0C;IAChE,OAAO,CAAC,iBAAiB,CAA0C;IACnE,OAAO,CAAC,oBAAoB,CAA0C;IACtE,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,YAAY,CAAwB;IAE5C,OAAO,CAAC,kBAAkB,CAA+B;IACzD,OAAO,CAAC,eAAe,CAAwB;IAO/C,OAAO,CAAC,aAAa,CAAqB;IAC1C,OAAO,CAAC,cAAc,CAAgC;IACtD,OAAO,CAAC,gBAAgB,CAAgF;IAGxG,OAAO,CAAC,aAAa,CAAwB;IAK7C,OAAO,CAAC,UAAU,CAAsD;IACxE,OAAO,CAAC,SAAS,CAAiC;gBAEtC,MAAM,EAAE,MAAM;IAoJ1B,OAAO,CAAC,cAAc,CAA2C;IAEjE,sFAAsF;IACtF,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAIrD,mBAAmB,CAAC,WAAW,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,KAAK,WAAW,GAAG,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI;IAI9G,aAAa,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO;IAIrC;;;sCAGkC;IAClC,gBAAgB,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO;IAIxC,eAAe,CAAC,EAAE,EAAE,SAAS,GAAG,IAAI;IAMpC,OAAO,CAAC,UAAU;IAOlB,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAiBnC,SAAS,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI;IAIvD,4DAA4D;IAC5D,MAAM,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,KAAK,IAAI,GAAG,IAAI;IAI/C,QAAQ,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,GAAG,IAAI;IAInE,kGAAkG;IAClG,YAAY,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,SAAS,KAAK,OAAO,GAAG,IAAI;IAI1F,iDAAiD;IACjD,eAAe,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,GAAG,IAAI;IAW5D,eAAe,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,KAAK,IAAI,GAAG,IAAI;IAIxD,kBAAkB,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,KAAK,IAAI,GAAG,IAAI;IAI3D,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,GAAG,IAAI;IAQ/C,cAAc,IAAI,MAAM;IAIxB,oBAAoB,IAAI,MAAM;IAI9B;;oDAEgD;IAChD,aAAa,IAAI,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAMpD,KAAK,IAAI,IAAI;CASd"}
package/dist/ws-server.js CHANGED
@@ -6,10 +6,23 @@ export class WsServer {
6
6
  wss;
7
7
  commandCallback = null;
8
8
  rawMessageCallback = null;
9
+ binaryCallback = null;
10
+ onPongCallback = null;
9
11
  onConnectCallback = null;
10
12
  onDisconnectCallback = null;
11
13
  clientAlive = new Map();
12
14
  esp32Clients = new Set();
15
+ // Per-IP connect timestamps for the flap guard (window: 30s, threshold: >6).
16
+ recentConnectsByIp = new Map();
17
+ flappingClients = new Set();
18
+ // IPs that have ever sent device_info this daemon lifetime. A board that
19
+ // predates the `?clientType=esp32` tag (XTeink 6ccbe140) is only recognized
20
+ // AFTER its first device_info — one connect too late, because the initial
21
+ // burst already went out untagged. Remembering the IP makes every subsequent
22
+ // connect board-class from byte one, so the ≤4096B invariant holds and a
23
+ // heap-tight board can't be killed by its own welcome payload.
24
+ knownBoardIps = new Set();
25
+ socketRemoteIp = new Map();
13
26
  eventTransformer = null;
14
27
  // Clients that registered as the Ulanzi Studio plugin. Their WebSocket
15
28
  // presence is the daemon's D200H connectivity signal.
@@ -60,10 +73,31 @@ export class WsServer {
60
73
  debug('WS', `Remote client authenticated from ${remoteIp}`);
61
74
  }
62
75
  const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
76
+ this.socketRemoteIp.set(ws, remoteIp);
63
77
  if (url.searchParams.get('clientType') === 'esp32' || url.searchParams.get('esp32') === '1') {
64
78
  this.esp32Clients.add(ws);
65
79
  debug('WS', 'ESP32 WiFi client tagged from query');
66
80
  }
81
+ else if (this.knownBoardIps.has(remoteIp)) {
82
+ this.esp32Clients.add(ws);
83
+ debug('WS', `ESP32 WiFi client tagged from known board IP ${remoteIp}`);
84
+ }
85
+ // Flap guard: a client whose IP reconnected >6 times in the last 30s is
86
+ // marked flapping. It is still served (rejecting outright would blind a
87
+ // recovering board forever), but sendInitialState skips the expensive
88
+ // burst for it — a marginal-RF board must not be able to resonate the
89
+ // daemon into event-loop saturation (connect → heavy serialize → client
90
+ // dies on the burst → immediate reconnect → repeat).
91
+ {
92
+ const now = Date.now();
93
+ const recent = (this.recentConnectsByIp.get(remoteIp) ?? []).filter((t) => now - t < 30_000);
94
+ recent.push(now);
95
+ this.recentConnectsByIp.set(remoteIp, recent);
96
+ if (recent.length > 6) {
97
+ this.flappingClients.add(ws);
98
+ debug('WS', `Flapping client ${remoteIp}: ${recent.length} connects/30s — initial burst reduced`);
99
+ }
100
+ }
67
101
  debug('WS', 'Plugin connected');
68
102
  this.clientAlive.set(ws, true);
69
103
  // Send current state to newly connected client
@@ -72,8 +106,20 @@ export class WsServer {
72
106
  }
73
107
  ws.on('pong', () => {
74
108
  this.clientAlive.set(ws, true);
109
+ // A display-only board sends no application messages while idle, so a
110
+ // message-based lastSeen brands every healthy quiet board "stale". Its
111
+ // pong every WS_PING_INTERVAL_MS is real liveness — surface it.
112
+ this.onPongCallback?.(ws);
75
113
  });
76
- ws.on('message', (data) => {
114
+ ws.on('message', (data, isBinary) => {
115
+ // Binary frames are device audio (voice capture), not JSON — route
116
+ // them out before the parser, which would otherwise log a parse error
117
+ // per 1 KB PCM frame.
118
+ if (isBinary) {
119
+ const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
120
+ this.binaryCallback?.(buf, ws);
121
+ return;
122
+ }
77
123
  try {
78
124
  const msg = JSON.parse(data.toString());
79
125
  debug('WS', `recv cmd: ${msg.type}`);
@@ -109,6 +155,8 @@ export class WsServer {
109
155
  debug('WS', 'Plugin disconnected');
110
156
  this.clientAlive.delete(ws);
111
157
  this.esp32Clients.delete(ws);
158
+ this.flappingClients.delete(ws);
159
+ this.socketRemoteIp.delete(ws);
112
160
  this.tuiClients.delete(ws);
113
161
  if (this.ulanziClients.delete(ws) && this.ulanziClients.size === 0) {
114
162
  debug('WS', 'Ulanzi plugin gone — D200H disconnected');
@@ -133,8 +181,18 @@ export class WsServer {
133
181
  isEsp32Client(ws) {
134
182
  return this.esp32Clients.has(ws);
135
183
  }
184
+ /** True when this socket's remote IP has been reconnecting fast enough to
185
+ * count as flapping (see the connection handler). Callers use it to skip
186
+ * the heavy parts of the initial burst so a marginal client can't resonate
187
+ * the daemon into saturation. */
188
+ isFlappingClient(ws) {
189
+ return this.flappingClients.has(ws);
190
+ }
136
191
  markEsp32Client(ws) {
137
192
  this.esp32Clients.add(ws);
193
+ const ip = this.socketRemoteIp.get(ws);
194
+ if (ip)
195
+ this.knownBoardIps.add(ip);
138
196
  }
139
197
  payloadFor(event, client, cachedPayload) {
140
198
  if (!this.eventTransformer)
@@ -170,6 +228,13 @@ export class WsServer {
170
228
  onCommand(callback) {
171
229
  this.commandCallback = callback;
172
230
  }
231
+ /** Register a callback for binary frames (device audio). */
232
+ onPong(callback) {
233
+ this.onPongCallback = callback;
234
+ }
235
+ onBinary(callback) {
236
+ this.binaryCallback = callback;
237
+ }
173
238
  /** Register a callback for raw messages before PluginCommand dispatch. Return true to consume. */
174
239
  onRawMessage(callback) {
175
240
  this.rawMessageCallback = callback;
@@ -1 +1 @@
1
- {"version":3,"file":"ws-server.js","sourceRoot":"","sources":["../src/ws-server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAGhD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExD,MAAM,OAAO,QAAQ;IACX,GAAG,CAAkB;IACrB,eAAe,GAA0C,IAAI,CAAC;IAC9D,kBAAkB,GAA0E,IAAI,CAAC;IACjG,iBAAiB,GAAqC,IAAI,CAAC;IAC3D,oBAAoB,GAAqC,IAAI,CAAC;IAC9D,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC5C,YAAY,GAAG,IAAI,GAAG,EAAa,CAAC;IACpC,gBAAgB,GAA2E,IAAI,CAAC;IACxG,uEAAuE;IACvE,sDAAsD;IAC9C,aAAa,GAAG,IAAI,GAAG,EAAa,CAAC;IAC7C,6DAA6D;IAC7D,wEAAwE;IACxE,0EAA0E;IAC1E,oDAAoD;IAC5C,UAAU,GAAG,IAAI,GAAG,EAA2C,CAAC;IAChE,SAAS,CAAiC;IAElD,YAAY,MAAc;QACxB,IAAI,CAAC,GAAG,GAAG,IAAI,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAE3C,yEAAyE;QACzE,yEAAyE;QACzE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC3B,KAAK,CAAC,IAAI,EAAE,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,qDAAqD;QACrD,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,MAAM,IAAI,GAAgB,EAAE,CAAC;YAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBAClC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC;oBACvC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACd,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;gBAChC,EAAE,CAAC,IAAI,EAAE,CAAC;YACZ,CAAC;YACD,qEAAqE;YACrE,qEAAqE;YACrE,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;gBACtB,KAAK,CAAC,IAAI,EAAE,iCAAiC,CAAC,CAAC;gBAC/C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC5B,EAAE,CAAC,SAAS,EAAE,CAAC;YACjB,CAAC;QACH,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAExB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,GAAoB,EAAE,EAAE;YACrD,oCAAoC;YACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;YAChD,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,UAAU,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;gBACjF,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAClD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC1B,KAAK,CAAC,IAAI,EAAE,mCAAmC,QAAQ,kBAAkB,CAAC,CAAC;oBAC3E,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;oBAC/B,OAAO;gBACT,CAAC;gBACD,KAAK,CAAC,IAAI,EAAE,oCAAoC,QAAQ,EAAE,CAAC,CAAC;YAC9D,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,UAAU,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;YACjF,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,OAAO,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC;gBAC5F,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC1B,KAAK,CAAC,IAAI,EAAE,qCAAqC,CAAC,CAAC;YACrD,CAAC;YAED,KAAK,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAE/B,+CAA+C;YAC/C,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;YAED,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACjB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAA4B,CAAC;oBACnE,KAAK,CAAC,IAAI,EAAE,aAAa,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;oBACrC,2DAA2D;oBAC3D,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,IAAI,GAAG,CAAC,UAAU,KAAK,eAAe,EAAE,CAAC;wBACzE,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;wBACxC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC3B,IAAI,CAAC,GAAG,EAAE,CAAC;4BACT,KAAK,CAAC,IAAI,EAAE,4CAA4C,CAAC,CAAC;wBAC5D,CAAC;oBACH,CAAC;oBACD,iEAAiE;oBACjE,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,IAAI,GAAG,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;wBAC/D,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAEvD,CAAC;wBACT,MAAM,IAAI,GAAG,OAAO,GAAG,EAAE,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC;wBAC/E,MAAM,EAAE,GAAG,OAAO,GAAG,EAAE,EAAE,KAAK,QAAQ,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;wBACjE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;wBACtC,KAAK,CAAC,IAAI,EAAE,6BAA6B,EAAE,EAAE,CAAC,CAAC;oBACjD,CAAC;oBACD,4EAA4E;oBAC5E,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;wBAChE,OAAO,CAAC,UAAU;oBACpB,CAAC;oBACD,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;wBACzB,IAAI,CAAC,eAAe,CAAC,GAA+B,CAAC,CAAC;oBACxD,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,KAAK,CAAC,IAAI,EAAE,4BAA4B,GAAG,EAAE,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBAClB,KAAK,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;gBACnC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC5B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBACnE,KAAK,CAAC,IAAI,EAAE,yCAAyC,CAAC,CAAC;gBACzD,CAAC;gBACD,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;oBAC9B,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACrB,KAAK,CAAC,IAAI,EAAE,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,cAAc,GAAwC,EAAE,CAAC;IAEjE,sFAAsF;IACtF,WAAW,CAAC,IAAkC;QAC5C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,mBAAmB,CAAC,WAAmF;QACrG,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;IACtC,CAAC;IAED,aAAa,CAAC,EAAa;QACzB,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,eAAe,CAAC,EAAa;QAC3B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC;IAEO,UAAU,CAAC,KAAkB,EAAE,MAAiB,EAAE,aAAsB;QAC9E,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,aAAa,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1E,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAC9B,OAAO,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACxG,CAAC;IAED,SAAS,CAAC,KAAkB;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC;QAC1C,KAAK,CAAC,IAAI,EAAE,aAAa,KAAK,CAAC,IAAI,QAAQ,WAAW,UAAU,CAAC,CAAC;QAClE,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACzC,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC9D,IAAI,CAAC,aAAa;oBAAE,SAAS;gBAC7B,IAAI,CAAC;oBAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QACD,iDAAiD;QACjD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,IAAI,CAAC;gBAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,SAAS,CAAC,QAAsC;QAC9C,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;IAClC,CAAC;IAED,kGAAkG;IAClG,YAAY,CAAC,QAAsE;QACjF,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAED,iDAAiD;IACjD,eAAe,CAAC,KAAkB,EAAE,MAAiB;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC9D,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC9D,IAAI,CAAC,aAAa;oBAAE,SAAS;gBAC7B,IAAI,CAAC;oBAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;IAED,eAAe,CAAC,QAAiC;QAC/C,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC;IACpC,CAAC;IAED,kBAAkB,CAAC,QAAiC;QAClD,IAAI,CAAC,oBAAoB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,EAAa,EAAE,KAAkB;QACtC,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,IAAI,CAAC;gBAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,oBAAoB;QAClB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;IACjC,CAAC;IAED;;oDAEgD;IAChD,aAAa;QACX,MAAM,IAAI,GAAG,IAAI,GAAG,EAAwC,CAAC;QAC7D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK;QACH,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9B,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,4DAA4D;QAC5D,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;CACF"}
1
+ {"version":3,"file":"ws-server.js","sourceRoot":"","sources":["../src/ws-server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAGhD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExD,MAAM,OAAO,QAAQ;IACX,GAAG,CAAkB;IACrB,eAAe,GAA0C,IAAI,CAAC;IAC9D,kBAAkB,GAA0E,IAAI,CAAC;IACjG,cAAc,GAAuD,IAAI,CAAC;IAC1E,cAAc,GAAqC,IAAI,CAAC;IACxD,iBAAiB,GAAqC,IAAI,CAAC;IAC3D,oBAAoB,GAAqC,IAAI,CAAC;IAC9D,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC5C,YAAY,GAAG,IAAI,GAAG,EAAa,CAAC;IAC5C,6EAA6E;IACrE,kBAAkB,GAAG,IAAI,GAAG,EAAoB,CAAC;IACjD,eAAe,GAAG,IAAI,GAAG,EAAa,CAAC;IAC/C,yEAAyE;IACzE,4EAA4E;IAC5E,0EAA0E;IAC1E,6EAA6E;IAC7E,yEAAyE;IACzE,+DAA+D;IACvD,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,cAAc,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC9C,gBAAgB,GAA2E,IAAI,CAAC;IACxG,uEAAuE;IACvE,sDAAsD;IAC9C,aAAa,GAAG,IAAI,GAAG,EAAa,CAAC;IAC7C,6DAA6D;IAC7D,wEAAwE;IACxE,0EAA0E;IAC1E,oDAAoD;IAC5C,UAAU,GAAG,IAAI,GAAG,EAA2C,CAAC;IAChE,SAAS,CAAiC;IAElD,YAAY,MAAc;QACxB,IAAI,CAAC,GAAG,GAAG,IAAI,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAE3C,yEAAyE;QACzE,yEAAyE;QACzE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC3B,KAAK,CAAC,IAAI,EAAE,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,qDAAqD;QACrD,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,MAAM,IAAI,GAAgB,EAAE,CAAC;YAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBAClC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC;oBACvC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACd,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;gBAChC,EAAE,CAAC,IAAI,EAAE,CAAC;YACZ,CAAC;YACD,qEAAqE;YACrE,qEAAqE;YACrE,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;gBACtB,KAAK,CAAC,IAAI,EAAE,iCAAiC,CAAC,CAAC;gBAC/C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC5B,EAAE,CAAC,SAAS,EAAE,CAAC;YACjB,CAAC;QACH,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAExB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,GAAoB,EAAE,EAAE;YACrD,oCAAoC;YACpC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;YAChD,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,UAAU,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;gBACjF,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAClD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC1B,KAAK,CAAC,IAAI,EAAE,mCAAmC,QAAQ,kBAAkB,CAAC,CAAC;oBAC3E,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;oBAC/B,OAAO;gBACT,CAAC;gBACD,KAAK,CAAC,IAAI,EAAE,oCAAoC,QAAQ,EAAE,CAAC,CAAC;YAC9D,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,UAAU,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;YACjF,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YACtC,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,OAAO,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC;gBAC5F,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC1B,KAAK,CAAC,IAAI,EAAE,qCAAqC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC1B,KAAK,CAAC,IAAI,EAAE,gDAAgD,QAAQ,EAAE,CAAC,CAAC;YAC1E,CAAC;YAED,wEAAwE;YACxE,wEAAwE;YACxE,sEAAsE;YACtE,sEAAsE;YACtE,wEAAwE;YACxE,qDAAqD;YACrD,CAAC;gBACC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;gBAC7F,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAC9C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC7B,KAAK,CAAC,IAAI,EAAE,mBAAmB,QAAQ,KAAK,MAAM,CAAC,MAAM,uCAAuC,CAAC,CAAC;gBACpG,CAAC;YACH,CAAC;YAED,KAAK,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAE/B,+CAA+C;YAC/C,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;YAED,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACjB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC/B,sEAAsE;gBACtE,uEAAuE;gBACvE,gEAAgE;gBAChE,IAAI,CAAC,cAAc,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE;gBAClC,mEAAmE;gBACnE,sEAAsE;gBACtE,sBAAsB;gBACtB,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAmB,CAAC,CAAC;oBAC5E,IAAI,CAAC,cAAc,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC/B,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAA4B,CAAC;oBACnE,KAAK,CAAC,IAAI,EAAE,aAAa,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;oBACrC,2DAA2D;oBAC3D,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,IAAI,GAAG,CAAC,UAAU,KAAK,eAAe,EAAE,CAAC;wBACzE,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;wBACxC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC3B,IAAI,CAAC,GAAG,EAAE,CAAC;4BACT,KAAK,CAAC,IAAI,EAAE,4CAA4C,CAAC,CAAC;wBAC5D,CAAC;oBACH,CAAC;oBACD,iEAAiE;oBACjE,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,IAAI,GAAG,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;wBAC/D,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAEvD,CAAC;wBACT,MAAM,IAAI,GAAG,OAAO,GAAG,EAAE,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC;wBAC/E,MAAM,EAAE,GAAG,OAAO,GAAG,EAAE,EAAE,KAAK,QAAQ,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;wBACjE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;wBACtC,KAAK,CAAC,IAAI,EAAE,6BAA6B,EAAE,EAAE,CAAC,CAAC;oBACjD,CAAC;oBACD,4EAA4E;oBAC5E,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;wBAChE,OAAO,CAAC,UAAU;oBACpB,CAAC;oBACD,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;wBACzB,IAAI,CAAC,eAAe,CAAC,GAA+B,CAAC,CAAC;oBACxD,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,KAAK,CAAC,IAAI,EAAE,4BAA4B,GAAG,EAAE,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBAClB,KAAK,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;gBACnC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC5B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC7B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAChC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC/B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBACnE,KAAK,CAAC,IAAI,EAAE,yCAAyC,CAAC,CAAC;gBACzD,CAAC;gBACD,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;oBAC9B,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACrB,KAAK,CAAC,IAAI,EAAE,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,cAAc,GAAwC,EAAE,CAAC;IAEjE,sFAAsF;IACtF,WAAW,CAAC,IAAkC;QAC5C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,mBAAmB,CAAC,WAAmF;QACrG,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;IACtC,CAAC;IAED,aAAa,CAAC,EAAa;QACzB,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED;;;sCAGkC;IAClC,gBAAgB,CAAC,EAAa;QAC5B,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,eAAe,CAAC,EAAa;QAC3B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvC,IAAI,EAAE;YAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrC,CAAC;IAEO,UAAU,CAAC,KAAkB,EAAE,MAAiB,EAAE,aAAsB;QAC9E,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,aAAa,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1E,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAC9B,OAAO,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACxG,CAAC;IAED,SAAS,CAAC,KAAkB;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC;QAC1C,KAAK,CAAC,IAAI,EAAE,aAAa,KAAK,CAAC,IAAI,QAAQ,WAAW,UAAU,CAAC,CAAC;QAClE,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACzC,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC9D,IAAI,CAAC,aAAa;oBAAE,SAAS;gBAC7B,IAAI,CAAC;oBAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QACD,iDAAiD;QACjD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,IAAI,CAAC;gBAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,SAAS,CAAC,QAAsC;QAC9C,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;IAClC,CAAC;IAED,4DAA4D;IAC5D,MAAM,CAAC,QAAiC;QACtC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC;IACjC,CAAC;IAED,QAAQ,CAAC,QAAmD;QAC1D,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC;IACjC,CAAC;IAED,kGAAkG;IAClG,YAAY,CAAC,QAAsE;QACjF,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAED,iDAAiD;IACjD,eAAe,CAAC,KAAkB,EAAE,MAAiB;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC9D,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC9D,IAAI,CAAC,aAAa;oBAAE,SAAS;gBAC7B,IAAI,CAAC;oBAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;IAED,eAAe,CAAC,QAAiC;QAC/C,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC;IACpC,CAAC;IAED,kBAAkB,CAAC,QAAiC;QAClD,IAAI,CAAC,oBAAoB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,EAAa,EAAE,KAAkB;QACtC,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,IAAI,CAAC;gBAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC;IAC/B,CAAC;IAED,oBAAoB;QAClB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;IACjC,CAAC;IAED;;oDAEgD;IAChD,aAAa;QACX,MAAM,IAAI,GAAG,IAAI,GAAG,EAAwC,CAAC;QAC7D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK;QACH,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9B,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,4DAA4D;QAC5D,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;CACF"}
@@ -1,4 +1,6 @@
1
1
  import Foundation
2
+ import AVFoundation
3
+ import Speech
2
4
  #if canImport(FoundationModels)
3
5
  import FoundationModels
4
6
  #endif
@@ -9,6 +11,14 @@ struct HelperRequest: Decodable {
9
11
  let prompt: String?
10
12
  let instructions: String?
11
13
  let temperature: Double?
14
+ /// `transcribe`: absolute path to a WAV file the daemon already wrote.
15
+ let wav: String?
16
+ /// BCP-47 locale hint, e.g. "ko-KR". Falls back to the current locale then en-US.
17
+ let locale: String?
18
+ /// `speak`: text to synthesize, with optional voice/rate overrides.
19
+ let text: String?
20
+ let voice: String?
21
+ let rate: Double?
12
22
  }
13
23
 
14
24
  @main
@@ -35,6 +45,21 @@ struct AgentDeckFMHelper {
35
45
  return
36
46
  }
37
47
 
48
+ if request.type == "transcribe" {
49
+ await handleTranscribe(request)
50
+ return
51
+ }
52
+
53
+ if request.type == "speak" {
54
+ await handleSpeak(request)
55
+ return
56
+ }
57
+
58
+ if request.type == "synthesize" {
59
+ await handleSynthesize(request)
60
+ return
61
+ }
62
+
38
63
  guard let prompt = request.prompt, !prompt.isEmpty else {
39
64
  write(["id": request.id, "error": "bad_request", "reason": "missing prompt"])
40
65
  return
@@ -69,6 +94,192 @@ struct AgentDeckFMHelper {
69
94
  #endif
70
95
  }
71
96
 
97
+ // MARK: - Speech to text
98
+
99
+ /// On-device transcription of a WAV the daemon already captured (from the
100
+ /// host mic or streamed off a board). `requiresOnDeviceRecognition` keeps
101
+ /// audio local — the captured speech routinely contains project and code
102
+ /// names that must not reach Apple's servers. Mirrors the Swift daemon's
103
+ /// `VoiceSpeechTranscriber` so both daemons transcribe identically.
104
+ private static func handleTranscribe(_ request: HelperRequest) async {
105
+ guard let path = request.wav, !path.isEmpty else {
106
+ write(["id": request.id, "error": "bad_request", "reason": "missing wav path"])
107
+ return
108
+ }
109
+ guard FileManager.default.fileExists(atPath: path) else {
110
+ write(["id": request.id, "error": "bad_request", "reason": "wav not found: \(path)"])
111
+ return
112
+ }
113
+
114
+ let status = await ensureSpeechAuthorization()
115
+ guard status == .authorized else {
116
+ write([
117
+ "id": request.id,
118
+ "error": "unauthorized",
119
+ "reason": "speech recognition not authorized (\(status.rawValue)) — grant it in System Settings › Privacy & Security › Speech Recognition",
120
+ ])
121
+ return
122
+ }
123
+
124
+ var locales: [Locale] = []
125
+ if let tag = request.locale, !tag.isEmpty { locales.append(Locale(identifier: tag)) }
126
+ locales.append(Locale.current)
127
+ locales.append(Locale(identifier: "en-US"))
128
+
129
+ guard let recognizer = locales.lazy.compactMap({ SFSpeechRecognizer(locale: $0) }).first(where: { $0.isAvailable }) else {
130
+ write([
131
+ "id": request.id,
132
+ "error": "unavailable",
133
+ "reason": "no available on-device recognizer — the dictation model may still be downloading",
134
+ ])
135
+ return
136
+ }
137
+
138
+ let speechRequest = SFSpeechURLRecognitionRequest(url: URL(fileURLWithPath: path))
139
+ speechRequest.shouldReportPartialResults = false
140
+ speechRequest.requiresOnDeviceRecognition = true
141
+ speechRequest.taskHint = .dictation
142
+
143
+ let result: [String: Any] = await withCheckedContinuation { continuation in
144
+ let box = ResumeOnce(continuation)
145
+ _ = recognizer.recognitionTask(with: speechRequest) { result, error in
146
+ if let error {
147
+ box.resume(["id": request.id, "error": "speech_error", "reason": String(describing: error)])
148
+ return
149
+ }
150
+ guard let result, result.isFinal else { return }
151
+ let text = result.bestTranscription.formattedString
152
+ .trimmingCharacters(in: .whitespacesAndNewlines)
153
+ box.resume(["id": request.id, "text": text])
154
+ }
155
+ }
156
+ write(result)
157
+ }
158
+
159
+ private static func ensureSpeechAuthorization() async -> SFSpeechRecognizerAuthorizationStatus {
160
+ let current = SFSpeechRecognizer.authorizationStatus()
161
+ guard current == .notDetermined else { return current }
162
+ return await withCheckedContinuation { continuation in
163
+ let box = ResumeOnce(continuation)
164
+ SFSpeechRecognizer.requestAuthorization { box.resume($0) }
165
+ }
166
+ }
167
+
168
+ // MARK: - Text to speech
169
+
170
+ /// Speak a reply through the host's audio output. Kept in the same helper
171
+ /// so the voice round trip needs exactly one bundled binary.
172
+ private static func handleSpeak(_ request: HelperRequest) async {
173
+ guard let text = request.text, !text.isEmpty else {
174
+ write(["id": request.id, "error": "bad_request", "reason": "missing text"])
175
+ return
176
+ }
177
+ let utterance = AVSpeechUtterance(string: text)
178
+ if let voiceId = request.voice, let v = AVSpeechSynthesisVoice(identifier: voiceId) {
179
+ utterance.voice = v
180
+ } else if let tag = request.locale, let v = AVSpeechSynthesisVoice(language: tag) {
181
+ utterance.voice = v
182
+ }
183
+ if let rate = request.rate { utterance.rate = Float(rate) }
184
+
185
+ let synthesizer = AVSpeechSynthesizer()
186
+ let delegate = SpeakDelegate()
187
+ synthesizer.delegate = delegate
188
+ await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
189
+ delegate.onFinish = ResumeOnce(continuation)
190
+ synthesizer.speak(utterance)
191
+ }
192
+ write(["id": request.id, "spoken": true])
193
+ }
194
+
195
+ /// Synthesize to a WAV file instead of the host's speakers, so the daemon
196
+ /// can stream the audio somewhere else — a battery-powered board holding
197
+ /// the only speaker the user is near. Fixed at 16 kHz mono PCM16: that is
198
+ /// what the boards' I2S amplifiers are configured for and what keeps a
199
+ /// spoken reply inside a WiFi budget an ESP32 can actually drain.
200
+ private static func handleSynthesize(_ request: HelperRequest) async {
201
+ guard let text = request.text, !text.isEmpty else {
202
+ write(["id": request.id, "error": "bad_request", "reason": "missing text"])
203
+ return
204
+ }
205
+ guard let outPath = request.wav, !outPath.isEmpty else {
206
+ write(["id": request.id, "error": "bad_request", "reason": "missing wav output path"])
207
+ return
208
+ }
209
+ let sampleRate = 16000.0
210
+ guard let outFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
211
+ sampleRate: sampleRate,
212
+ channels: 1,
213
+ interleaved: true) else {
214
+ write(["id": request.id, "error": "synthesis_failed", "reason": "output format unavailable"])
215
+ return
216
+ }
217
+
218
+ let utterance = AVSpeechUtterance(string: text)
219
+ if let voiceId = request.voice, let v = AVSpeechSynthesisVoice(identifier: voiceId) {
220
+ utterance.voice = v
221
+ } else if let tag = request.locale, let v = AVSpeechSynthesisVoice(language: tag) {
222
+ utterance.voice = v
223
+ }
224
+ if let rate = request.rate { utterance.rate = Float(rate) }
225
+
226
+ // The write callback is not documented to run on any particular queue,
227
+ // and it appends to shared state, so the accumulator owns a lock.
228
+ let sink = PCMSink()
229
+ let synthesizer = AVSpeechSynthesizer()
230
+ await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
231
+ let done = ResumeOnce(continuation)
232
+ synthesizer.write(utterance) { buffer in
233
+ guard let input = buffer as? AVAudioPCMBuffer else { return }
234
+ // A zero-length buffer is the documented end-of-synthesis
235
+ // signal; without it this call never completes.
236
+ if input.frameLength == 0 {
237
+ done.resume(())
238
+ return
239
+ }
240
+ sink.append(input, to: outFormat)
241
+ }
242
+ }
243
+
244
+ if let failure = sink.failure {
245
+ write(["id": request.id, "error": "synthesis_failed", "reason": failure])
246
+ return
247
+ }
248
+ let pcm = sink.pcm
249
+ guard !pcm.isEmpty else {
250
+ write(["id": request.id, "error": "synthesis_failed", "reason": "no audio produced"])
251
+ return
252
+ }
253
+ do {
254
+ try wavData(pcm: pcm, sampleRate: Int(sampleRate)).write(to: URL(fileURLWithPath: outPath))
255
+ } catch {
256
+ write(["id": request.id, "error": "synthesis_failed", "reason": String(describing: error)])
257
+ return
258
+ }
259
+ write([
260
+ "id": request.id,
261
+ "wav": outPath,
262
+ "sampleRate": Int(sampleRate),
263
+ "samples": pcm.count / 2,
264
+ "durationMs": Int(Double(pcm.count / 2) / sampleRate * 1000.0),
265
+ ])
266
+ }
267
+
268
+ /// Canonical 44-byte RIFF header + samples. Kept here rather than shelling
269
+ /// out so the helper stays the single bundled binary.
270
+ private static func wavData(pcm: Data, sampleRate: Int) -> Data {
271
+ var out = Data()
272
+ func u32(_ v: Int) { withUnsafeBytes(of: UInt32(v).littleEndian) { out.append(contentsOf: $0) } }
273
+ func u16(_ v: Int) { withUnsafeBytes(of: UInt16(v).littleEndian) { out.append(contentsOf: $0) } }
274
+ out.append(contentsOf: Array("RIFF".utf8)); u32(36 + pcm.count)
275
+ out.append(contentsOf: Array("WAVE".utf8))
276
+ out.append(contentsOf: Array("fmt ".utf8)); u32(16)
277
+ u16(1); u16(1); u32(sampleRate); u32(sampleRate * 2); u16(2); u16(16)
278
+ out.append(contentsOf: Array("data".utf8)); u32(pcm.count)
279
+ out.append(pcm)
280
+ return out
281
+ }
282
+
72
283
  private static func healthResponse(id: Int) -> [String: Any] {
73
284
  #if canImport(FoundationModels)
74
285
  if #available(macOS 26.0, *) {
@@ -110,3 +321,95 @@ struct AgentDeckFMHelper {
110
321
  FileHandle.standardOutput.write(Data((text + "\n").utf8))
111
322
  }
112
323
  }
324
+
325
+ /// One-shot continuation guard: speech and synthesis callbacks can fire more
326
+ /// than once, and resuming a continuation twice traps.
327
+ private final class ResumeOnce<T>: @unchecked Sendable {
328
+ private var continuation: CheckedContinuation<T, Never>?
329
+ private let lock = NSLock()
330
+
331
+ init(_ continuation: CheckedContinuation<T, Never>) {
332
+ self.continuation = continuation
333
+ }
334
+
335
+ func resume(_ value: T) {
336
+ lock.lock()
337
+ let c = continuation
338
+ continuation = nil
339
+ lock.unlock()
340
+ c?.resume(returning: value)
341
+ }
342
+ }
343
+
344
+ /// Accumulates synthesized audio, resampling each delivered buffer to the
345
+ /// target format. The converter is built from the first buffer because the
346
+ /// synthesizer picks its own rate and sample type per voice.
347
+ private final class PCMSink: @unchecked Sendable {
348
+ private let lock = NSLock()
349
+ private var buffer = Data()
350
+ private var converter: AVAudioConverter?
351
+ private var error: String?
352
+
353
+ var pcm: Data {
354
+ lock.lock(); defer { lock.unlock() }
355
+ return buffer
356
+ }
357
+
358
+ var failure: String? {
359
+ lock.lock(); defer { lock.unlock() }
360
+ return error
361
+ }
362
+
363
+ func append(_ input: AVAudioPCMBuffer, to outFormat: AVAudioFormat) {
364
+ lock.lock(); defer { lock.unlock() }
365
+ if error != nil { return }
366
+ if converter == nil {
367
+ converter = AVAudioConverter(from: input.format, to: outFormat)
368
+ if converter == nil {
369
+ error = "no converter from \(input.format) to \(outFormat)"
370
+ return
371
+ }
372
+ }
373
+ guard let converter else { return }
374
+ // Round up, and leave slack: the converter may emit slightly more than
375
+ // the naive ratio when it flushes filter state.
376
+ let ratio = outFormat.sampleRate / input.format.sampleRate
377
+ let capacity = AVAudioFrameCount((Double(input.frameLength) * ratio).rounded(.up)) + 1024
378
+ guard let output = AVAudioPCMBuffer(pcmFormat: outFormat, frameCapacity: capacity) else {
379
+ error = "output buffer allocation failed"
380
+ return
381
+ }
382
+ var supplied = false
383
+ var convertError: NSError?
384
+ let status = converter.convert(to: output, error: &convertError) { _, outStatus in
385
+ if supplied {
386
+ outStatus.pointee = .noDataNow
387
+ return nil
388
+ }
389
+ supplied = true
390
+ outStatus.pointee = .haveData
391
+ return input
392
+ }
393
+ if status == .error {
394
+ error = convertError?.localizedDescription ?? "conversion failed"
395
+ return
396
+ }
397
+ guard let channel = output.int16ChannelData, output.frameLength > 0 else { return }
398
+ let count = Int(output.frameLength)
399
+ channel[0].withMemoryRebound(to: UInt8.self, capacity: count * 2) { raw in
400
+ buffer.append(raw, count: count * 2)
401
+ }
402
+ }
403
+ }
404
+
405
+ private final class SpeakDelegate: NSObject, AVSpeechSynthesizerDelegate {
406
+ var onFinish: ResumeOnce<Void>?
407
+
408
+ func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) {
409
+ onFinish?.resume(())
410
+ }
411
+
412
+ func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) {
413
+ onFinish?.resume(())
414
+ }
415
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agentdeck/bridge",
3
- "version": "1.0.2",
3
+ "version": "1.0.3",
4
4
  "type": "module",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -28,8 +28,8 @@
28
28
  "qrcode": "^1.5.4",
29
29
  "strip-ansi": "^7.2.0",
30
30
  "ws": "^8.20.0",
31
- "@agentdeck/hooks": "1.0.2",
32
- "@agentdeck/shared": "1.0.2"
31
+ "@agentdeck/shared": "1.0.3",
32
+ "@agentdeck/hooks": "1.0.3"
33
33
  },
34
34
  "optionalDependencies": {
35
35
  "@picovoice/porcupine-node": "^4.0.2",
@@ -1,62 +0,0 @@
1
- /**
2
- * D200H HID Protocol — D200-compatible 1024-byte packet protocol
3
- *
4
- * Packet structure:
5
- * [0-1] Header: 0x7C 0x7C
6
- * [2-3] Command: big-endian uint16
7
- * [4-7] Length: little-endian uint32
8
- * [8-1023] Payload (zero-padded)
9
- *
10
- * Image delivery: ZIP(manifest.json + icons/*.png) sent in chunked 1024-byte packets.
11
- * ZIP quirk: bytes at offset 1016, 2040, 3064... must NOT be 0x00 or 0x7C.
12
- */
13
- export declare const CMD: {
14
- readonly SET_BUTTONS: 1;
15
- readonly PARTIAL_UPDATE: 13;
16
- readonly SET_SMALL_WINDOW: 6;
17
- readonly SET_BRIGHTNESS: 10;
18
- readonly SET_LABEL_STYLE: 11;
19
- readonly IN_BUTTON: 257;
20
- readonly IN_DEVICE_INFO: 771;
21
- };
22
- export declare function buildPacket(command: number, payload: Buffer, totalLength?: number): Buffer;
23
- /**
24
- * Build all packets for a ZIP file transfer (SET_BUTTONS or PARTIAL_UPDATE).
25
- * First packet has the header; subsequent packets are raw 1024-byte chunks.
26
- */
27
- export declare function buildZipPackets(zipData: Buffer, command?: number): Buffer[];
28
- export declare function buildBrightnessPacket(brightness: number): Buffer;
29
- export declare function buildSmallWindowPacket(mode: number, cpu: number, mem: number, time: string, gpu: number): Buffer;
30
- export interface ButtonEvent {
31
- index: number;
32
- pressed: boolean;
33
- state: number;
34
- }
35
- export interface DeviceInfo {
36
- serialNumber: string;
37
- firmwareVersion: string;
38
- deviceType: string;
39
- hardwareVersion: string;
40
- }
41
- export type IncomingEvent = {
42
- type: 'button';
43
- data: ButtonEvent;
44
- } | {
45
- type: 'device_info';
46
- data: DeviceInfo;
47
- } | {
48
- type: 'unknown';
49
- command: number;
50
- raw: Buffer;
51
- };
52
- export declare function parseIncoming(data: Buffer): IncomingEvent | null;
53
- /**
54
- * Check if a ZIP has invalid bytes at 1024-byte chunk boundaries.
55
- * At offset 1016, 2040, 3064... the byte must not be 0x00 or 0x7C.
56
- */
57
- export declare function validateZipBoundaries(zipData: Buffer): boolean;
58
- /**
59
- * Generate a random dummy string to pad the ZIP until boundaries are valid.
60
- */
61
- export declare function generateDummyPadding(attempt: number): string;
62
- //# sourceMappingURL=hid-protocol.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"hid-protocol.d.ts","sourceRoot":"","sources":["../../src/d200h/hid-protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAaH,eAAO,MAAM,GAAG;;;;;;;;CAQN,CAAC;AAIX,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAY1F;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,MAAwB,GAAG,MAAM,EAAE,CAiB5F;AAED,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAGhE;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAGhH;AAID,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GACzC;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAoChE;AAID;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAO9D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D"}