infinicode 2.8.85 → 2.8.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,7 +18,7 @@
18
18
  * registration mechanics is reimplemented here.
19
19
  */
20
20
  import { spawn, execFile } from 'node:child_process';
21
- import { existsSync } from 'node:fs';
21
+ import { existsSync, readFileSync, writeFileSync } from 'node:fs';
22
22
  import { hostname, networkInterfaces } from 'node:os';
23
23
  import { join } from 'node:path';
24
24
  import chalk from 'chalk';
@@ -103,6 +103,48 @@ function validateComposePath(path) {
103
103
  throw new Error(`no docker-compose.yaml/.yml found in '${path}'. Pass --path pointing at a ROBOVOICE checkout ` +
104
104
  `(the directory containing docker-compose.yaml), not this repo.`);
105
105
  }
106
+ /** Persist the single-track greeting fix in a ROBOVOICE checkout. */
107
+ function ensureRobovoiceSingleAudioTrack(path) {
108
+ const candidates = [
109
+ join(path, 'voice_agent.py'),
110
+ join(path, 'app', 'voice_agent.py'),
111
+ join(path, 'src', 'voice_agent.py'),
112
+ ];
113
+ const file = candidates.find((candidate) => existsSync(candidate));
114
+ if (!file) {
115
+ return { patched: false, warning: 'voice_agent.py was not found; skipping RoboPark greeting compatibility patch' };
116
+ }
117
+ const source = readFileSync(file, 'utf8');
118
+ if (source.includes('Initial greeting completed through AgentSession audio track')) {
119
+ return { patched: false, file };
120
+ }
121
+ const startMarker = ' # Synthesize the greeting first';
122
+ const endMarker = ' except Exception as e:';
123
+ const start = source.indexOf(startMarker);
124
+ const end = start >= 0 ? source.indexOf(endMarker, start) : -1;
125
+ if (start < 0 || end < 0) {
126
+ return {
127
+ patched: false,
128
+ file,
129
+ warning: 'voice_agent.py does not contain the known duplicate greeting-track block; no automatic patch applied',
130
+ };
131
+ }
132
+ const newline = source.includes('\r\n') ? '\r\n' : '\n';
133
+ const replacement = [
134
+ ' # Use the existing conversation track. A dedicated greeting',
135
+ ' # track makes Linux robots compete for the same ALSA device.',
136
+ ' import inspect as _inspect',
137
+ ' speech = session.say(greeting, allow_interruptions=False)',
138
+ ' if _inspect.isawaitable(speech):',
139
+ ' await speech',
140
+ " elif hasattr(speech, 'wait_for_playout'):",
141
+ ' await speech.wait_for_playout()',
142
+ " logger.info('Initial greeting completed through AgentSession audio track')",
143
+ '',
144
+ ].join(newline);
145
+ writeFileSync(file, source.slice(0, start) + replacement + source.slice(end), 'utf8');
146
+ return { patched: true, file };
147
+ }
106
148
  export async function roboparkAgentUp(opts) {
107
149
  console.log(chalk.bold('\n robopark agent up'));
108
150
  console.log(chalk.dim(' ' + '─'.repeat(52)));
@@ -125,8 +167,18 @@ export async function roboparkAgentUp(opts) {
125
167
  console.log(chalk.dim(` using: ${compose.cmd} ${compose.baseArgs.join(' ')}`.trimEnd()));
126
168
  console.log(` path: ${chalk.cyan(opts.path)}`);
127
169
  console.log();
170
+ const greetingPatch = ensureRobovoiceSingleAudioTrack(opts.path);
171
+ if (greetingPatch.patched) {
172
+ console.log(chalk.green(` ✓ persisted single-track RoboPark greeting in ${greetingPatch.file}`));
173
+ }
174
+ else if (greetingPatch.warning) {
175
+ console.log(chalk.yellow(` ⚠ ${greetingPatch.warning}`));
176
+ }
128
177
  console.log(chalk.bold(' 1. starting ROBOVOICE docker stack (docker compose up -d)…'));
129
- const upCode = await runCompose(compose, ['up', '-d'], opts.path);
178
+ // Rebuild only when compatibility source changed, making the fix survive
179
+ // container recreation without slowing normal restarts.
180
+ const upArgs = greetingPatch.patched ? ['up', '-d', '--build'] : ['up', '-d'];
181
+ const upCode = await runCompose(compose, upArgs, opts.path);
130
182
  if (upCode !== 0) {
131
183
  console.log(chalk.red(`\n ✗ docker compose up -d exited with code ${upCode}`));
132
184
  process.exit(upCode);
@@ -9,7 +9,8 @@
9
9
  */
10
10
  import chalk from 'chalk';
11
11
  import { spawn, execSync } from 'node:child_process';
12
- import { existsSync } from 'node:fs';
12
+ import { cpSync, existsSync, mkdirSync, statSync } from 'node:fs';
13
+ import { homedir } from 'node:os';
13
14
  import { dirname, join } from 'node:path';
14
15
  import { fileURLToPath } from 'node:url';
15
16
  function pkgDir() {
@@ -68,6 +69,33 @@ function findPython() {
68
69
  // ENOENT/"not found" error from the OS rather than a silent fallback.
69
70
  return process.platform === 'win32' ? 'python' : 'python3';
70
71
  }
72
+ function resolveSchedulerDataDir(explicit) {
73
+ if (explicit)
74
+ return explicit;
75
+ if (process.env.SCHEDULER_DATA_DIR)
76
+ return process.env.SCHEDULER_DATA_DIR;
77
+ const stable = join(homedir(), '.robopark', 'scheduler');
78
+ const stableDb = join(stable, 'scheduler.db');
79
+ if (existsSync(stableDb))
80
+ return stable;
81
+ // Older releases stored data beside whichever npm installation launched
82
+ // the scheduler. Local, global and npx installs therefore created separate
83
+ // databases. Migrate the largest existing DB once into stable user storage.
84
+ const legacyCandidates = [join(pkgDir(), '..', 'data')];
85
+ if (process.platform === 'win32' && process.env.APPDATA) {
86
+ legacyCandidates.push(join(process.env.APPDATA, 'npm', 'node_modules', 'data'));
87
+ }
88
+ const legacy = legacyCandidates
89
+ .filter((candidate, index, all) => all.indexOf(candidate) === index)
90
+ .filter(candidate => existsSync(join(candidate, 'scheduler.db')))
91
+ .sort((a, b) => statSync(join(b, 'scheduler.db')).size - statSync(join(a, 'scheduler.db')).size)[0];
92
+ mkdirSync(stable, { recursive: true });
93
+ if (legacy) {
94
+ cpSync(legacy, stable, { recursive: true, force: false, errorOnExist: false });
95
+ console.log(chalk.yellow(` migrated scheduler data: ${legacy} -> ${stable}`));
96
+ }
97
+ return stable;
98
+ }
71
99
  /** Resolve the actual infinicode CLI entry point so `robopark setup --start`
72
100
  * works even when global bins are not on PATH.
73
101
  *
@@ -110,6 +138,7 @@ export async function roboparkServe(opts) {
110
138
  const python = findPython();
111
139
  const schedulerDir = dirname(schedulerPath);
112
140
  const requirements = join(schedulerDir, 'requirements.txt');
141
+ const dataDir = resolveSchedulerDataDir(opts.dataDir);
113
142
  // Ensure scheduler Python dependencies are installed before starting.
114
143
  if (existsSync(requirements)) {
115
144
  console.log(chalk.dim(' ensuring scheduler Python deps…'));
@@ -125,7 +154,7 @@ export async function roboparkServe(opts) {
125
154
  ...process.env,
126
155
  SCHEDULER_HOST: host,
127
156
  SCHEDULER_PORT: String(port),
128
- SCHEDULER_DATA_DIR: opts.dataDir ?? join(pkgDir(), '..', 'data'),
157
+ SCHEDULER_DATA_DIR: dataDir,
129
158
  };
130
159
  if (opts.gateway)
131
160
  env.INFINIBOT_GATEWAY_URL = opts.gateway;
@@ -138,6 +167,7 @@ export async function roboparkServe(opts) {
138
167
  console.log(` scheduler: ${chalk.cyan(schedulerPath)}`);
139
168
  console.log(` python: ${chalk.cyan(python)}`);
140
169
  console.log(` listen: ${chalk.cyan(`${host}:${port}`)}`);
170
+ console.log(` data: ${chalk.cyan(dataDir)}`);
141
171
  if (opts.gateway)
142
172
  console.log(` gateway: ${chalk.cyan(opts.gateway)}`);
143
173
  console.log();
@@ -0,0 +1,111 @@
1
+ # RoboPark Production Memory
2
+
3
+ Last verified: 2026-07-16
4
+
5
+ This records production facts future operators and coding agents must
6
+ preserve. Never add enrollment tokens, mesh tokens, API keys, or device tokens.
7
+
8
+ ## Verified Architecture
9
+
10
+ - `livekit-1` is the Windows hub.
11
+ - Mesh/dashboard: TCP `47913`.
12
+ - RoboPark scheduler: TCP `8080`, proxied under `/robopark`.
13
+ - LiveKit: TCP `7880`.
14
+ - Production voice worker container: `caal-agent`.
15
+ - BMW maps to device `dev_0422e545` and character `vixen` / `VIXEN (BMW)`.
16
+ - Robots publish camera and microphone tracks to LiveKit. Park camera preview
17
+ uses the RoboVision relay rather than LiveKit video.
18
+
19
+ ## BMW Launch Device Lock
20
+
21
+ - Camera: `/dev/video0` (`H65 USB CAMERA`).
22
+ - Microphone: scheduler ID `2`, `Usb Audio Device: USB Audio (hw:3,0)`.
23
+ - Speaker: scheduler ID `1`, `USB Audio Device: - (hw:2,0)`.
24
+ - Linux live playback must use ALSA `plughw:2,0` through `aplay`.
25
+ - Production mic gain defaults to `ROBOPARK_MIC_GAIN=4.0` and is bounded to
26
+ `1.0..12.0` before PCM frames enter LiveKit.
27
+ - RoboVision/sounddevice numeric IDs are not PyAudio numeric IDs.
28
+ - Do not use HDMI or metadata-only `/dev/video*` devices as launch defaults.
29
+
30
+ ## Verified Production Flow
31
+
32
+ BMW passed these measured stages in one production session:
33
+
34
+ 1. `scheduler_session`
35
+ 2. `livekit_join`
36
+ 3. `tts_subscribed`
37
+ 4. `playback_started`
38
+ 5. `camera_published`
39
+ 6. `microphone_published`
40
+ 7. `robot_online`
41
+ 8. `camera_ready`
42
+ 9. `microphone_ready`
43
+ 10. `speaker_ready`
44
+
45
+ `playback_started` means the first remote TTS frame reached the robot. Never
46
+ report end-to-end audio delivery without this event.
47
+
48
+ ## Root Causes Fixed
49
+
50
+ ### Silent Greeting
51
+
52
+ The old ROBOVOICE greeting published a second dedicated LiveKit audio track.
53
+ The Pi subscribed to both tracks and attempted to open the same speaker twice.
54
+ ROBOVOICE must call `session.say(...)` so greeting and conversation share one
55
+ track. `robopark agent-up` now detects the old source block, patches the
56
+ ROBOVOICE checkout, and rebuilds Compose once.
57
+
58
+ ### Linux Speaker Playback
59
+
60
+ The preview agent previously used PyAudio for the selected speaker. BMW's
61
+ proven output is ALSA `aplay -D plughw:2,0`. Infinicode `2.8.86` uses that
62
+ same endpoint for live TTS whenever a Linux output selection contains `hw:`.
63
+
64
+ ### Silent Conversation After Greeting
65
+
66
+ Publishing a LiveKit microphone track does not prove that PCM capture started.
67
+ On Linux the production capture path is `arecord -D plughw:3,0 -f S16_LE -r
68
+ 48000 -c 1`; it must work even when PyAudio is unavailable. The Pi now waits
69
+ for actual PCM frames before reporting `microphone_published`, reports a blocked
70
+ stage when capture does not start within five seconds, and applies bounded mic
71
+ gain before publication.
72
+
73
+ ### Duplicate Speaker Tests
74
+
75
+ Only one robot process may claim a scheduler `speaker_test`. The scheduler
76
+ uses an atomic lease, and supervisor-status queries exclude speaker-test rows.
77
+
78
+ ### BMW Alias Configuration
79
+
80
+ Scheduler routes canonicalize alias `bmw` to `dev_0422e545` before reading
81
+ character, voice-stack, and greeting configuration.
82
+
83
+ ## Operational Rules
84
+
85
+ - Run one unified robot runtime per robot. Do not manually start preview,
86
+ vision, or mesh beside the systemd runtime.
87
+ - Do not run audio diagnostics during a live conversation.
88
+ - A direct ALSA test does not validate production if production uses another
89
+ API, index space, sample format, or opens the device twice.
90
+ - After a hub reboot, verify `47913`, `8080`, and `7880` before robot media.
91
+ - `tts_subscribed` alone is insufficient; require `playback_started`.
92
+ - Greeting `playback_started` alone does not prove conversation. A complete
93
+ turn also requires a user/STT event, an LLM response, and response playback.
94
+ - Container-local changes are temporary. Persist voice changes in the
95
+ ROBOVOICE checkout/image or use `robopark agent-up --path <checkout>`.
96
+
97
+ ## Minimal Verification
98
+
99
+ Run from the hub:
100
+
101
+ ```powershell
102
+ try { Invoke-RestMethod -Method Post http://127.0.0.1:8080/api/robots/bmw/pipeline-test/stop | Out-Null } catch {}
103
+ Start-Sleep 3
104
+ Invoke-RestMethod -Method Post http://127.0.0.1:8080/api/robots/bmw/pipeline-test/start | Out-Null
105
+ Start-Sleep 30
106
+ $s = Invoke-RestMethod http://127.0.0.1:8080/api/robots/bmw/pipeline-status
107
+ $id = $s.latest_session.id
108
+ $s.events | Where-Object session_id -eq $id | Select-Object timestamp,stage,status,message
109
+ ```
110
+
111
+ Required result: `playback_started ok`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "infinicode",
3
- "version": "2.8.85",
3
+ "version": "2.8.87",
4
4
  "description": "OpenKernel — provider-agnostic AI execution kernel. Native coding agent + mission-driven execution runtime.",
5
5
  "type": "module",
6
6
  "main": "./dist/kernel/index.js",
@@ -29,8 +29,9 @@
29
29
  "static",
30
30
  "packages/robopark/scheduler",
31
31
  "packages/robopark/pi-client",
32
- "packages/robopark/vision",
33
- ".opencode/plugins",
32
+ "packages/robopark/vision",
33
+ "docs/ROBOPARK_PRODUCTION_MEMORY.md",
34
+ ".opencode/plugins",
34
35
  ".opencode/themes",
35
36
  ".opencode/tui.json",
36
37
  "README.md"
@@ -1,4 +1,7 @@
1
- # robopark
1
+ # robopark
2
+
3
+ Production architecture, device locks, verified stages, and incident memory
4
+ are recorded in [`../../docs/ROBOPARK_PRODUCTION_MEMORY.md`](../../docs/ROBOPARK_PRODUCTION_MEMORY.md).
2
5
 
3
6
  The **RoboPark fleet control CLI** — set up, watch, and drive a room full of
4
7
  talking robots from one command. `robopark` is the operator front-end; it rides
@@ -604,6 +604,9 @@ async def init_db():
604
604
  ("sessions", "voice_stack_id", "TEXT"),
605
605
  ("sessions", "initiated_by", "TEXT"),
606
606
  ("sessions", "event_token_hash", "TEXT"),
607
+ # Speaker tests are executed only by preview_agent, which can
608
+ # release and restore the active media publisher around ALSA I/O.
609
+ ("shell_requests", "claimed_at", "TEXT"),
607
610
  ):
608
611
  try:
609
612
  await db.execute(f"ALTER TABLE {_table} ADD COLUMN {_column} {_coldef}")
@@ -3317,8 +3320,9 @@ async def device_supervisor_status(device_id: str, payload: SupervisorStatusRepo
3317
3320
  pending = []
3318
3321
  try:
3319
3322
  async with db.execute(
3320
- "SELECT service_name, action, kind, params, id FROM shell_requests "
3321
- "WHERE device_id = ? AND completed_at IS NULL ORDER BY requested_at",
3323
+ "SELECT service_name, action, kind, params, id FROM shell_requests "
3324
+ "WHERE device_id = ? AND completed_at IS NULL AND kind != 'speaker_test' "
3325
+ "ORDER BY requested_at",
3322
3326
  (device_id,),
3323
3327
  ) as c:
3324
3328
  shell_rows = [dict(r) for r in await c.fetchall()]
@@ -3696,13 +3700,26 @@ async def next_device_speaker_test(device_id: str, authorization: Optional[str]
3696
3700
  """Allow the always-running preview agent to execute audio tests."""
3697
3701
  if not await _authorize_device(device_id, authorization):
3698
3702
  raise HTTPException(401, "Invalid device token")
3703
+ now = datetime.utcnow()
3704
+ stale_before = (now - timedelta(seconds=45)).isoformat()
3699
3705
  async with aiosqlite.connect(DB_PATH) as db:
3700
3706
  db.row_factory = aiosqlite.Row
3707
+ await db.execute("BEGIN IMMEDIATE")
3701
3708
  async with db.execute(
3702
3709
  "SELECT id, params FROM shell_requests WHERE device_id = ? AND kind = 'speaker_test' "
3703
- "AND completed_at IS NULL ORDER BY requested_at LIMIT 1", (device_id,),
3710
+ "AND completed_at IS NULL AND (claimed_at IS NULL OR claimed_at < ?) "
3711
+ "ORDER BY requested_at LIMIT 1", (device_id, stale_before),
3704
3712
  ) as cursor:
3705
3713
  row = await cursor.fetchone()
3714
+ if row:
3715
+ claimed = await db.execute(
3716
+ "UPDATE shell_requests SET claimed_at = ? WHERE id = ? AND completed_at IS NULL "
3717
+ "AND (claimed_at IS NULL OR claimed_at < ?)",
3718
+ (now.isoformat(), row["id"], stale_before),
3719
+ )
3720
+ if claimed.rowcount != 1:
3721
+ row = None
3722
+ await db.commit()
3706
3723
  return {"request": {"id": row["id"], "params": json.loads(row["params"] or "{}")} if row else None}
3707
3724
 
3708
3725
 
@@ -4171,25 +4188,35 @@ async def _auto_elevenlabs_voice(robot_name: Optional[str], current: Optional[st
4171
4188
 
4172
4189
 
4173
4190
  async def _resolve_robot_voice_config(robot_id: str) -> RobotVoiceConfig:
4174
- """Public helper to resolve the effective voice config for a robot id."""
4175
- async with aiosqlite.connect(DB_PATH) as db:
4176
- db.row_factory = aiosqlite.Row
4177
- stack_dict = await _resolve_voice_stack_for_robot(db, robot_id)
4191
+ """Public helper to resolve the effective voice config for a robot id."""
4192
+ async with aiosqlite.connect(DB_PATH) as db:
4193
+ db.row_factory = aiosqlite.Row
4194
+ # Dashboard routes commonly use the human robot name ("bmw"), while
4195
+ # assignments are persisted against the enrolled device id. Resolve
4196
+ # that alias once so character, stack and greetings use one identity.
4197
+ async with db.execute(
4198
+ "SELECT id FROM devices WHERE id = ? OR lower(name) = lower(?) "
4199
+ "ORDER BY CASE WHEN id = ? THEN 0 ELSE 1 END LIMIT 1",
4200
+ (robot_id, robot_id, robot_id),
4201
+ ) as c:
4202
+ canonical_device = await c.fetchone()
4203
+ canonical_id = canonical_device["id"] if canonical_device else robot_id
4204
+ stack_dict = await _resolve_voice_stack_for_robot(db, canonical_id)
4178
4205
 
4179
4206
  # Determine character preset + name
4180
4207
  character_id = None
4181
4208
  character_name = None
4182
4209
  system_prompt = None
4183
4210
  async with db.execute(
4184
- "SELECT character_preset_id FROM robot_voice_configs WHERE robot_id = ?", (robot_id,)
4211
+ "SELECT character_preset_id FROM robot_voice_configs WHERE robot_id = ?", (canonical_id,)
4185
4212
  ) as c:
4186
4213
  rvc = await c.fetchone()
4187
4214
  if rvc and rvc["character_preset_id"]:
4188
4215
  character_id = rvc["character_preset_id"]
4189
4216
  else:
4190
4217
  async with db.execute(
4191
- "SELECT character_id FROM robots WHERE id = ? UNION ALL SELECT character_id FROM devices WHERE id = ?",
4192
- (robot_id, robot_id),
4218
+ "SELECT character_id FROM robots WHERE id = ? UNION ALL SELECT character_id FROM devices WHERE id = ?",
4219
+ (canonical_id, canonical_id),
4193
4220
  ) as c:
4194
4221
  row = await c.fetchone()
4195
4222
  if row:
@@ -4204,21 +4231,23 @@ async def _resolve_robot_voice_config(robot_id: str) -> RobotVoiceConfig:
4204
4231
  system_prompt = row["system_prompt"]
4205
4232
  if not character_name:
4206
4233
  async with db.execute(
4207
- "SELECT name FROM robots WHERE id = ? UNION ALL SELECT name FROM devices WHERE id = ?",
4208
- (robot_id, robot_id),
4234
+ "SELECT name FROM robots WHERE id = ? UNION ALL SELECT name FROM devices WHERE id = ?",
4235
+ (canonical_id, canonical_id),
4209
4236
  ) as c:
4210
4237
  row = await c.fetchone()
4211
4238
  if row:
4212
4239
  character_name = row["name"]
4213
4240
 
4214
- async with db.execute("SELECT greeting_phrases FROM devices WHERE id = ?", (robot_id,)) as c:
4241
+ async with db.execute("SELECT greeting_phrases FROM devices WHERE id = ?", (canonical_id,)) as c:
4215
4242
  device_row = await c.fetchone()
4216
4243
  greeting_phrases = []
4217
4244
  if device_row and device_row["greeting_phrases"]:
4218
4245
  try:
4219
4246
  greeting_phrases = [str(p) for p in _json.loads(device_row["greeting_phrases"]) if str(p).strip()]
4220
- except Exception:
4221
- greeting_phrases = []
4247
+ except Exception:
4248
+ greeting_phrases = []
4249
+ if not greeting_phrases and (character_name or character_id):
4250
+ greeting_phrases = [f"Hello, I am {character_name or character_id}. How can I help you?"]
4222
4251
  if stack_dict:
4223
4252
  stack_dict["tts_voice"] = await _auto_elevenlabs_voice(character_name or character_id, stack_dict.get("tts_voice"))
4224
4253
  voice_stack = VoiceStack(**stack_dict) if stack_dict else None
@@ -1134,6 +1134,7 @@ class LiveKitPublisher:
1134
1134
  self._tasks: list[asyncio.Task] = []
1135
1135
  self._capture: Optional["VideoCapture"] = None
1136
1136
  self._mic_capture: Optional["AudioCapture"] = None
1137
+ self._mic_streaming = asyncio.Event()
1137
1138
  # Motion sampling state belongs to the publisher instance. Keeping it
1138
1139
  # initialized here prevents shutdown/reopen paths from raising while
1139
1140
  # the camera is being handed between preview and voice sessions.
@@ -1212,13 +1213,12 @@ class LiveKitPublisher:
1212
1213
  self.video_source.capture_frame(first_frame)
1213
1214
  await self.agent._report_pipeline("camera_published", "ok", "Camera track published")
1214
1215
 
1215
- if audio_enabled:
1216
- self.audio_source = self.rtc.AudioSource(48000, 1)
1217
- self.audio_track = self.rtc.LocalAudioTrack.create_audio_track("microphone", self.audio_source)
1218
- aopts = self.rtc.TrackPublishOptions()
1219
- aopts.source = self.rtc.TrackSource.SOURCE_MICROPHONE
1216
+ if audio_enabled:
1217
+ self.audio_source = self.rtc.AudioSource(48000, 1)
1218
+ self.audio_track = self.rtc.LocalAudioTrack.create_audio_track("microphone", self.audio_source)
1219
+ aopts = self.rtc.TrackPublishOptions()
1220
+ aopts.source = self.rtc.TrackSource.SOURCE_MICROPHONE
1220
1221
  await self.room.local_participant.publish_track(self.audio_track, aopts)
1221
- await self.agent._report_pipeline("microphone_published", "ok", "Microphone track published")
1222
1222
 
1223
1223
  # Register speaker playback (subscribe to the voice agent's TTS audio
1224
1224
  # track) BEFORE opening the camera. Camera open is a slow/occasionally
@@ -1227,50 +1227,111 @@ class LiveKitPublisher:
1227
1227
  # entire event loop — including receiving the agent's greeting audio
1228
1228
  # — until it finished, so a short "Hello friend!" greeting could be
1229
1229
  # over and gone before we ever got a chance to subscribe to it.
1230
- if has_audio() and audio_enabled:
1231
- self._tasks.append(asyncio.create_task(self._audio_loop()))
1230
+ if audio_enabled:
1231
+ self._tasks.append(asyncio.create_task(self._audio_loop()))
1232
+ try:
1233
+ await asyncio.wait_for(self._mic_streaming.wait(), timeout=5.0)
1234
+ await self.agent._report_pipeline(
1235
+ "microphone_published", "ok", "Microphone track published with live PCM frames"
1236
+ )
1237
+ except asyncio.TimeoutError:
1238
+ await self.agent._report_pipeline(
1239
+ "microphone_published", "blocked", "Microphone track published but PCM capture did not start"
1240
+ )
1241
+ raise RuntimeError(f"microphone capture did not start for {self.agent.audio_capture_device}")
1232
1242
 
1233
1243
  if self._capture and self.video_source:
1234
1244
  self._tasks.append(asyncio.create_task(self._video_loop()))
1235
1245
 
1236
- async def _play_remote_audio(self, track, sid: str) -> None:
1237
- try:
1238
- import pyaudio
1239
- except Exception as e:
1240
- logger.warning(f"pyaudio unavailable for playback: {e}")
1241
- return
1242
- OUT_RATE = 48000
1243
- OUT_CHANNELS = 2
1244
- pa = pyaudio.PyAudio()
1245
- # Same MME-vs-WASAPI gotcha as mic capture (see PyAudioCapture._resolve_device):
1246
- # PyAudio's global default output device resolves through MME, which on this
1247
- # machine will accept writes and report success with zero exceptions while
1248
- # never actually reaching the real speakers. Prefer WASAPI's default output,
1249
- # which is what Windows' own volume mixer/meter is backed by.
1250
- output_device_index = None
1246
+ async def _play_remote_audio(self, track, sid: str) -> None:
1247
+ OUT_RATE = 48000
1248
+ OUT_CHANNELS = 2
1251
1249
  selected_output = str(self.agent.audio_playback_device or "default")
1252
- if selected_output.strip().isdigit():
1253
- output_device_index = int(selected_output.strip())
1254
- try:
1255
- if selected_output.lower() == "default":
1256
- wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
1257
- idx = wasapi.get("defaultOutputDevice")
1258
- if idx is not None and idx >= 0:
1259
- output_device_index = idx
1260
- elif output_device_index is None:
1261
- needle = selected_output.lower()
1262
- for i in range(pa.get_device_count()):
1263
- info = pa.get_device_info_by_index(i)
1264
- if info.get("maxOutputChannels", 0) > 0 and needle in str(info.get("name", "")).lower():
1265
- output_device_index = i
1266
- break
1267
- except Exception as e:
1268
- logger.debug(f"audio out: WASAPI default output lookup failed, using PyAudio default: {e}")
1269
- out = pa.open(
1270
- format=pyaudio.paInt16, channels=OUT_CHANNELS, rate=OUT_RATE, output=True,
1271
- output_device_index=output_device_index,
1272
- frames_per_buffer=960,
1273
- )
1250
+ pa = None
1251
+ output_device_index = None
1252
+
1253
+ # RoboVision inventories Linux devices through sounddevice/PortAudio,
1254
+ # but the numeric indices are not stable across PyAudio builds. More
1255
+ # importantly, BMW's production USB adapter is already proven through
1256
+ # ALSA's plughw conversion path. Use that exact endpoint for live TTS
1257
+ # instead of reopening the unrelated PyAudio index.
1258
+ if sys.platform.startswith("linux") and "hw:" in selected_output:
1259
+ import re
1260
+ import subprocess
1261
+
1262
+ match = re.search(r"\b(hw:\d+,\d+)\b", selected_output)
1263
+ if not match:
1264
+ logger.warning(f"audio out: no ALSA hardware address in {selected_output!r}")
1265
+ return
1266
+ alsa_device = f"plug{match.group(1)}"
1267
+ process = subprocess.Popen(
1268
+ [
1269
+ "aplay", "-q", "-D", alsa_device, "-t", "raw",
1270
+ "-f", "S16_LE", "-r", str(OUT_RATE), "-c", str(OUT_CHANNELS),
1271
+ ],
1272
+ stdin=subprocess.PIPE,
1273
+ stderr=subprocess.PIPE,
1274
+ )
1275
+ if process.stdin is None:
1276
+ logger.warning(f"audio out: aplay did not expose stdin for {alsa_device}")
1277
+ process.kill()
1278
+ return
1279
+
1280
+ class _AplayOutput:
1281
+ def write(self, chunk: bytes) -> None:
1282
+ if process.poll() is not None:
1283
+ detail = ""
1284
+ if process.stderr is not None:
1285
+ detail = process.stderr.read().decode("utf-8", errors="replace").strip()
1286
+ raise OSError(detail or f"aplay exited {process.returncode}")
1287
+ process.stdin.write(chunk)
1288
+ process.stdin.flush()
1289
+
1290
+ def stop_stream(self) -> None:
1291
+ if process.stdin and not process.stdin.closed:
1292
+ process.stdin.close()
1293
+ try:
1294
+ process.wait(timeout=2)
1295
+ except subprocess.TimeoutExpired:
1296
+ process.terminate()
1297
+
1298
+ def close(self) -> None:
1299
+ return
1300
+
1301
+ out = _AplayOutput()
1302
+ output_device_index = alsa_device
1303
+ else:
1304
+ try:
1305
+ import pyaudio
1306
+ except Exception as e:
1307
+ logger.warning(f"pyaudio unavailable for playback: {e}")
1308
+ return
1309
+ pa = pyaudio.PyAudio()
1310
+ # Same MME-vs-WASAPI gotcha as mic capture (see
1311
+ # PyAudioCapture._resolve_device): prefer the endpoint backing the
1312
+ # Windows volume mixer instead of the silent MME default.
1313
+ if selected_output.strip().isdigit():
1314
+ output_device_index = int(selected_output.strip())
1315
+ try:
1316
+ if selected_output.lower() == "default":
1317
+ wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
1318
+ idx = wasapi.get("defaultOutputDevice")
1319
+ if idx is not None and idx >= 0:
1320
+ output_device_index = idx
1321
+ elif output_device_index is None:
1322
+ needle = selected_output.lower()
1323
+ for i in range(pa.get_device_count()):
1324
+ info = pa.get_device_info_by_index(i)
1325
+ if info.get("maxOutputChannels", 0) > 0 and needle in str(info.get("name", "")).lower():
1326
+ output_device_index = i
1327
+ break
1328
+ except Exception as e:
1329
+ logger.debug(f"audio out: WASAPI default output lookup failed, using PyAudio default: {e}")
1330
+ out = pa.open(
1331
+ format=pyaudio.paInt16, channels=OUT_CHANNELS, rate=OUT_RATE, output=True,
1332
+ output_device_index=output_device_index,
1333
+ frames_per_buffer=960,
1334
+ )
1274
1335
  logger.info(f"audio out: opened playback stream for {sid} (device_index={output_device_index})")
1275
1336
  frame_count = 0
1276
1337
  mismatch_logged = False
@@ -1409,9 +1470,10 @@ class LiveKitPublisher:
1409
1470
  f"audio out: {sid} received {frame_count} frames, "
1410
1471
  f"writer wrote {written_frames[0]} chunks, queue backlog at close={write_queue.qsize()}"
1411
1472
  )
1412
- out.stop_stream()
1413
- out.close()
1414
- pa.terminate()
1473
+ out.stop_stream()
1474
+ out.close()
1475
+ if pa is not None:
1476
+ pa.terminate()
1415
1477
 
1416
1478
  async def stop(self) -> None:
1417
1479
  self._stop_event.set()
@@ -1505,7 +1567,8 @@ class LiveKitPublisher:
1505
1567
  samples_per_frame = int(48000 * frame_ms / 1000)
1506
1568
  samples_per_batch = int(48000 * BATCH_MS / 1000)
1507
1569
  bytes_per_frame = samples_per_frame * 2 # int16 mono
1508
- import audioop
1570
+ import audioop
1571
+ mic_gain = max(1.0, min(float(os.getenv("ROBOPARK_MIC_GAIN", "4.0")), 12.0))
1509
1572
  _diag_peak = 0
1510
1573
  _diag_count = 0
1511
1574
  _diag_last_log = time.monotonic()
@@ -1521,11 +1584,14 @@ class LiveKitPublisher:
1521
1584
  data = bytes(batch.data)
1522
1585
  _pub_start = time.monotonic()
1523
1586
  for off in range(0, len(data) - bytes_per_frame + 1, bytes_per_frame):
1524
- chunk = data[off:off + bytes_per_frame]
1525
- frame = self.rtc.AudioFrame(
1587
+ chunk = data[off:off + bytes_per_frame]
1588
+ if mic_gain != 1.0:
1589
+ chunk = audioop.mul(chunk, 2, mic_gain)
1590
+ frame = self.rtc.AudioFrame(
1526
1591
  data=chunk, sample_rate=48000, num_channels=1, samples_per_channel=samples_per_frame,
1527
1592
  )
1528
- await self.audio_source.capture_frame(frame)
1593
+ await self.audio_source.capture_frame(frame)
1594
+ self._mic_streaming.set()
1529
1595
  p = audioop.max(chunk, 2)
1530
1596
  _diag_peak = max(_diag_peak, p)
1531
1597
  _diag_count += 1