infinicode 2.8.85 → 2.8.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/robopark/agent-ctl.js +54 -2
- package/dist/robopark/serve.js +32 -2
- package/docs/ROBOPARK_PRODUCTION_MEMORY.md +111 -0
- package/package.json +4 -3
- package/packages/robopark/README.md +4 -1
- package/packages/robopark/scheduler/main.py +44 -15
- package/packages/robopark/scheduler/preview_agent.py +118 -52
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* registration mechanics is reimplemented here.
|
|
19
19
|
*/
|
|
20
20
|
import { spawn, execFile } from 'node:child_process';
|
|
21
|
-
import { existsSync } from 'node:fs';
|
|
21
|
+
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
22
22
|
import { hostname, networkInterfaces } from 'node:os';
|
|
23
23
|
import { join } from 'node:path';
|
|
24
24
|
import chalk from 'chalk';
|
|
@@ -103,6 +103,48 @@ function validateComposePath(path) {
|
|
|
103
103
|
throw new Error(`no docker-compose.yaml/.yml found in '${path}'. Pass --path pointing at a ROBOVOICE checkout ` +
|
|
104
104
|
`(the directory containing docker-compose.yaml), not this repo.`);
|
|
105
105
|
}
|
|
106
|
+
/** Persist the single-track greeting fix in a ROBOVOICE checkout. */
|
|
107
|
+
function ensureRobovoiceSingleAudioTrack(path) {
|
|
108
|
+
const candidates = [
|
|
109
|
+
join(path, 'voice_agent.py'),
|
|
110
|
+
join(path, 'app', 'voice_agent.py'),
|
|
111
|
+
join(path, 'src', 'voice_agent.py'),
|
|
112
|
+
];
|
|
113
|
+
const file = candidates.find((candidate) => existsSync(candidate));
|
|
114
|
+
if (!file) {
|
|
115
|
+
return { patched: false, warning: 'voice_agent.py was not found; skipping RoboPark greeting compatibility patch' };
|
|
116
|
+
}
|
|
117
|
+
const source = readFileSync(file, 'utf8');
|
|
118
|
+
if (source.includes('Initial greeting completed through AgentSession audio track')) {
|
|
119
|
+
return { patched: false, file };
|
|
120
|
+
}
|
|
121
|
+
const startMarker = ' # Synthesize the greeting first';
|
|
122
|
+
const endMarker = ' except Exception as e:';
|
|
123
|
+
const start = source.indexOf(startMarker);
|
|
124
|
+
const end = start >= 0 ? source.indexOf(endMarker, start) : -1;
|
|
125
|
+
if (start < 0 || end < 0) {
|
|
126
|
+
return {
|
|
127
|
+
patched: false,
|
|
128
|
+
file,
|
|
129
|
+
warning: 'voice_agent.py does not contain the known duplicate greeting-track block; no automatic patch applied',
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
const newline = source.includes('\r\n') ? '\r\n' : '\n';
|
|
133
|
+
const replacement = [
|
|
134
|
+
' # Use the existing conversation track. A dedicated greeting',
|
|
135
|
+
' # track makes Linux robots compete for the same ALSA device.',
|
|
136
|
+
' import inspect as _inspect',
|
|
137
|
+
' speech = session.say(greeting, allow_interruptions=False)',
|
|
138
|
+
' if _inspect.isawaitable(speech):',
|
|
139
|
+
' await speech',
|
|
140
|
+
" elif hasattr(speech, 'wait_for_playout'):",
|
|
141
|
+
' await speech.wait_for_playout()',
|
|
142
|
+
" logger.info('Initial greeting completed through AgentSession audio track')",
|
|
143
|
+
'',
|
|
144
|
+
].join(newline);
|
|
145
|
+
writeFileSync(file, source.slice(0, start) + replacement + source.slice(end), 'utf8');
|
|
146
|
+
return { patched: true, file };
|
|
147
|
+
}
|
|
106
148
|
export async function roboparkAgentUp(opts) {
|
|
107
149
|
console.log(chalk.bold('\n robopark agent up'));
|
|
108
150
|
console.log(chalk.dim(' ' + '─'.repeat(52)));
|
|
@@ -125,8 +167,18 @@ export async function roboparkAgentUp(opts) {
|
|
|
125
167
|
console.log(chalk.dim(` using: ${compose.cmd} ${compose.baseArgs.join(' ')}`.trimEnd()));
|
|
126
168
|
console.log(` path: ${chalk.cyan(opts.path)}`);
|
|
127
169
|
console.log();
|
|
170
|
+
const greetingPatch = ensureRobovoiceSingleAudioTrack(opts.path);
|
|
171
|
+
if (greetingPatch.patched) {
|
|
172
|
+
console.log(chalk.green(` ✓ persisted single-track RoboPark greeting in ${greetingPatch.file}`));
|
|
173
|
+
}
|
|
174
|
+
else if (greetingPatch.warning) {
|
|
175
|
+
console.log(chalk.yellow(` ⚠ ${greetingPatch.warning}`));
|
|
176
|
+
}
|
|
128
177
|
console.log(chalk.bold(' 1. starting ROBOVOICE docker stack (docker compose up -d)…'));
|
|
129
|
-
|
|
178
|
+
// Rebuild only when compatibility source changed, making the fix survive
|
|
179
|
+
// container recreation without slowing normal restarts.
|
|
180
|
+
const upArgs = greetingPatch.patched ? ['up', '-d', '--build'] : ['up', '-d'];
|
|
181
|
+
const upCode = await runCompose(compose, upArgs, opts.path);
|
|
130
182
|
if (upCode !== 0) {
|
|
131
183
|
console.log(chalk.red(`\n ✗ docker compose up -d exited with code ${upCode}`));
|
|
132
184
|
process.exit(upCode);
|
package/dist/robopark/serve.js
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import chalk from 'chalk';
|
|
11
11
|
import { spawn, execSync } from 'node:child_process';
|
|
12
|
-
import { existsSync } from 'node:fs';
|
|
12
|
+
import { cpSync, existsSync, mkdirSync, statSync } from 'node:fs';
|
|
13
|
+
import { homedir } from 'node:os';
|
|
13
14
|
import { dirname, join } from 'node:path';
|
|
14
15
|
import { fileURLToPath } from 'node:url';
|
|
15
16
|
function pkgDir() {
|
|
@@ -68,6 +69,33 @@ function findPython() {
|
|
|
68
69
|
// ENOENT/"not found" error from the OS rather than a silent fallback.
|
|
69
70
|
return process.platform === 'win32' ? 'python' : 'python3';
|
|
70
71
|
}
|
|
72
|
+
function resolveSchedulerDataDir(explicit) {
|
|
73
|
+
if (explicit)
|
|
74
|
+
return explicit;
|
|
75
|
+
if (process.env.SCHEDULER_DATA_DIR)
|
|
76
|
+
return process.env.SCHEDULER_DATA_DIR;
|
|
77
|
+
const stable = join(homedir(), '.robopark', 'scheduler');
|
|
78
|
+
const stableDb = join(stable, 'scheduler.db');
|
|
79
|
+
if (existsSync(stableDb))
|
|
80
|
+
return stable;
|
|
81
|
+
// Older releases stored data beside whichever npm installation launched
|
|
82
|
+
// the scheduler. Local, global and npx installs therefore created separate
|
|
83
|
+
// databases. Migrate the largest existing DB once into stable user storage.
|
|
84
|
+
const legacyCandidates = [join(pkgDir(), '..', 'data')];
|
|
85
|
+
if (process.platform === 'win32' && process.env.APPDATA) {
|
|
86
|
+
legacyCandidates.push(join(process.env.APPDATA, 'npm', 'node_modules', 'data'));
|
|
87
|
+
}
|
|
88
|
+
const legacy = legacyCandidates
|
|
89
|
+
.filter((candidate, index, all) => all.indexOf(candidate) === index)
|
|
90
|
+
.filter(candidate => existsSync(join(candidate, 'scheduler.db')))
|
|
91
|
+
.sort((a, b) => statSync(join(b, 'scheduler.db')).size - statSync(join(a, 'scheduler.db')).size)[0];
|
|
92
|
+
mkdirSync(stable, { recursive: true });
|
|
93
|
+
if (legacy) {
|
|
94
|
+
cpSync(legacy, stable, { recursive: true, force: false, errorOnExist: false });
|
|
95
|
+
console.log(chalk.yellow(` migrated scheduler data: ${legacy} -> ${stable}`));
|
|
96
|
+
}
|
|
97
|
+
return stable;
|
|
98
|
+
}
|
|
71
99
|
/** Resolve the actual infinicode CLI entry point so `robopark setup --start`
|
|
72
100
|
* works even when global bins are not on PATH.
|
|
73
101
|
*
|
|
@@ -110,6 +138,7 @@ export async function roboparkServe(opts) {
|
|
|
110
138
|
const python = findPython();
|
|
111
139
|
const schedulerDir = dirname(schedulerPath);
|
|
112
140
|
const requirements = join(schedulerDir, 'requirements.txt');
|
|
141
|
+
const dataDir = resolveSchedulerDataDir(opts.dataDir);
|
|
113
142
|
// Ensure scheduler Python dependencies are installed before starting.
|
|
114
143
|
if (existsSync(requirements)) {
|
|
115
144
|
console.log(chalk.dim(' ensuring scheduler Python deps…'));
|
|
@@ -125,7 +154,7 @@ export async function roboparkServe(opts) {
|
|
|
125
154
|
...process.env,
|
|
126
155
|
SCHEDULER_HOST: host,
|
|
127
156
|
SCHEDULER_PORT: String(port),
|
|
128
|
-
SCHEDULER_DATA_DIR:
|
|
157
|
+
SCHEDULER_DATA_DIR: dataDir,
|
|
129
158
|
};
|
|
130
159
|
if (opts.gateway)
|
|
131
160
|
env.INFINIBOT_GATEWAY_URL = opts.gateway;
|
|
@@ -138,6 +167,7 @@ export async function roboparkServe(opts) {
|
|
|
138
167
|
console.log(` scheduler: ${chalk.cyan(schedulerPath)}`);
|
|
139
168
|
console.log(` python: ${chalk.cyan(python)}`);
|
|
140
169
|
console.log(` listen: ${chalk.cyan(`${host}:${port}`)}`);
|
|
170
|
+
console.log(` data: ${chalk.cyan(dataDir)}`);
|
|
141
171
|
if (opts.gateway)
|
|
142
172
|
console.log(` gateway: ${chalk.cyan(opts.gateway)}`);
|
|
143
173
|
console.log();
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# RoboPark Production Memory
|
|
2
|
+
|
|
3
|
+
Last verified: 2026-07-16
|
|
4
|
+
|
|
5
|
+
This records production facts future operators and coding agents must
|
|
6
|
+
preserve. Never add enrollment tokens, mesh tokens, API keys, or device tokens.
|
|
7
|
+
|
|
8
|
+
## Verified Architecture
|
|
9
|
+
|
|
10
|
+
- `livekit-1` is the Windows hub.
|
|
11
|
+
- Mesh/dashboard: TCP `47913`.
|
|
12
|
+
- RoboPark scheduler: TCP `8080`, proxied under `/robopark`.
|
|
13
|
+
- LiveKit: TCP `7880`.
|
|
14
|
+
- Production voice worker container: `caal-agent`.
|
|
15
|
+
- BMW maps to device `dev_0422e545` and character `vixen` / `VIXEN (BMW)`.
|
|
16
|
+
- Robots publish camera and microphone tracks to LiveKit. Park camera preview
|
|
17
|
+
uses the RoboVision relay rather than LiveKit video.
|
|
18
|
+
|
|
19
|
+
## BMW Launch Device Lock
|
|
20
|
+
|
|
21
|
+
- Camera: `/dev/video0` (`H65 USB CAMERA`).
|
|
22
|
+
- Microphone: scheduler ID `2`, `Usb Audio Device: USB Audio (hw:3,0)`.
|
|
23
|
+
- Speaker: scheduler ID `1`, `USB Audio Device: - (hw:2,0)`.
|
|
24
|
+
- Linux live playback must use ALSA `plughw:2,0` through `aplay`.
|
|
25
|
+
- Production mic gain defaults to `ROBOPARK_MIC_GAIN=4.0` and is bounded to
|
|
26
|
+
`1.0..12.0` before PCM frames enter LiveKit.
|
|
27
|
+
- RoboVision/sounddevice numeric IDs are not PyAudio numeric IDs.
|
|
28
|
+
- Do not use HDMI or metadata-only `/dev/video*` devices as launch defaults.
|
|
29
|
+
|
|
30
|
+
## Verified Production Flow
|
|
31
|
+
|
|
32
|
+
BMW passed these measured stages in one production session:
|
|
33
|
+
|
|
34
|
+
1. `scheduler_session`
|
|
35
|
+
2. `livekit_join`
|
|
36
|
+
3. `tts_subscribed`
|
|
37
|
+
4. `playback_started`
|
|
38
|
+
5. `camera_published`
|
|
39
|
+
6. `microphone_published`
|
|
40
|
+
7. `robot_online`
|
|
41
|
+
8. `camera_ready`
|
|
42
|
+
9. `microphone_ready`
|
|
43
|
+
10. `speaker_ready`
|
|
44
|
+
|
|
45
|
+
`playback_started` means the first remote TTS frame reached the robot. Never
|
|
46
|
+
report end-to-end audio delivery without this event.
|
|
47
|
+
|
|
48
|
+
## Root Causes Fixed
|
|
49
|
+
|
|
50
|
+
### Silent Greeting
|
|
51
|
+
|
|
52
|
+
The old ROBOVOICE greeting published a second dedicated LiveKit audio track.
|
|
53
|
+
The Pi subscribed to both tracks and attempted to open the same speaker twice.
|
|
54
|
+
ROBOVOICE must call `session.say(...)` so greeting and conversation share one
|
|
55
|
+
track. `robopark agent-up` now detects the old source block, patches the
|
|
56
|
+
ROBOVOICE checkout, and rebuilds Compose once.
|
|
57
|
+
|
|
58
|
+
### Linux Speaker Playback
|
|
59
|
+
|
|
60
|
+
The preview agent previously used PyAudio for the selected speaker. BMW's
|
|
61
|
+
proven output is ALSA `aplay -D plughw:2,0`. Infinicode `2.8.86` uses that
|
|
62
|
+
same endpoint for live TTS whenever a Linux output selection contains `hw:`.
|
|
63
|
+
|
|
64
|
+
### Silent Conversation After Greeting
|
|
65
|
+
|
|
66
|
+
Publishing a LiveKit microphone track does not prove that PCM capture started.
|
|
67
|
+
On Linux the production capture path is `arecord -D plughw:3,0 -f S16_LE -r
|
|
68
|
+
48000 -c 1`; it must work even when PyAudio is unavailable. The Pi now waits
|
|
69
|
+
for actual PCM frames before reporting `microphone_published`, reports a blocked
|
|
70
|
+
stage when capture does not start within five seconds, and applies bounded mic
|
|
71
|
+
gain before publication.
|
|
72
|
+
|
|
73
|
+
### Duplicate Speaker Tests
|
|
74
|
+
|
|
75
|
+
Only one robot process may claim a scheduler `speaker_test`. The scheduler
|
|
76
|
+
uses an atomic lease, and supervisor-status queries exclude speaker-test rows.
|
|
77
|
+
|
|
78
|
+
### BMW Alias Configuration
|
|
79
|
+
|
|
80
|
+
Scheduler routes canonicalize alias `bmw` to `dev_0422e545` before reading
|
|
81
|
+
character, voice-stack, and greeting configuration.
|
|
82
|
+
|
|
83
|
+
## Operational Rules
|
|
84
|
+
|
|
85
|
+
- Run one unified robot runtime per robot. Do not manually start preview,
|
|
86
|
+
vision, or mesh beside the systemd runtime.
|
|
87
|
+
- Do not run audio diagnostics during a live conversation.
|
|
88
|
+
- A direct ALSA test does not validate production if production uses another
|
|
89
|
+
API, index space, sample format, or opens the device twice.
|
|
90
|
+
- After a hub reboot, verify `47913`, `8080`, and `7880` before robot media.
|
|
91
|
+
- `tts_subscribed` alone is insufficient; require `playback_started`.
|
|
92
|
+
- Greeting `playback_started` alone does not prove conversation. A complete
|
|
93
|
+
turn also requires a user/STT event, an LLM response, and response playback.
|
|
94
|
+
- Container-local changes are temporary. Persist voice changes in the
|
|
95
|
+
ROBOVOICE checkout/image or use `robopark agent-up --path <checkout>`.
|
|
96
|
+
|
|
97
|
+
## Minimal Verification
|
|
98
|
+
|
|
99
|
+
Run from the hub:
|
|
100
|
+
|
|
101
|
+
```powershell
|
|
102
|
+
try { Invoke-RestMethod -Method Post http://127.0.0.1:8080/api/robots/bmw/pipeline-test/stop | Out-Null } catch {}
|
|
103
|
+
Start-Sleep 3
|
|
104
|
+
Invoke-RestMethod -Method Post http://127.0.0.1:8080/api/robots/bmw/pipeline-test/start | Out-Null
|
|
105
|
+
Start-Sleep 30
|
|
106
|
+
$s = Invoke-RestMethod http://127.0.0.1:8080/api/robots/bmw/pipeline-status
|
|
107
|
+
$id = $s.latest_session.id
|
|
108
|
+
$s.events | Where-Object session_id -eq $id | Select-Object timestamp,stage,status,message
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Required result: `playback_started ok`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "infinicode",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.87",
|
|
4
4
|
"description": "OpenKernel — provider-agnostic AI execution kernel. Native coding agent + mission-driven execution runtime.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/kernel/index.js",
|
|
@@ -29,8 +29,9 @@
|
|
|
29
29
|
"static",
|
|
30
30
|
"packages/robopark/scheduler",
|
|
31
31
|
"packages/robopark/pi-client",
|
|
32
|
-
"packages/robopark/vision",
|
|
33
|
-
".
|
|
32
|
+
"packages/robopark/vision",
|
|
33
|
+
"docs/ROBOPARK_PRODUCTION_MEMORY.md",
|
|
34
|
+
".opencode/plugins",
|
|
34
35
|
".opencode/themes",
|
|
35
36
|
".opencode/tui.json",
|
|
36
37
|
"README.md"
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
# robopark
|
|
1
|
+
# robopark
|
|
2
|
+
|
|
3
|
+
Production architecture, device locks, verified stages, and incident memory
|
|
4
|
+
are recorded in [`../../docs/ROBOPARK_PRODUCTION_MEMORY.md`](../../docs/ROBOPARK_PRODUCTION_MEMORY.md).
|
|
2
5
|
|
|
3
6
|
The **RoboPark fleet control CLI** — set up, watch, and drive a room full of
|
|
4
7
|
talking robots from one command. `robopark` is the operator front-end; it rides
|
|
@@ -604,6 +604,9 @@ async def init_db():
|
|
|
604
604
|
("sessions", "voice_stack_id", "TEXT"),
|
|
605
605
|
("sessions", "initiated_by", "TEXT"),
|
|
606
606
|
("sessions", "event_token_hash", "TEXT"),
|
|
607
|
+
# Speaker tests are executed only by preview_agent, which can
|
|
608
|
+
# release and restore the active media publisher around ALSA I/O.
|
|
609
|
+
("shell_requests", "claimed_at", "TEXT"),
|
|
607
610
|
):
|
|
608
611
|
try:
|
|
609
612
|
await db.execute(f"ALTER TABLE {_table} ADD COLUMN {_column} {_coldef}")
|
|
@@ -3317,8 +3320,9 @@ async def device_supervisor_status(device_id: str, payload: SupervisorStatusRepo
|
|
|
3317
3320
|
pending = []
|
|
3318
3321
|
try:
|
|
3319
3322
|
async with db.execute(
|
|
3320
|
-
"SELECT service_name, action, kind, params, id FROM shell_requests "
|
|
3321
|
-
"WHERE device_id = ? AND completed_at IS NULL
|
|
3323
|
+
"SELECT service_name, action, kind, params, id FROM shell_requests "
|
|
3324
|
+
"WHERE device_id = ? AND completed_at IS NULL AND kind != 'speaker_test' "
|
|
3325
|
+
"ORDER BY requested_at",
|
|
3322
3326
|
(device_id,),
|
|
3323
3327
|
) as c:
|
|
3324
3328
|
shell_rows = [dict(r) for r in await c.fetchall()]
|
|
@@ -3696,13 +3700,26 @@ async def next_device_speaker_test(device_id: str, authorization: Optional[str]
|
|
|
3696
3700
|
"""Allow the always-running preview agent to execute audio tests."""
|
|
3697
3701
|
if not await _authorize_device(device_id, authorization):
|
|
3698
3702
|
raise HTTPException(401, "Invalid device token")
|
|
3703
|
+
now = datetime.utcnow()
|
|
3704
|
+
stale_before = (now - timedelta(seconds=45)).isoformat()
|
|
3699
3705
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
3700
3706
|
db.row_factory = aiosqlite.Row
|
|
3707
|
+
await db.execute("BEGIN IMMEDIATE")
|
|
3701
3708
|
async with db.execute(
|
|
3702
3709
|
"SELECT id, params FROM shell_requests WHERE device_id = ? AND kind = 'speaker_test' "
|
|
3703
|
-
"AND completed_at IS NULL
|
|
3710
|
+
"AND completed_at IS NULL AND (claimed_at IS NULL OR claimed_at < ?) "
|
|
3711
|
+
"ORDER BY requested_at LIMIT 1", (device_id, stale_before),
|
|
3704
3712
|
) as cursor:
|
|
3705
3713
|
row = await cursor.fetchone()
|
|
3714
|
+
if row:
|
|
3715
|
+
claimed = await db.execute(
|
|
3716
|
+
"UPDATE shell_requests SET claimed_at = ? WHERE id = ? AND completed_at IS NULL "
|
|
3717
|
+
"AND (claimed_at IS NULL OR claimed_at < ?)",
|
|
3718
|
+
(now.isoformat(), row["id"], stale_before),
|
|
3719
|
+
)
|
|
3720
|
+
if claimed.rowcount != 1:
|
|
3721
|
+
row = None
|
|
3722
|
+
await db.commit()
|
|
3706
3723
|
return {"request": {"id": row["id"], "params": json.loads(row["params"] or "{}")} if row else None}
|
|
3707
3724
|
|
|
3708
3725
|
|
|
@@ -4171,25 +4188,35 @@ async def _auto_elevenlabs_voice(robot_name: Optional[str], current: Optional[st
|
|
|
4171
4188
|
|
|
4172
4189
|
|
|
4173
4190
|
async def _resolve_robot_voice_config(robot_id: str) -> RobotVoiceConfig:
|
|
4174
|
-
"""Public helper to resolve the effective voice config for a robot id."""
|
|
4175
|
-
async with aiosqlite.connect(DB_PATH) as db:
|
|
4176
|
-
db.row_factory = aiosqlite.Row
|
|
4177
|
-
|
|
4191
|
+
"""Public helper to resolve the effective voice config for a robot id."""
|
|
4192
|
+
async with aiosqlite.connect(DB_PATH) as db:
|
|
4193
|
+
db.row_factory = aiosqlite.Row
|
|
4194
|
+
# Dashboard routes commonly use the human robot name ("bmw"), while
|
|
4195
|
+
# assignments are persisted against the enrolled device id. Resolve
|
|
4196
|
+
# that alias once so character, stack and greetings use one identity.
|
|
4197
|
+
async with db.execute(
|
|
4198
|
+
"SELECT id FROM devices WHERE id = ? OR lower(name) = lower(?) "
|
|
4199
|
+
"ORDER BY CASE WHEN id = ? THEN 0 ELSE 1 END LIMIT 1",
|
|
4200
|
+
(robot_id, robot_id, robot_id),
|
|
4201
|
+
) as c:
|
|
4202
|
+
canonical_device = await c.fetchone()
|
|
4203
|
+
canonical_id = canonical_device["id"] if canonical_device else robot_id
|
|
4204
|
+
stack_dict = await _resolve_voice_stack_for_robot(db, canonical_id)
|
|
4178
4205
|
|
|
4179
4206
|
# Determine character preset + name
|
|
4180
4207
|
character_id = None
|
|
4181
4208
|
character_name = None
|
|
4182
4209
|
system_prompt = None
|
|
4183
4210
|
async with db.execute(
|
|
4184
|
-
"SELECT character_preset_id FROM robot_voice_configs WHERE robot_id = ?", (
|
|
4211
|
+
"SELECT character_preset_id FROM robot_voice_configs WHERE robot_id = ?", (canonical_id,)
|
|
4185
4212
|
) as c:
|
|
4186
4213
|
rvc = await c.fetchone()
|
|
4187
4214
|
if rvc and rvc["character_preset_id"]:
|
|
4188
4215
|
character_id = rvc["character_preset_id"]
|
|
4189
4216
|
else:
|
|
4190
4217
|
async with db.execute(
|
|
4191
|
-
"SELECT character_id FROM robots WHERE id = ? UNION ALL SELECT character_id FROM devices WHERE id = ?",
|
|
4192
|
-
(
|
|
4218
|
+
"SELECT character_id FROM robots WHERE id = ? UNION ALL SELECT character_id FROM devices WHERE id = ?",
|
|
4219
|
+
(canonical_id, canonical_id),
|
|
4193
4220
|
) as c:
|
|
4194
4221
|
row = await c.fetchone()
|
|
4195
4222
|
if row:
|
|
@@ -4204,21 +4231,23 @@ async def _resolve_robot_voice_config(robot_id: str) -> RobotVoiceConfig:
|
|
|
4204
4231
|
system_prompt = row["system_prompt"]
|
|
4205
4232
|
if not character_name:
|
|
4206
4233
|
async with db.execute(
|
|
4207
|
-
"SELECT name FROM robots WHERE id = ? UNION ALL SELECT name FROM devices WHERE id = ?",
|
|
4208
|
-
(
|
|
4234
|
+
"SELECT name FROM robots WHERE id = ? UNION ALL SELECT name FROM devices WHERE id = ?",
|
|
4235
|
+
(canonical_id, canonical_id),
|
|
4209
4236
|
) as c:
|
|
4210
4237
|
row = await c.fetchone()
|
|
4211
4238
|
if row:
|
|
4212
4239
|
character_name = row["name"]
|
|
4213
4240
|
|
|
4214
|
-
async with db.execute("SELECT greeting_phrases FROM devices WHERE id = ?", (
|
|
4241
|
+
async with db.execute("SELECT greeting_phrases FROM devices WHERE id = ?", (canonical_id,)) as c:
|
|
4215
4242
|
device_row = await c.fetchone()
|
|
4216
4243
|
greeting_phrases = []
|
|
4217
4244
|
if device_row and device_row["greeting_phrases"]:
|
|
4218
4245
|
try:
|
|
4219
4246
|
greeting_phrases = [str(p) for p in _json.loads(device_row["greeting_phrases"]) if str(p).strip()]
|
|
4220
|
-
except Exception:
|
|
4221
|
-
greeting_phrases = []
|
|
4247
|
+
except Exception:
|
|
4248
|
+
greeting_phrases = []
|
|
4249
|
+
if not greeting_phrases and (character_name or character_id):
|
|
4250
|
+
greeting_phrases = [f"Hello, I am {character_name or character_id}. How can I help you?"]
|
|
4222
4251
|
if stack_dict:
|
|
4223
4252
|
stack_dict["tts_voice"] = await _auto_elevenlabs_voice(character_name or character_id, stack_dict.get("tts_voice"))
|
|
4224
4253
|
voice_stack = VoiceStack(**stack_dict) if stack_dict else None
|
|
@@ -1134,6 +1134,7 @@ class LiveKitPublisher:
|
|
|
1134
1134
|
self._tasks: list[asyncio.Task] = []
|
|
1135
1135
|
self._capture: Optional["VideoCapture"] = None
|
|
1136
1136
|
self._mic_capture: Optional["AudioCapture"] = None
|
|
1137
|
+
self._mic_streaming = asyncio.Event()
|
|
1137
1138
|
# Motion sampling state belongs to the publisher instance. Keeping it
|
|
1138
1139
|
# initialized here prevents shutdown/reopen paths from raising while
|
|
1139
1140
|
# the camera is being handed between preview and voice sessions.
|
|
@@ -1212,13 +1213,12 @@ class LiveKitPublisher:
|
|
|
1212
1213
|
self.video_source.capture_frame(first_frame)
|
|
1213
1214
|
await self.agent._report_pipeline("camera_published", "ok", "Camera track published")
|
|
1214
1215
|
|
|
1215
|
-
if audio_enabled:
|
|
1216
|
-
self.audio_source = self.rtc.AudioSource(48000, 1)
|
|
1217
|
-
self.audio_track = self.rtc.LocalAudioTrack.create_audio_track("microphone", self.audio_source)
|
|
1218
|
-
aopts = self.rtc.TrackPublishOptions()
|
|
1219
|
-
aopts.source = self.rtc.TrackSource.SOURCE_MICROPHONE
|
|
1216
|
+
if audio_enabled:
|
|
1217
|
+
self.audio_source = self.rtc.AudioSource(48000, 1)
|
|
1218
|
+
self.audio_track = self.rtc.LocalAudioTrack.create_audio_track("microphone", self.audio_source)
|
|
1219
|
+
aopts = self.rtc.TrackPublishOptions()
|
|
1220
|
+
aopts.source = self.rtc.TrackSource.SOURCE_MICROPHONE
|
|
1220
1221
|
await self.room.local_participant.publish_track(self.audio_track, aopts)
|
|
1221
|
-
await self.agent._report_pipeline("microphone_published", "ok", "Microphone track published")
|
|
1222
1222
|
|
|
1223
1223
|
# Register speaker playback (subscribe to the voice agent's TTS audio
|
|
1224
1224
|
# track) BEFORE opening the camera. Camera open is a slow/occasionally
|
|
@@ -1227,50 +1227,111 @@ class LiveKitPublisher:
|
|
|
1227
1227
|
# entire event loop — including receiving the agent's greeting audio
|
|
1228
1228
|
# — until it finished, so a short "Hello friend!" greeting could be
|
|
1229
1229
|
# over and gone before we ever got a chance to subscribe to it.
|
|
1230
|
-
if
|
|
1231
|
-
self._tasks.append(asyncio.create_task(self._audio_loop()))
|
|
1230
|
+
if audio_enabled:
|
|
1231
|
+
self._tasks.append(asyncio.create_task(self._audio_loop()))
|
|
1232
|
+
try:
|
|
1233
|
+
await asyncio.wait_for(self._mic_streaming.wait(), timeout=5.0)
|
|
1234
|
+
await self.agent._report_pipeline(
|
|
1235
|
+
"microphone_published", "ok", "Microphone track published with live PCM frames"
|
|
1236
|
+
)
|
|
1237
|
+
except asyncio.TimeoutError:
|
|
1238
|
+
await self.agent._report_pipeline(
|
|
1239
|
+
"microphone_published", "blocked", "Microphone track published but PCM capture did not start"
|
|
1240
|
+
)
|
|
1241
|
+
raise RuntimeError(f"microphone capture did not start for {self.agent.audio_capture_device}")
|
|
1232
1242
|
|
|
1233
1243
|
if self._capture and self.video_source:
|
|
1234
1244
|
self._tasks.append(asyncio.create_task(self._video_loop()))
|
|
1235
1245
|
|
|
1236
|
-
async def _play_remote_audio(self, track, sid: str) -> None:
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
except Exception as e:
|
|
1240
|
-
logger.warning(f"pyaudio unavailable for playback: {e}")
|
|
1241
|
-
return
|
|
1242
|
-
OUT_RATE = 48000
|
|
1243
|
-
OUT_CHANNELS = 2
|
|
1244
|
-
pa = pyaudio.PyAudio()
|
|
1245
|
-
# Same MME-vs-WASAPI gotcha as mic capture (see PyAudioCapture._resolve_device):
|
|
1246
|
-
# PyAudio's global default output device resolves through MME, which on this
|
|
1247
|
-
# machine will accept writes and report success with zero exceptions while
|
|
1248
|
-
# never actually reaching the real speakers. Prefer WASAPI's default output,
|
|
1249
|
-
# which is what Windows' own volume mixer/meter is backed by.
|
|
1250
|
-
output_device_index = None
|
|
1246
|
+
async def _play_remote_audio(self, track, sid: str) -> None:
|
|
1247
|
+
OUT_RATE = 48000
|
|
1248
|
+
OUT_CHANNELS = 2
|
|
1251
1249
|
selected_output = str(self.agent.audio_playback_device or "default")
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1250
|
+
pa = None
|
|
1251
|
+
output_device_index = None
|
|
1252
|
+
|
|
1253
|
+
# RoboVision inventories Linux devices through sounddevice/PortAudio,
|
|
1254
|
+
# but the numeric indices are not stable across PyAudio builds. More
|
|
1255
|
+
# importantly, BMW's production USB adapter is already proven through
|
|
1256
|
+
# ALSA's plughw conversion path. Use that exact endpoint for live TTS
|
|
1257
|
+
# instead of reopening the unrelated PyAudio index.
|
|
1258
|
+
if sys.platform.startswith("linux") and "hw:" in selected_output:
|
|
1259
|
+
import re
|
|
1260
|
+
import subprocess
|
|
1261
|
+
|
|
1262
|
+
match = re.search(r"\b(hw:\d+,\d+)\b", selected_output)
|
|
1263
|
+
if not match:
|
|
1264
|
+
logger.warning(f"audio out: no ALSA hardware address in {selected_output!r}")
|
|
1265
|
+
return
|
|
1266
|
+
alsa_device = f"plug{match.group(1)}"
|
|
1267
|
+
process = subprocess.Popen(
|
|
1268
|
+
[
|
|
1269
|
+
"aplay", "-q", "-D", alsa_device, "-t", "raw",
|
|
1270
|
+
"-f", "S16_LE", "-r", str(OUT_RATE), "-c", str(OUT_CHANNELS),
|
|
1271
|
+
],
|
|
1272
|
+
stdin=subprocess.PIPE,
|
|
1273
|
+
stderr=subprocess.PIPE,
|
|
1274
|
+
)
|
|
1275
|
+
if process.stdin is None:
|
|
1276
|
+
logger.warning(f"audio out: aplay did not expose stdin for {alsa_device}")
|
|
1277
|
+
process.kill()
|
|
1278
|
+
return
|
|
1279
|
+
|
|
1280
|
+
class _AplayOutput:
|
|
1281
|
+
def write(self, chunk: bytes) -> None:
|
|
1282
|
+
if process.poll() is not None:
|
|
1283
|
+
detail = ""
|
|
1284
|
+
if process.stderr is not None:
|
|
1285
|
+
detail = process.stderr.read().decode("utf-8", errors="replace").strip()
|
|
1286
|
+
raise OSError(detail or f"aplay exited {process.returncode}")
|
|
1287
|
+
process.stdin.write(chunk)
|
|
1288
|
+
process.stdin.flush()
|
|
1289
|
+
|
|
1290
|
+
def stop_stream(self) -> None:
|
|
1291
|
+
if process.stdin and not process.stdin.closed:
|
|
1292
|
+
process.stdin.close()
|
|
1293
|
+
try:
|
|
1294
|
+
process.wait(timeout=2)
|
|
1295
|
+
except subprocess.TimeoutExpired:
|
|
1296
|
+
process.terminate()
|
|
1297
|
+
|
|
1298
|
+
def close(self) -> None:
|
|
1299
|
+
return
|
|
1300
|
+
|
|
1301
|
+
out = _AplayOutput()
|
|
1302
|
+
output_device_index = alsa_device
|
|
1303
|
+
else:
|
|
1304
|
+
try:
|
|
1305
|
+
import pyaudio
|
|
1306
|
+
except Exception as e:
|
|
1307
|
+
logger.warning(f"pyaudio unavailable for playback: {e}")
|
|
1308
|
+
return
|
|
1309
|
+
pa = pyaudio.PyAudio()
|
|
1310
|
+
# Same MME-vs-WASAPI gotcha as mic capture (see
|
|
1311
|
+
# PyAudioCapture._resolve_device): prefer the endpoint backing the
|
|
1312
|
+
# Windows volume mixer instead of the silent MME default.
|
|
1313
|
+
if selected_output.strip().isdigit():
|
|
1314
|
+
output_device_index = int(selected_output.strip())
|
|
1315
|
+
try:
|
|
1316
|
+
if selected_output.lower() == "default":
|
|
1317
|
+
wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
|
|
1318
|
+
idx = wasapi.get("defaultOutputDevice")
|
|
1319
|
+
if idx is not None and idx >= 0:
|
|
1320
|
+
output_device_index = idx
|
|
1321
|
+
elif output_device_index is None:
|
|
1322
|
+
needle = selected_output.lower()
|
|
1323
|
+
for i in range(pa.get_device_count()):
|
|
1324
|
+
info = pa.get_device_info_by_index(i)
|
|
1325
|
+
if info.get("maxOutputChannels", 0) > 0 and needle in str(info.get("name", "")).lower():
|
|
1326
|
+
output_device_index = i
|
|
1327
|
+
break
|
|
1328
|
+
except Exception as e:
|
|
1329
|
+
logger.debug(f"audio out: WASAPI default output lookup failed, using PyAudio default: {e}")
|
|
1330
|
+
out = pa.open(
|
|
1331
|
+
format=pyaudio.paInt16, channels=OUT_CHANNELS, rate=OUT_RATE, output=True,
|
|
1332
|
+
output_device_index=output_device_index,
|
|
1333
|
+
frames_per_buffer=960,
|
|
1334
|
+
)
|
|
1274
1335
|
logger.info(f"audio out: opened playback stream for {sid} (device_index={output_device_index})")
|
|
1275
1336
|
frame_count = 0
|
|
1276
1337
|
mismatch_logged = False
|
|
@@ -1409,9 +1470,10 @@ class LiveKitPublisher:
|
|
|
1409
1470
|
f"audio out: {sid} received {frame_count} frames, "
|
|
1410
1471
|
f"writer wrote {written_frames[0]} chunks, queue backlog at close={write_queue.qsize()}"
|
|
1411
1472
|
)
|
|
1412
|
-
out.stop_stream()
|
|
1413
|
-
out.close()
|
|
1414
|
-
pa
|
|
1473
|
+
out.stop_stream()
|
|
1474
|
+
out.close()
|
|
1475
|
+
if pa is not None:
|
|
1476
|
+
pa.terminate()
|
|
1415
1477
|
|
|
1416
1478
|
async def stop(self) -> None:
|
|
1417
1479
|
self._stop_event.set()
|
|
@@ -1505,7 +1567,8 @@ class LiveKitPublisher:
|
|
|
1505
1567
|
samples_per_frame = int(48000 * frame_ms / 1000)
|
|
1506
1568
|
samples_per_batch = int(48000 * BATCH_MS / 1000)
|
|
1507
1569
|
bytes_per_frame = samples_per_frame * 2 # int16 mono
|
|
1508
|
-
import audioop
|
|
1570
|
+
import audioop
|
|
1571
|
+
mic_gain = max(1.0, min(float(os.getenv("ROBOPARK_MIC_GAIN", "4.0")), 12.0))
|
|
1509
1572
|
_diag_peak = 0
|
|
1510
1573
|
_diag_count = 0
|
|
1511
1574
|
_diag_last_log = time.monotonic()
|
|
@@ -1521,11 +1584,14 @@ class LiveKitPublisher:
|
|
|
1521
1584
|
data = bytes(batch.data)
|
|
1522
1585
|
_pub_start = time.monotonic()
|
|
1523
1586
|
for off in range(0, len(data) - bytes_per_frame + 1, bytes_per_frame):
|
|
1524
|
-
chunk = data[off:off + bytes_per_frame]
|
|
1525
|
-
|
|
1587
|
+
chunk = data[off:off + bytes_per_frame]
|
|
1588
|
+
if mic_gain != 1.0:
|
|
1589
|
+
chunk = audioop.mul(chunk, 2, mic_gain)
|
|
1590
|
+
frame = self.rtc.AudioFrame(
|
|
1526
1591
|
data=chunk, sample_rate=48000, num_channels=1, samples_per_channel=samples_per_frame,
|
|
1527
1592
|
)
|
|
1528
|
-
await self.audio_source.capture_frame(frame)
|
|
1593
|
+
await self.audio_source.capture_frame(frame)
|
|
1594
|
+
self._mic_streaming.set()
|
|
1529
1595
|
p = audioop.max(chunk, 2)
|
|
1530
1596
|
_diag_peak = max(_diag_peak, p)
|
|
1531
1597
|
_diag_count += 1
|