infinicode 2.8.100 → 2.8.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/kernel/federation/dashboard-html.d.ts +1 -1
- package/dist/kernel/federation/dashboard-html.js +387 -90
- package/dist/robopark/add-robot.d.ts +2 -0
- package/dist/robopark/add-robot.js +39 -8
- package/dist/robopark/agent-ctl.js +42 -26
- package/dist/robopark/robot-runtime.js +10 -1
- package/dist/robopark-cli.js +2 -0
- package/docs/ROBOPARK_PRODUCTION_MEMORY.md +26 -0
- package/package.json +1 -1
- package/packages/robopark/scheduler/main.py +174 -10
- package/packages/robopark/scheduler/preview_agent.py +72 -19
|
@@ -4,6 +4,8 @@ import { type ExplicitContextOpts } from './profile.js';
|
|
|
4
4
|
export interface AddRobotOptions extends ExplicitContextOpts {
|
|
5
5
|
name: string;
|
|
6
6
|
start?: boolean;
|
|
7
|
+
autoStart?: boolean;
|
|
8
|
+
yes?: boolean;
|
|
7
9
|
/** Force a specific park-map pad/character preset id instead of the
|
|
8
10
|
* fuzzy name-based match below — e.g. `--character jaguar`. */
|
|
9
11
|
character?: string;
|
|
@@ -24,8 +24,32 @@
|
|
|
24
24
|
import chalk from 'chalk';
|
|
25
25
|
import { resolveContext } from './profile.js';
|
|
26
26
|
import { roboparkSetup } from './setup.js';
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
function robotReachableSchedulerUrl(schedulerUrl, hubUrl) {
|
|
28
|
+
if (!hubUrl)
|
|
29
|
+
return schedulerUrl;
|
|
30
|
+
try {
|
|
31
|
+
// Port 8080 is intentionally private to the hub. Enrollment, character
|
|
32
|
+
// lookup, heartbeats, and robot sessions must all use the authenticated
|
|
33
|
+
// mesh proxy, whether the hidden scheduler is loopback or a 192.168.x.x
|
|
34
|
+
// address that is only meaningful on the hub's own LAN.
|
|
35
|
+
new URL(schedulerUrl);
|
|
36
|
+
new URL(hubUrl);
|
|
37
|
+
return `${hubUrl.replace(/\/+$/, '')}/robopark`;
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
// Keep the original value so the existing request reports a useful error.
|
|
41
|
+
}
|
|
42
|
+
return schedulerUrl;
|
|
43
|
+
}
|
|
44
|
+
function schedulerApiUrl(schedulerUrl, path, token) {
|
|
45
|
+
const url = new URL(`${schedulerUrl.replace(/\/+$/, '')}${path}`);
|
|
46
|
+
if (token && url.pathname.startsWith('/robopark/')) {
|
|
47
|
+
url.searchParams.set('token', token);
|
|
48
|
+
}
|
|
49
|
+
return url.toString();
|
|
50
|
+
}
|
|
51
|
+
async function mintEnrollmentToken(schedulerUrl, name, characterId, token) {
|
|
52
|
+
const url = schedulerApiUrl(schedulerUrl, '/api/devices', token);
|
|
29
53
|
const res = await fetch(url, {
|
|
30
54
|
method: 'POST',
|
|
31
55
|
headers: { 'content-type': 'application/json' },
|
|
@@ -37,10 +61,10 @@ async function mintEnrollmentToken(schedulerUrl, name, characterId) {
|
|
|
37
61
|
}
|
|
38
62
|
return (await res.json());
|
|
39
63
|
}
|
|
40
|
-
async function pickCharacterPreset(schedulerUrl, name) {
|
|
64
|
+
async function pickCharacterPreset(schedulerUrl, name, token) {
|
|
41
65
|
const lower = name.toLowerCase();
|
|
42
66
|
try {
|
|
43
|
-
const res = await fetch(
|
|
67
|
+
const res = await fetch(schedulerApiUrl(schedulerUrl, '/api/character-presets', token));
|
|
44
68
|
if (!res.ok)
|
|
45
69
|
return undefined;
|
|
46
70
|
const presets = (await res.json());
|
|
@@ -83,11 +107,16 @@ export async function roboparkAddRobot(config, opts) {
|
|
|
83
107
|
console.log(chalk.red(' ✗ no scheduler URL resolved. Pass --scheduler-url/--hub-url, run `robopark hub-use` first, or run this on/near the hub.'));
|
|
84
108
|
process.exit(1);
|
|
85
109
|
}
|
|
86
|
-
|
|
110
|
+
const schedulerUrl = robotReachableSchedulerUrl(ctx.schedulerUrl, ctx.hubUrl);
|
|
111
|
+
if (schedulerUrl.endsWith('/robopark') && !ctx.token) {
|
|
112
|
+
console.log(chalk.red(' ✗ hub mesh token is required to use the scheduler proxy. Pass --token or run `robopark hub-use` first.'));
|
|
113
|
+
process.exit(1);
|
|
114
|
+
}
|
|
115
|
+
console.log(` scheduler: ${chalk.cyan(schedulerUrl)}`);
|
|
87
116
|
console.log(` name: ${chalk.cyan(opts.name)}${ctx.site ? chalk.dim(' @ ' + ctx.site) : ''}`);
|
|
88
117
|
console.log(` network: ${chalk.cyan(network)}`);
|
|
89
118
|
console.log();
|
|
90
|
-
const characterId = opts.character?.trim() || await pickCharacterPreset(
|
|
119
|
+
const characterId = opts.character?.trim() || await pickCharacterPreset(schedulerUrl, opts.name, ctx.token);
|
|
91
120
|
if (characterId) {
|
|
92
121
|
console.log(` character: ${chalk.cyan(characterId)}${opts.character ? chalk.dim(' (forced via --character)') : chalk.dim(' (auto-matched)')}`);
|
|
93
122
|
}
|
|
@@ -96,7 +125,7 @@ export async function roboparkAddRobot(config, opts) {
|
|
|
96
125
|
}
|
|
97
126
|
let minted;
|
|
98
127
|
try {
|
|
99
|
-
minted = await mintEnrollmentToken(
|
|
128
|
+
minted = await mintEnrollmentToken(schedulerUrl, opts.name, characterId, ctx.token);
|
|
100
129
|
}
|
|
101
130
|
catch (e) {
|
|
102
131
|
console.log(chalk.red(` ✗ failed to mint enrollment token: ${e.message}`));
|
|
@@ -120,6 +149,8 @@ export async function roboparkAddRobot(config, opts) {
|
|
|
120
149
|
schedulerPort: ctx.schedulerUrl ? String(new URL(ctx.schedulerUrl).port || '8080') : undefined,
|
|
121
150
|
enrollmentToken: minted.enrollment_token,
|
|
122
151
|
start: true,
|
|
152
|
+
autoStart: opts.autoStart !== false,
|
|
153
|
+
yes: opts.yes !== false,
|
|
123
154
|
lan: opts.lan,
|
|
124
155
|
tailscale: opts.tailscale,
|
|
125
156
|
});
|
|
@@ -140,7 +171,7 @@ export async function roboparkAddRobot(config, opts) {
|
|
|
140
171
|
argv.push('--scheduler-port', schedulerPort);
|
|
141
172
|
}
|
|
142
173
|
argv.push(network === 'tailscale' ? '--tailscale' : '--lan');
|
|
143
|
-
argv.push('--enrollment-token', minted.enrollment_token, '--start', '--auto-start');
|
|
174
|
+
argv.push('--enrollment-token', minted.enrollment_token, '--start', '--auto-start', '--yes');
|
|
144
175
|
console.log(chalk.dim(` run this ON THE ROBOT${characterId ? ` (binds to pad: ${characterId})` : ''}:`));
|
|
145
176
|
console.log(' ' + chalk.cyan(commandString(argv)));
|
|
146
177
|
if (!ctx.hubUrl) {
|
|
@@ -115,35 +115,51 @@ function ensureRobovoiceSingleAudioTrack(path) {
|
|
|
115
115
|
return { patched: false, warning: 'voice_agent.py was not found; skipping RoboPark greeting compatibility patch' };
|
|
116
116
|
}
|
|
117
117
|
const source = readFileSync(file, 'utf8');
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
118
|
+
let updated = source;
|
|
119
|
+
let patched = false;
|
|
120
|
+
const newline = source.includes('\r\n') ? '\r\n' : '\n';
|
|
121
|
+
const singleTrackMarker = 'Initial greeting completed through AgentSession audio track';
|
|
121
122
|
const startMarker = ' # Synthesize the greeting first';
|
|
122
123
|
const endMarker = ' except Exception as e:';
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
124
|
+
if (!updated.includes(singleTrackMarker)) {
|
|
125
|
+
const start = updated.indexOf(startMarker);
|
|
126
|
+
const end = start >= 0 ? updated.indexOf(endMarker, start) : -1;
|
|
127
|
+
if (start < 0 || end < 0) {
|
|
128
|
+
return {
|
|
129
|
+
patched: false,
|
|
130
|
+
file,
|
|
131
|
+
warning: 'voice_agent.py does not contain the known duplicate greeting-track block; no automatic patch applied',
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
const replacement = [
|
|
135
|
+
' # Use the existing conversation track. A dedicated greeting',
|
|
136
|
+
' # track makes Linux robots compete for the same ALSA device.',
|
|
137
|
+
' import inspect as _inspect',
|
|
138
|
+
' speech = session.say(greeting, allow_interruptions=False)',
|
|
139
|
+
' if _inspect.isawaitable(speech):',
|
|
140
|
+
' await speech',
|
|
141
|
+
" elif hasattr(speech, 'wait_for_playout'):",
|
|
142
|
+
' await speech.wait_for_playout()',
|
|
143
|
+
" logger.info('Initial greeting completed through AgentSession audio track')",
|
|
144
|
+
'',
|
|
145
|
+
].join(newline);
|
|
146
|
+
updated = updated.slice(0, start) + replacement + updated.slice(end);
|
|
147
|
+
patched = true;
|
|
131
148
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
return { patched: true, file };
|
|
149
|
+
// The job is dispatched after the robot participant joins. Waiting for its
|
|
150
|
+
// camera adds camera startup to voice latency even though audio is ready.
|
|
151
|
+
const cameraGate = / await asyncio\.wait_for\(robot_camera_ready\.wait\(\), timeout\s*=\s*[0-9.]+\)/;
|
|
152
|
+
if (cameraGate.test(updated)) {
|
|
153
|
+
updated = updated.replace(cameraGate, [
|
|
154
|
+
' # The robot is already in the room when this job is dispatched.',
|
|
155
|
+
' # Yield once for subscription signalling; do not gate voice on camera.',
|
|
156
|
+
' await asyncio.sleep(0)',
|
|
157
|
+
].join(newline));
|
|
158
|
+
patched = true;
|
|
159
|
+
}
|
|
160
|
+
if (patched)
|
|
161
|
+
writeFileSync(file, updated, 'utf8');
|
|
162
|
+
return { patched, file };
|
|
147
163
|
}
|
|
148
164
|
export async function roboparkAgentUp(opts) {
|
|
149
165
|
console.log(chalk.bold('\n robopark agent up'));
|
|
@@ -135,6 +135,9 @@ export async function roboparkRobotRuntime(opts) {
|
|
|
135
135
|
// Motion remains enabled, but it reads RoboVision's shared stream. Only
|
|
136
136
|
// RoboVision may open the physical V4L2 device.
|
|
137
137
|
LOCAL_CAMERA_MOTION: 'true',
|
|
138
|
+
// The first production sound is the greeting. A beep would occupy the
|
|
139
|
+
// same exclusive ALSA device and delay or clip TTS.
|
|
140
|
+
ROBOPARK_MOTION_CUE: 'false',
|
|
138
141
|
// Fleet hardware profile: resolve volatile ALSA hw:X,Y coordinates
|
|
139
142
|
// from these USB product labels on every inventory refresh.
|
|
140
143
|
ROBOPARK_AUDIO_INPUT_MATCH: 'Usb Audio Device: USB Audio',
|
|
@@ -143,8 +146,14 @@ export async function roboparkRobotRuntime(opts) {
|
|
|
143
146
|
// only while audible robot speech is playing, then hold briefly for
|
|
144
147
|
// room echo decay so VAD cannot interrupt the robot with its own voice.
|
|
145
148
|
ROBOPARK_HALF_DUPLEX: 'true',
|
|
146
|
-
ROBOPARK_ECHO_TAIL_MS: '
|
|
149
|
+
ROBOPARK_ECHO_TAIL_MS: '350',
|
|
147
150
|
ROBOPARK_ECHO_GATE_PEAK: '96',
|
|
151
|
+
// Keep ordinary speaker echo muted but reopen the microphone when a
|
|
152
|
+
// nearby voice rises decisively above the learned acoustic echo floor.
|
|
153
|
+
ROBOPARK_ADAPTIVE_BARGE_IN: 'true',
|
|
154
|
+
ROBOPARK_BARGE_IN_MIN_PEAK: '2200',
|
|
155
|
+
ROBOPARK_BARGE_IN_ECHO_RATIO: '2.4',
|
|
156
|
+
ROBOPARK_BARGE_IN_HOLD_MS: '1400',
|
|
148
157
|
// Inventory changes should reach the Control Center promptly.
|
|
149
158
|
HEARTBEAT_INTERVAL: '5',
|
|
150
159
|
},
|
package/dist/robopark-cli.js
CHANGED
|
@@ -284,6 +284,8 @@ program
|
|
|
284
284
|
.option('--lan', 'generate a robot command for the same local network (default)')
|
|
285
285
|
.option('--tailscale', 'generate a robot command for a Tailscale-reachable hub')
|
|
286
286
|
.option('--start', 'start the robot node on this machine now, instead of printing the one-liner')
|
|
287
|
+
.option('--auto-start', 'register the robot runtime to start on boot')
|
|
288
|
+
.option('--yes', 'non-interactive production setup')
|
|
287
289
|
.action(async (name, opts) => {
|
|
288
290
|
await roboparkAddRobot(config, { name, ...opts });
|
|
289
291
|
});
|
|
@@ -212,3 +212,29 @@ Required result: `playback_started ok`.
|
|
|
212
212
|
- `ROBOPARK_HALF_DUPLEX`, `ROBOPARK_ECHO_TAIL_MS`, and
|
|
213
213
|
`ROBOPARK_ECHO_GATE_PEAK` are explicit runtime controls. Disable half-duplex
|
|
214
214
|
only after deploying real acoustic echo cancellation.
|
|
215
|
+
|
|
216
|
+
## Instant Greeting Contract (2.8.101+)
|
|
217
|
+
|
|
218
|
+
- Motion reporting is asynchronous and never delays session allocation.
|
|
219
|
+
- RoboVision's shared camera stream is not stopped or reopened on a trigger.
|
|
220
|
+
- The old fixed 500 ms trigger delay is removed.
|
|
221
|
+
- The local motion beep is disabled by default because it occupies the same
|
|
222
|
+
exclusive ALSA speaker as TTS. `ROBOPARK_MOTION_CUE=true` is diagnostics only.
|
|
223
|
+
- The voice worker uses the persistent AgentSession audio track and starts the
|
|
224
|
+
greeting after robot join; it does not wait for camera subscription.
|
|
225
|
+
- Network, scheduler, LiveKit, and TTS synthesis still impose real latency, but
|
|
226
|
+
RoboPark adds no intentional serial delay before greeting generation.
|
|
227
|
+
|
|
228
|
+
## Adaptive Interruption Contract (2.8.102+)
|
|
229
|
+
|
|
230
|
+
- The preset greeting remains non-interruptible so ambient startup noise cannot
|
|
231
|
+
cancel it before the first audible frame.
|
|
232
|
+
- Normal responses retain LiveKit AgentSession interruption handling.
|
|
233
|
+
- Robot half-duplex suppression learns the microphone's active speaker-echo
|
|
234
|
+
floor. Three consecutive near-field peaks above both the absolute threshold
|
|
235
|
+
and the adaptive echo ratio reopen microphone PCM for the VAD.
|
|
236
|
+
- The barge-in window remains open for 1.4 seconds, allowing LiveKit to cancel
|
|
237
|
+
TTS; ordinary speaker echo remains suppressed with the existing decay tail.
|
|
238
|
+
- Vision responses inherit the selected voice-stack output language. With an
|
|
239
|
+
automatic profile, the original user transcript language is authoritative,
|
|
240
|
+
including direct terminal vision-tool responses.
|
package/package.json
CHANGED
|
@@ -3192,6 +3192,93 @@ PIPELINE_STAGES = (
|
|
|
3192
3192
|
)
|
|
3193
3193
|
PIPELINE_STATUSES = {"pending", "running", "ok", "failed", "blocked", "skipped"}
|
|
3194
3194
|
|
|
3195
|
+
PIPELINE_STAGE_LABELS = {
|
|
3196
|
+
"robot_online": "Robot online",
|
|
3197
|
+
"camera_ready": "Camera detected",
|
|
3198
|
+
"microphone_ready": "Microphone detected",
|
|
3199
|
+
"speaker_ready": "Speaker detected",
|
|
3200
|
+
"motion_detected": "Waiting for motion",
|
|
3201
|
+
"scheduler_session": "Allocating session",
|
|
3202
|
+
"livekit_join": "Joining LiveKit",
|
|
3203
|
+
"camera_published": "Publishing camera",
|
|
3204
|
+
"microphone_published": "Publishing microphone",
|
|
3205
|
+
"voice_worker": "Starting voice worker",
|
|
3206
|
+
"stt_listening": "Listening to visitor",
|
|
3207
|
+
"llm_response": "Preparing response",
|
|
3208
|
+
"tts_subscribed": "Preparing robot voice",
|
|
3209
|
+
"playback_started": "Speaking through robot",
|
|
3210
|
+
"motor_sequence": "Running motor sequence",
|
|
3211
|
+
"session_ended": "Closing session",
|
|
3212
|
+
}
|
|
3213
|
+
|
|
3214
|
+
|
|
3215
|
+
def _pipeline_current_state(events: list[dict], *, active: bool,
|
|
3216
|
+
production_mode: bool, online: bool) -> dict:
|
|
3217
|
+
"""Derive one operator-facing live stage from persistent pipeline events."""
|
|
3218
|
+
if not online:
|
|
3219
|
+
return {
|
|
3220
|
+
"stage": "robot_online", "status": "blocked",
|
|
3221
|
+
"label": PIPELINE_STAGE_LABELS["robot_online"],
|
|
3222
|
+
"message": "Robot heartbeat is stale or missing", "since": None,
|
|
3223
|
+
}
|
|
3224
|
+
if not active:
|
|
3225
|
+
return {
|
|
3226
|
+
"stage": "motion_detected",
|
|
3227
|
+
"status": "waiting" if production_mode else "paused",
|
|
3228
|
+
"label": "Armed - waiting for motion" if production_mode else "Production paused",
|
|
3229
|
+
"message": "Motion detection is armed" if production_mode else "Enable production mode to arm this robot",
|
|
3230
|
+
"since": None,
|
|
3231
|
+
}
|
|
3232
|
+
|
|
3233
|
+
scoped = [event for event in events if event.get("session_id")]
|
|
3234
|
+
for event in reversed(scoped):
|
|
3235
|
+
if event.get("status") in {"failed", "blocked"}:
|
|
3236
|
+
return {
|
|
3237
|
+
"stage": event["stage"], "status": event["status"],
|
|
3238
|
+
"label": PIPELINE_STAGE_LABELS.get(event["stage"], event["stage"]),
|
|
3239
|
+
"message": event.get("message") or "Pipeline requires attention",
|
|
3240
|
+
"since": event.get("timestamp"),
|
|
3241
|
+
}
|
|
3242
|
+
|
|
3243
|
+
conversational = {
|
|
3244
|
+
"voice_worker", "stt_listening", "llm_response", "tts_subscribed",
|
|
3245
|
+
"playback_started", "motor_sequence",
|
|
3246
|
+
}
|
|
3247
|
+
latest = next((event for event in reversed(scoped) if event.get("stage") in conversational), None)
|
|
3248
|
+
if latest:
|
|
3249
|
+
stage = latest["stage"]
|
|
3250
|
+
status = latest.get("status") or "running"
|
|
3251
|
+
if status in {"ok", "skipped"}:
|
|
3252
|
+
stage = {
|
|
3253
|
+
"voice_worker": "stt_listening",
|
|
3254
|
+
"stt_listening": "llm_response",
|
|
3255
|
+
"llm_response": "tts_subscribed",
|
|
3256
|
+
"tts_subscribed": "playback_started",
|
|
3257
|
+
"playback_started": "stt_listening",
|
|
3258
|
+
"motor_sequence": "stt_listening",
|
|
3259
|
+
}.get(stage, stage)
|
|
3260
|
+
status = "active"
|
|
3261
|
+
return {
|
|
3262
|
+
"stage": stage, "status": status,
|
|
3263
|
+
"label": PIPELINE_STAGE_LABELS.get(stage, stage),
|
|
3264
|
+
"message": latest.get("message") or PIPELINE_STAGE_LABELS.get(stage, stage),
|
|
3265
|
+
"since": latest.get("timestamp"),
|
|
3266
|
+
}
|
|
3267
|
+
|
|
3268
|
+
startup = (
|
|
3269
|
+
"scheduler_session", "livekit_join", "camera_published",
|
|
3270
|
+
"microphone_published", "voice_worker",
|
|
3271
|
+
)
|
|
3272
|
+
completed = {event.get("stage") for event in scoped if event.get("status") in {"ok", "skipped"}}
|
|
3273
|
+
stage = next((candidate for candidate in startup if candidate not in completed), "voice_worker")
|
|
3274
|
+
latest_startup = next((event for event in reversed(scoped) if event.get("stage") in startup), None)
|
|
3275
|
+
return {
|
|
3276
|
+
"stage": stage, "status": "active",
|
|
3277
|
+
"label": PIPELINE_STAGE_LABELS.get(stage, stage),
|
|
3278
|
+
"message": (latest_startup or {}).get("message") or "Production conversation is starting",
|
|
3279
|
+
"since": (latest_startup or {}).get("timestamp"),
|
|
3280
|
+
}
|
|
3281
|
+
|
|
3195
3282
|
async def _insert_pipeline_event(device_id: str, payload: PipelineEventPayload) -> dict:
|
|
3196
3283
|
if payload.stage not in PIPELINE_STAGES:
|
|
3197
3284
|
raise HTTPException(422, f"Unknown pipeline stage: {payload.stage}")
|
|
@@ -3286,16 +3373,29 @@ async def robot_pipeline_status(robot_id: str, limit: int = 80):
|
|
|
3286
3373
|
blockers.append("Microphone inventory has not been reported")
|
|
3287
3374
|
if not prereqs["speaker_ready"]:
|
|
3288
3375
|
blockers.append("Speaker inventory has not been reported")
|
|
3376
|
+
active = bool(latest_session and not latest_session["ended_at"])
|
|
3377
|
+
active_session_id = latest_session["id"] if active else None
|
|
3378
|
+
current_events = [
|
|
3379
|
+
event for event in events
|
|
3380
|
+
if not active_session_id or event.get("session_id") == active_session_id
|
|
3381
|
+
]
|
|
3382
|
+
current_stage = _pipeline_current_state(
|
|
3383
|
+
current_events,
|
|
3384
|
+
active=active,
|
|
3385
|
+
production_mode=bool(device["production_mode"]),
|
|
3386
|
+
online=prereqs["robot_online"],
|
|
3387
|
+
)
|
|
3289
3388
|
return {
|
|
3290
3389
|
"robot_id": device_id,
|
|
3291
3390
|
"robot_name": device["name"],
|
|
3292
|
-
"overall": "blocked" if blockers else ("running" if
|
|
3391
|
+
"overall": "blocked" if blockers else ("running" if active else "ready"),
|
|
3293
3392
|
"blockers": list(dict.fromkeys(blockers)),
|
|
3294
3393
|
"prerequisites": prereqs,
|
|
3295
3394
|
"telemetry": telemetry,
|
|
3296
3395
|
"latest_session": dict(latest_session) if latest_session else None,
|
|
3297
3396
|
"events": events,
|
|
3298
3397
|
"stages": list(PIPELINE_STAGES),
|
|
3398
|
+
"current_stage": current_stage,
|
|
3299
3399
|
}
|
|
3300
3400
|
|
|
3301
3401
|
@app.get("/api/robots/{robot_id}/pipeline-history")
|
|
@@ -5019,6 +5119,53 @@ async def delete_character_preset(preset_id: str):
|
|
|
5019
5119
|
return {"ok": True}
|
|
5020
5120
|
|
|
5021
5121
|
|
|
5122
|
+
class PresetPosition(BaseModel):
|
|
5123
|
+
id: str
|
|
5124
|
+
nx: float
|
|
5125
|
+
ny: float
|
|
5126
|
+
|
|
5127
|
+
|
|
5128
|
+
class PresetPositionBatch(BaseModel):
|
|
5129
|
+
positions: List[PresetPosition]
|
|
5130
|
+
|
|
5131
|
+
|
|
5132
|
+
@app.put("/api/character-presets/positions")
|
|
5133
|
+
async def update_preset_positions(payload: PresetPositionBatch):
|
|
5134
|
+
"""Batch-update (id, nx, ny) for the Park map. nx/ny are the
|
|
5135
|
+
normalized [0,1] pad coordinates the dashboard uses to place each
|
|
5136
|
+
robot on the iso park. Bounded + persisted so an operator can
|
|
5137
|
+
Ctrl+drag a robot on the Park map and have its position survive
|
|
5138
|
+
a refresh, server restart, or robot reconnect.
|
|
5139
|
+
|
|
5140
|
+
Designed for small, infrequent updates (a single drag releases
|
|
5141
|
+
one robot, plus an occasional "reset all" tool). The endpoint
|
|
5142
|
+
intentionally ignores all other preset fields — only the
|
|
5143
|
+
coordinates move; voice stack, name, system prompt, etc. are
|
|
5144
|
+
owned by the character-preset editor and are not touched here.
|
|
5145
|
+
"""
|
|
5146
|
+
if not payload.positions:
|
|
5147
|
+
return {"updated": 0}
|
|
5148
|
+
now = datetime.utcnow().isoformat()
|
|
5149
|
+
updated = 0
|
|
5150
|
+
async with aiosqlite.connect(DB_PATH) as db:
|
|
5151
|
+
for p in payload.positions:
|
|
5152
|
+
if not p.id:
|
|
5153
|
+
continue
|
|
5154
|
+
# Clamp nx/ny to the visible pad rectangle so a stray
|
|
5155
|
+
# drag can't park a robot off the island. A small inner
|
|
5156
|
+
# margin (0.04) keeps the avatar fully on the grass and
|
|
5157
|
+
# clear of the gold wall on the perimeter.
|
|
5158
|
+
nx = max(0.04, min(0.96, float(p.nx)))
|
|
5159
|
+
ny = max(0.04, min(0.96, float(p.ny)))
|
|
5160
|
+
cur = await db.execute(
|
|
5161
|
+
"UPDATE character_presets SET nx = ?, ny = ?, updated_at = ? WHERE id = ?",
|
|
5162
|
+
(nx, ny, now, p.id),
|
|
5163
|
+
)
|
|
5164
|
+
updated += cur.rowcount or 0
|
|
5165
|
+
await db.commit()
|
|
5166
|
+
return {"updated": updated}
|
|
5167
|
+
|
|
5168
|
+
|
|
5022
5169
|
class RobotVoiceConfigUpdate(BaseModel):
|
|
5023
5170
|
character_preset_id: Optional[str] = None
|
|
5024
5171
|
voice_stack_id: Optional[str] = None
|
|
@@ -5123,13 +5270,20 @@ async def _robot_telemetry(db, robot_id: str) -> Optional[dict]:
|
|
|
5123
5270
|
|
|
5124
5271
|
device_inventory = parse_json_field(device["device_inventory"], {}) if device else {}
|
|
5125
5272
|
supervisor_status = parse_json_field(device["supervisor_status"], []) if device else []
|
|
5126
|
-
session = None
|
|
5127
|
-
|
|
5273
|
+
session = None
|
|
5274
|
+
current_pipeline_events = []
|
|
5275
|
+
if robot["current_session_id"]:
|
|
5128
5276
|
async with db.execute(
|
|
5129
5277
|
"SELECT started_at, last_activity_at, ended_at FROM sessions WHERE id = ?",
|
|
5130
5278
|
(robot["current_session_id"],),
|
|
5131
|
-
) as c:
|
|
5132
|
-
session = await c.fetchone()
|
|
5279
|
+
) as c:
|
|
5280
|
+
session = await c.fetchone()
|
|
5281
|
+
async with db.execute(
|
|
5282
|
+
"SELECT * FROM pipeline_events WHERE robot_id = ? AND session_id = ? "
|
|
5283
|
+
"ORDER BY timestamp, id",
|
|
5284
|
+
(robot_id, robot["current_session_id"]),
|
|
5285
|
+
) as c:
|
|
5286
|
+
current_pipeline_events = [dict(row) for row in await c.fetchall()]
|
|
5133
5287
|
|
|
5134
5288
|
now = datetime.utcnow()
|
|
5135
5289
|
session_age = None
|
|
@@ -5212,8 +5366,14 @@ async def _robot_telemetry(db, robot_id: str) -> Optional[dict]:
|
|
|
5212
5366
|
(robot_id,),
|
|
5213
5367
|
) as c:
|
|
5214
5368
|
end_reasons = {r["end_reason"]: r["n"] for r in await c.fetchall()}
|
|
5215
|
-
drops = sum(n for reason, n in end_reasons.items() if reason in ABNORMAL_END_REASONS)
|
|
5216
|
-
drop_rate = round(drops / sessions_total, 4) if sessions_total else 0.0
|
|
5369
|
+
drops = sum(n for reason, n in end_reasons.items() if reason in ABNORMAL_END_REASONS)
|
|
5370
|
+
drop_rate = round(drops / sessions_total, 4) if sessions_total else 0.0
|
|
5371
|
+
current_stage = _pipeline_current_state(
|
|
5372
|
+
current_pipeline_events,
|
|
5373
|
+
active=bool(robot["current_session_id"] and session and not session["ended_at"]),
|
|
5374
|
+
production_mode=bool(device["production_mode"]) if device else False,
|
|
5375
|
+
online=heartbeat_age is not None and heartbeat_age <= 30,
|
|
5376
|
+
)
|
|
5217
5377
|
|
|
5218
5378
|
return {
|
|
5219
5379
|
"robot_id": robot_id,
|
|
@@ -5244,9 +5404,13 @@ async def _robot_telemetry(db, robot_id: str) -> Optional[dict]:
|
|
|
5244
5404
|
"supervisor_status": supervisor_status,
|
|
5245
5405
|
"supervisor_status_at": device["supervisor_status_at"] if device else None,
|
|
5246
5406
|
"production_mode": bool(device["production_mode"]) if device else False,
|
|
5247
|
-
"readiness_code": readiness_code,
|
|
5248
|
-
"readiness_message": readiness_message,
|
|
5249
|
-
|
|
5407
|
+
"readiness_code": readiness_code,
|
|
5408
|
+
"readiness_message": readiness_message,
|
|
5409
|
+
"current_stage": current_stage,
|
|
5410
|
+
"current_stage_id": current_stage["stage"],
|
|
5411
|
+
"current_stage_label": current_stage["label"],
|
|
5412
|
+
"current_stage_status": current_stage["status"],
|
|
5413
|
+
}
|
|
5250
5414
|
|
|
5251
5415
|
@app.get("/api/robots/{robot_id}/telemetry")
|
|
5252
5416
|
async def robot_telemetry(robot_id: str):
|
|
@@ -541,12 +541,17 @@ class PreviewAgent:
|
|
|
541
541
|
self.vision_trigger_cooldown = float(cfg.get("vision_trigger_cooldown", os.getenv("VISION_TRIGGER_COOLDOWN", DEFAULT_VISION_TRIGGER_COOLDOWN)))
|
|
542
542
|
self.vision_session_seconds = float(cfg.get("vision_session_seconds", os.getenv("VISION_SESSION_SECONDS", DEFAULT_VISION_SESSION_SECONDS)))
|
|
543
543
|
self.vision_silence_timeout = float(cfg.get("vision_silence_timeout", os.getenv("VISION_SILENCE_TIMEOUT", DEFAULT_VISION_SILENCE_TIMEOUT)))
|
|
544
|
-
self.local_camera_motion = str(
|
|
544
|
+
self.local_camera_motion = str(
|
|
545
545
|
# Production needs an always-on detector. The preview publisher
|
|
546
546
|
# remains the single camera owner when a session starts, so this
|
|
547
547
|
# does not require the separate OpenCV vision server.
|
|
548
548
|
cfg.get("local_camera_motion", os.getenv("LOCAL_CAMERA_MOTION", "true"))
|
|
549
|
-
).lower() in ("1", "true", "yes", "on")
|
|
549
|
+
).lower() in ("1", "true", "yes", "on")
|
|
550
|
+
# The character greeting is the production acknowledgement. A local
|
|
551
|
+
# motion beep serializes on the same ALSA output and delays allocation.
|
|
552
|
+
self.motion_cue_enabled = str(
|
|
553
|
+
cfg.get("motion_cue_enabled", os.getenv("ROBOPARK_MOTION_CUE", "false"))
|
|
554
|
+
).lower() in ("1", "true", "yes", "on")
|
|
550
555
|
|
|
551
556
|
self._shutdown = asyncio.Event()
|
|
552
557
|
self._task: Optional[asyncio.Task] = None
|
|
@@ -1178,20 +1183,19 @@ class PreviewAgent:
|
|
|
1178
1183
|
logger.warning("vision motion event received but not enrolled yet — ignoring")
|
|
1179
1184
|
return
|
|
1180
1185
|
self._vision_trigger_in_flight = True
|
|
1181
|
-
|
|
1186
|
+
# Observability must not add a scheduler round trip before the actual
|
|
1187
|
+
# session request. The reporter handles and logs its own errors.
|
|
1188
|
+
asyncio.create_task(self._report_pipeline(
|
|
1189
|
+
"motion_detected", "ok",
|
|
1190
|
+
f"Motion received from {payload.get('source', 'camera')}", once=False,
|
|
1191
|
+
))
|
|
1182
1192
|
logger.info("motion detected by RoboVisionAI_PI — requesting a session")
|
|
1183
|
-
#
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
# I/O. Camera/mic join and the preset greeting are the critical path.
|
|
1190
|
-
# The cue and TTS share one physical ALSA output. Finish and release
|
|
1191
|
-
# the cue before LiveKit can deliver the greeting; background playback
|
|
1192
|
-
# races aplay and makes greeting delivery intermittently fail busy.
|
|
1193
|
-
async with self._speaker_operation_lock:
|
|
1194
|
-
await asyncio.to_thread(_play_audio_effect, self.audio_playback_device, "motion")
|
|
1193
|
+
# RoboVision owns the physical camera and exposes a shared stream, so
|
|
1194
|
+
# there is no capture handle to tear down and no reason to sleep here.
|
|
1195
|
+
if self.motion_cue_enabled:
|
|
1196
|
+
logger.warning("ROBOPARK_MOTION_CUE is enabled; the diagnostic cue adds greeting latency")
|
|
1197
|
+
async with self._speaker_operation_lock:
|
|
1198
|
+
await asyncio.to_thread(_play_audio_effect, self.audio_playback_device, "motion")
|
|
1195
1199
|
try:
|
|
1196
1200
|
r = await self._session.post(
|
|
1197
1201
|
f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/request-session",
|
|
@@ -1294,8 +1298,24 @@ class LiveKitPublisher:
|
|
|
1294
1298
|
)
|
|
1295
1299
|
self._speaker_playback_active = threading.Event()
|
|
1296
1300
|
self._speaker_gate_until = 0.0
|
|
1301
|
+
self._speaker_started_at = 0.0
|
|
1302
|
+
self._barge_in_until = 0.0
|
|
1303
|
+
self._echo_mic_floor = 0.0
|
|
1304
|
+
self._barge_in_candidate_frames = 0
|
|
1297
1305
|
self._speaker_echo_tail = max(
|
|
1298
|
-
0.1, min(float(os.getenv("ROBOPARK_ECHO_TAIL_MS", "
|
|
1306
|
+
0.1, min(float(os.getenv("ROBOPARK_ECHO_TAIL_MS", "350")) / 1000.0, 2.0)
|
|
1307
|
+
)
|
|
1308
|
+
self._adaptive_barge_in = str(
|
|
1309
|
+
os.getenv("ROBOPARK_ADAPTIVE_BARGE_IN", "true")
|
|
1310
|
+
).lower() in ("1", "true", "yes", "on")
|
|
1311
|
+
self._barge_in_min_peak = max(
|
|
1312
|
+
256, min(int(os.getenv("ROBOPARK_BARGE_IN_MIN_PEAK", "2200")), 20000)
|
|
1313
|
+
)
|
|
1314
|
+
self._barge_in_ratio = max(
|
|
1315
|
+
1.25, min(float(os.getenv("ROBOPARK_BARGE_IN_ECHO_RATIO", "2.4")), 8.0)
|
|
1316
|
+
)
|
|
1317
|
+
self._barge_in_hold = max(
|
|
1318
|
+
0.4, min(float(os.getenv("ROBOPARK_BARGE_IN_HOLD_MS", "1400")) / 1000.0, 3.0)
|
|
1299
1319
|
)
|
|
1300
1320
|
# Motion sampling state belongs to the publisher instance. Keeping it
|
|
1301
1321
|
# initialized here prevents shutdown/reopen paths from raising while
|
|
@@ -1533,7 +1553,7 @@ class LiveKitPublisher:
|
|
|
1533
1553
|
import threading
|
|
1534
1554
|
write_queue: "queue.Queue[Optional[bytes]]" = queue.Queue()
|
|
1535
1555
|
written_frames = [0]
|
|
1536
|
-
PREBUFFER_CHUNKS =
|
|
1556
|
+
PREBUFFER_CHUNKS = 3
|
|
1537
1557
|
stream_failed = threading.Event()
|
|
1538
1558
|
playback_reported = threading.Event()
|
|
1539
1559
|
event_loop = asyncio.get_running_loop()
|
|
@@ -1553,6 +1573,10 @@ class LiveKitPublisher:
|
|
|
1553
1573
|
def _open_echo_gate() -> None:
|
|
1554
1574
|
if not self._half_duplex:
|
|
1555
1575
|
return
|
|
1576
|
+
if not self._speaker_playback_active.is_set():
|
|
1577
|
+
self._speaker_started_at = time.monotonic()
|
|
1578
|
+
self._echo_mic_floor = 0.0
|
|
1579
|
+
self._barge_in_candidate_frames = 0
|
|
1556
1580
|
self._speaker_playback_active.set()
|
|
1557
1581
|
self._speaker_gate_until = time.monotonic() + self._speaker_echo_tail
|
|
1558
1582
|
|
|
@@ -1841,11 +1865,40 @@ class LiveKitPublisher:
|
|
|
1841
1865
|
_pub_start = time.monotonic()
|
|
1842
1866
|
for off in range(0, len(data) - bytes_per_frame + 1, bytes_per_frame):
|
|
1843
1867
|
chunk = data[off:off + bytes_per_frame]
|
|
1868
|
+
now = time.monotonic()
|
|
1844
1869
|
echo_gate_active = self._half_duplex and (
|
|
1845
1870
|
self._speaker_playback_active.is_set()
|
|
1846
|
-
or
|
|
1871
|
+
or now < self._speaker_gate_until
|
|
1847
1872
|
)
|
|
1848
|
-
|
|
1873
|
+
# Learn the microphone's speaker-echo floor while output is
|
|
1874
|
+
# active. A nearby visitor speaking produces a fast peak well
|
|
1875
|
+
# above that floor; reopen the mic briefly so LiveKit VAD can
|
|
1876
|
+
# cancel normal TTS. The initial 300 ms remains protected.
|
|
1877
|
+
_, raw_peak = _pcm16_scale_and_peak(chunk, 1.0)
|
|
1878
|
+
if echo_gate_active and self._adaptive_barge_in:
|
|
1879
|
+
if self._echo_mic_floor <= 0:
|
|
1880
|
+
self._echo_mic_floor = float(raw_peak)
|
|
1881
|
+
else:
|
|
1882
|
+
self._echo_mic_floor = self._echo_mic_floor * 0.92 + raw_peak * 0.08
|
|
1883
|
+
threshold = max(
|
|
1884
|
+
self._barge_in_min_peak,
|
|
1885
|
+
int(self._echo_mic_floor * self._barge_in_ratio),
|
|
1886
|
+
)
|
|
1887
|
+
warmed = now - self._speaker_started_at >= 0.3
|
|
1888
|
+
if warmed and raw_peak >= threshold:
|
|
1889
|
+
self._barge_in_candidate_frames += 1
|
|
1890
|
+
else:
|
|
1891
|
+
self._barge_in_candidate_frames = 0
|
|
1892
|
+
if self._barge_in_candidate_frames >= 3:
|
|
1893
|
+
if now >= self._barge_in_until:
|
|
1894
|
+
logger.info(
|
|
1895
|
+
"audio_loop: adaptive barge-in opened mic "
|
|
1896
|
+
"(peak=%d threshold=%d echo_floor=%d)",
|
|
1897
|
+
raw_peak, threshold, int(self._echo_mic_floor),
|
|
1898
|
+
)
|
|
1899
|
+
self._barge_in_until = now + self._barge_in_hold
|
|
1900
|
+
barge_in_active = self._adaptive_barge_in and now < self._barge_in_until
|
|
1901
|
+
if echo_gate_active and not barge_in_active:
|
|
1849
1902
|
# Preserve 20 ms frame cadence; only suppress content.
|
|
1850
1903
|
# Stopping publication would create gaps and destabilize
|
|
1851
1904
|
# VAD/endpointing when listening resumes.
|