omnius 1.0.405 → 1.0.406

package/dist/index.js

@@ -555840,13 +555840,24 @@ var init_live_media_loop = __esm({
           for (const face of result.faces.slice(0, 8)) {
             try {
               const rec = await visualMemory.execute({
-                action: "recognize",
+                action: "identify",
                 image: face.crop_path,
                 auto_setup: false,
                 timeout_ms: 15e3,
                 format: "json"
               });
               face.identity_candidates = rec.success ? String(rec.llmContent || rec.output || "").slice(0, 1200) : rec.error;
+              if (rec.success) {
+                try {
+                  const parsed = JSON.parse(String(rec.llmContent || rec.output || "{}"));
+                  const identified = parsed.faces?.find((candidate) => candidate.identified && candidate.name);
+                  if (identified?.name) {
+                    face.identity = identified.name;
+                    face.confidence = identified.confidence ?? face.confidence;
+                  }
+                } catch {
+                }
+              }
             } catch (err) {
               face.identity_candidates = err instanceof Error ? err.message : String(err);
             }
@@ -601917,6 +601928,166 @@ function nowIso3(ms) {
 function cleanPreview(value2, maxChars) {
   return String(value2 ?? "").replace(/\r/g, "").split("\n").map((line) => line.trimEnd()).filter((line) => line.trim().length > 0).join("\n").slice(0, maxChars);
 }
+function boundedText(value2, maxChars) {
+  return cleanPreview(value2, maxChars).replace(/\n{3,}/g, "\n\n");
+}
+function eventAge(now2, observedAt) {
+  return `${Math.max(0, Math.round((now2 - observedAt) / 1e3))}s`;
+}
+function stableEventId(kind, source, observedAt, suffix) {
+  const compact4 = suffix.replace(/[^a-z0-9_.:-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "event";
+  return `${kind}:${source}:${Math.round(observedAt)}:${compact4}`;
+}
+function stablePersonId(source, observedAt, suffix) {
+  const compact4 = suffix.replace(/[^a-z0-9_.:-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "person";
+  return `person:${source}:${Math.round(observedAt)}:${compact4}`;
+}
+function mergeRecentById(existing, incoming, limit) {
+  const merged = /* @__PURE__ */ new Map();
+  for (const item of [...existing ?? [], ...incoming]) merged.set(item.id, item);
+  return [...merged.values()].sort((a2, b) => b.observedAt - a2.observedAt).slice(0, limit);
+}
+function parseJsonObject3(value2) {
+  if (typeof value2 !== "string" || !value2.trim()) return null;
+  try {
+    const parsed = JSON.parse(value2);
+    return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+function parseLiveMediaPayload(value2) {
+  const parsed = parseJsonObject3(value2);
+  if (!parsed) return null;
+  if (!Array.isArray(parsed["objects"]) && !Array.isArray(parsed["tracks"]) && !Array.isArray(parsed["faces"])) return null;
+  return parsed;
+}
+function observedAtFromOffset(sampledAt, payload, offsetSec) {
+  const duration = Math.max(0, Number(payload.duration_observed_sec ?? 0));
+  const offset = Math.max(0, Number(offsetSec ?? duration));
+  if (!duration) return sampledAt;
+  return Math.round(sampledAt - Math.max(0, duration - offset) * 1e3);
+}
+function parseFaceIdentity(face) {
+  if (typeof face.identity === "string" && face.identity.trim()) {
+    return {
+      status: "known",
+      name: face.identity.trim(),
+      confidence: typeof face.confidence === "number" ? face.confidence : void 0,
+      evidence: "identity field from live media loop"
+    };
+  }
+  const raw = face.identity_candidates;
+  const parsed = typeof raw === "string" ? parseJsonObject3(raw) : typeof raw === "object" && raw !== null ? raw : null;
+  const faces = Array.isArray(parsed?.["faces"]) ? parsed["faces"] : [];
+  const identified = faces.find((candidate) => candidate["identified"] === true && typeof candidate["name"] === "string" && candidate["name"]);
+  if (identified) {
+    return {
+      status: "known",
+      name: String(identified["name"]).trim(),
+      confidence: typeof identified["confidence"] === "number" ? identified["confidence"] : void 0,
+      evidence: "visual_memory identify match"
+    };
+  }
+  const best = faces.find((candidate) => typeof candidate["confidence"] === "number");
+  return {
+    status: "unknown",
+    confidence: typeof best?.["confidence"] === "number" ? best["confidence"] : typeof face.confidence === "number" ? face.confidence : void 0,
+    evidence: typeof raw === "string" && raw.trim() ? boundedText(raw, 500) : "no known visual-memory face match"
+  };
+}
+function observationsFromLiveMediaPayload(payload, sampledAt, fallbackSource) {
+  if (!payload) return { events: [], people: [] };
+  const source = String(payload.source || fallbackSource || "camera");
+  const events = [];
+  const people = [];
+  for (const track of (payload.tracks ?? []).slice(0, 16)) {
+    const label = String(track.label || "object");
+    const observedAt = observedAtFromOffset(sampledAt, payload, Number(track.last_seen_sec ?? track.first_seen_sec ?? 0));
+    const trackId = track.track_id === void 0 || track.track_id === null ? "" : String(track.track_id);
+    events.push({
+      id: stableEventId("camera_track", source, observedAt, `${label}:${trackId}`),
+      kind: "camera_track",
+      observedAt,
+      source,
+      title: `Tracked ${label}`,
+      summary: `track=${trackId || "none"} first=${Number(track.first_seen_sec ?? 0).toFixed(1)}s last=${Number(track.last_seen_sec ?? 0).toFixed(1)}s observations=${Number(track.observations ?? 0)} mean_conf=${Number(track.mean_confidence ?? 0).toFixed(2)}`,
+      confidence: typeof track.mean_confidence === "number" ? track.mean_confidence : void 0,
+      trackId
+    });
+  }
+  for (const object of (payload.objects ?? []).slice(0, 24)) {
+    const label = String(object.label || "object");
+    const observedAt = observedAtFromOffset(sampledAt, payload, Number(object.timestamp_sec ?? 0));
+    const trackId = object.track_id === void 0 || object.track_id === null ? "" : String(object.track_id);
+    events.push({
+      id: stableEventId("camera_object", source, observedAt, `${label}:${trackId}:${object.frame_index ?? ""}`),
+      kind: "camera_object",
+      observedAt,
+      source,
+      title: `Detected ${label}`,
+      summary: `frame=${object.frame_index ?? "?"} t=${Number(object.timestamp_sec ?? 0).toFixed(1)}s conf=${Number(object.confidence ?? 0).toFixed(2)} track=${trackId || "none"} bbox=${JSON.stringify(object.bbox_xyxy ?? [])}`,
+      confidence: typeof object.confidence === "number" ? object.confidence : void 0,
+      trackId
+    });
+  }
+  for (const face of (payload.faces ?? []).slice(0, 16)) {
+    const observedAt = observedAtFromOffset(sampledAt, payload, Number(face.timestamp_sec ?? 0));
+    const identity3 = parseFaceIdentity(face);
+    const cropPath = typeof face.crop_path === "string" ? face.crop_path : void 0;
+    events.push({
+      id: stableEventId("camera_face", source, observedAt, cropPath || `${face.frame_index ?? ""}:${face.timestamp_sec ?? ""}`),
+      kind: "camera_face",
+      observedAt,
+      source,
+      title: identity3.status === "known" && identity3.name ? `Observed known face: ${identity3.name}` : "Observed unknown face",
+      summary: `frame=${face.frame_index ?? "?"} t=${Number(face.timestamp_sec ?? 0).toFixed(1)}s conf=${Number(identity3.confidence ?? face.confidence ?? 0).toFixed(2)} crop=${cropPath ?? "none"} bbox=${JSON.stringify(face.bbox_xyxy ?? [])}`,
+      confidence: identity3.confidence ?? face.confidence,
+      evidencePath: cropPath
+    });
+    people.push({
+      id: stablePersonId(source, observedAt, cropPath || `${face.frame_index ?? ""}:${face.timestamp_sec ?? ""}`),
+      observedAt,
+      source,
+      status: identity3.status,
+      name: identity3.name,
+      confidence: identity3.confidence ?? face.confidence,
+      cropPath,
+      evidence: identity3.evidence
+    });
+  }
+  const faceCount = people.length;
+  if (faceCount === 0) {
+    const personTracks = (payload.tracks ?? []).filter((track) => String(track.label || "").toLowerCase() === "person").slice(0, 8);
+    for (const track of personTracks) {
+      const observedAt = observedAtFromOffset(sampledAt, payload, Number(track.last_seen_sec ?? track.first_seen_sec ?? 0));
+      const trackId = track.track_id === void 0 || track.track_id === null ? "" : String(track.track_id);
+      people.push({
+        id: stablePersonId(source, observedAt, `track:${trackId || track.label || "person"}`),
+        observedAt,
+        source,
+        status: "unknown",
+        confidence: typeof track.mean_confidence === "number" ? track.mean_confidence : void 0,
+        trackId,
+        evidence: "person-class object track; no face crop was available"
+      });
+    }
+  }
+  for (const hit of (payload.trigger_hits ?? []).slice(0, 8)) {
+    const observedAt = sampledAt;
+    const name10 = String(hit.triggerName || hit.matched || "visual trigger");
+    events.push({
+      id: stableEventId("visual_trigger", source, observedAt, name10),
+      kind: "visual_trigger",
+      observedAt,
+      source,
+      title: `Visual trigger: ${name10}`,
+      summary: `matched=${String(hit.matched ?? "")} conf=${Number(hit.confidence ?? 0).toFixed(2)}`,
+      confidence: typeof hit.confidence === "number" ? hit.confidence : void 0
+    });
+  }
+  return { events, people };
+}
 function dedupeDevices(devices) {
   const seen = /* @__PURE__ */ new Set();
   const out = [];
@@ -602033,16 +602204,60 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
   const activeAudio = snapshot.config.audioEnabled || snapshot.config.audioOutputEnabled || Boolean(snapshot.audio);
   if (!activeVideo && !activeAudio) return "";
   lines.push("<live-sensor-context>");
+  lines.push("## LIVE SENSOR STATUS");
   lines.push("Live sensor context is operator-enabled ambient input. Treat it as current perceptual evidence for this turn; if it is stale, missing, or insufficient, use camera/audio tools to refresh before answering what you see or hear.");
+  lines.push("Identity rule: never invent names for observed people. If a person is unknown and identity matters, ask the user or the person for their name, then store the association with visual_memory(action='enroll', image='<face crop>', name='<name>') when a face crop is available, or multimodal_memory(action='meet', person_name='<name>') when live voice/name context is available.");
   lines.push(`Snapshot file: ${liveSnapshotPath(snapshot.repoRoot)}`);
   lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
   if (snapshot.config.selectedCamera) lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
   if (snapshot.config.selectedAudioInput) lines.push(`Selected audio input: ${snapshot.config.selectedAudioInput}`);
   if (snapshot.config.selectedAudioOutput) lines.push(`Selected audio output: ${snapshot.config.selectedAudioOutput}`);
+  const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
+  if (activeVideo) {
+    lines.push("");
+    lines.push("## LIVE CAMERA EVENTS");
+    if (cameraEvents.length === 0) {
+      lines.push("No structured camera events have been captured yet in this live session.");
+    } else {
+      for (const event of cameraEvents) {
+        const stale = now2 - event.observedAt > maxAgeMs ? " STALE" : "";
+        lines.push(`### ${event.title}`);
+        lines.push(`timestamp: ${nowIso3(event.observedAt)} age=${eventAge(now2, event.observedAt)} source=${event.source} kind=${event.kind}${stale}`);
+        if (event.confidence !== void 0) lines.push(`confidence: ${event.confidence.toFixed(2)}`);
+        if (event.trackId) lines.push(`track_id: ${event.trackId}`);
+        if (event.evidencePath) lines.push(`evidence: ${event.evidencePath}`);
+        lines.push(`summary: ${boundedText(event.summary, 500)}`);
+      }
+    }
+    const people = (snapshot.people ?? []).sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 12);
+    const known = people.filter((person) => person.status === "known");
+    const unknown = people.filter((person) => person.status === "unknown");
+    lines.push("");
+    lines.push("## LIVE INDIVIDUALS");
+    lines.push("Known individuals:");
+    if (known.length === 0) {
+      lines.push("- none currently recognized");
+    } else {
+      for (const person of known) {
+        lines.push(`- ${person.name ?? "known individual"} @ ${nowIso3(person.observedAt)} age=${eventAge(now2, person.observedAt)} source=${person.source}${person.confidence !== void 0 ? ` confidence=${person.confidence.toFixed(2)}` : ""}${person.cropPath ? ` crop=${person.cropPath}` : ""}`);
+      }
+    }
+    lines.push("Unknown individuals:");
+    if (unknown.length === 0) {
+      lines.push("- none currently visible");
+    } else {
+      for (const person of unknown) {
+        lines.push(`- unknown person @ ${nowIso3(person.observedAt)} age=${eventAge(now2, person.observedAt)} source=${person.source}${person.confidence !== void 0 ? ` confidence=${person.confidence.toFixed(2)}` : ""}${person.cropPath ? ` crop=${person.cropPath}` : ""}${person.trackId ? ` track=${person.trackId}` : ""}`);
+        if (person.evidence) lines.push(`  evidence: ${boundedText(person.evidence, 360)}`);
+      }
+    }
+    lines.push("Identity next step: when responding about an unknown individual, say they are unknown; ask for their name if appropriate. After the user/person provides a name, enroll the face crop with visual_memory or run multimodal_memory meet. Do not claim a name unless this section lists a known individual or a tool confirms it.");
+  }
   if (snapshot.video) {
     const age = now2 - snapshot.video.updatedAt;
     lines.push("");
-    lines.push(`[video latest: ${nowIso3(snapshot.video.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s source=${snapshot.video.source}${age > maxAgeMs ? " STALE" : ""}]`);
+    lines.push("## LIVE CAMERA LATEST FRAME");
+    lines.push(`timestamp: ${nowIso3(snapshot.video.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s source=${snapshot.video.source}${age > maxAgeMs ? " STALE" : ""}`);
     if (snapshot.video.error) lines.push(`error: ${snapshot.video.error}`);
     if (snapshot.video.framePath) lines.push(`frame: ${snapshot.video.framePath}`);
     if (snapshot.video.asciiContext) lines.push(snapshot.video.asciiContext);
@@ -602052,9 +602267,22 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
     }
   }
   if (snapshot.audio) {
+    const audioEvents = (snapshot.events ?? []).filter((event) => event.kind === "audio_transcript" || event.kind === "audio_sound").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 8);
+    if (audioEvents.length > 0) {
+      lines.push("");
+      lines.push("## LIVE AUDIO EVENTS");
+      for (const event of audioEvents) {
+        const stale = now2 - event.observedAt > maxAgeMs ? " STALE" : "";
+        lines.push(`### ${event.title}`);
+        lines.push(`timestamp: ${nowIso3(event.observedAt)} age=${eventAge(now2, event.observedAt)} source=${event.source} kind=${event.kind}${stale}`);
+        if (event.evidencePath) lines.push(`evidence: ${event.evidencePath}`);
+        lines.push(`summary: ${boundedText(event.summary, 700)}`);
+      }
+    }
     const age = now2 - snapshot.audio.updatedAt;
     lines.push("");
-    lines.push(`[audio latest: ${nowIso3(snapshot.audio.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s${age > maxAgeMs ? " STALE" : ""}]`);
+    lines.push("## LIVE AUDIO LATEST SAMPLE");
+    lines.push(`timestamp: ${nowIso3(snapshot.audio.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s${age > maxAgeMs ? " STALE" : ""}`);
     if (snapshot.audio.error) lines.push(`error: ${snapshot.audio.error}`);
     if (snapshot.audio.recordingPath) lines.push(`recording: ${snapshot.audio.recordingPath}`);
     if (snapshot.audio.transcript) {
@@ -602296,11 +602524,20 @@ var init_live_sensors = __esm({
         });
         if (!result.success) {
           const error = result.error || result.output || "camera capture failed";
+          const observedAt2 = Date.now();
           this.snapshot.video = {
-            updatedAt: Date.now(),
+            updatedAt: observedAt2,
             source,
             error
           };
+          this.snapshot.events = mergeRecentById(this.snapshot.events, [{
+            id: stableEventId("camera_error", source, observedAt2, error.slice(0, 80)),
+            kind: "camera_error",
+            observedAt: observedAt2,
+            source,
+            title: "Camera capture failed",
+            summary: error
+          }], 50);
           this.persist();
           return { ok: false, message: error };
         }
@@ -602309,13 +602546,23 @@ var init_live_sensors = __esm({
         const displayPath = relative11(this.repoRoot, framePath).startsWith("..") ? framePath : relative11(this.repoRoot, framePath);
         const preview = await buildImageAsciiPreview(framePath, { width: 72, height: 22 });
         const asciiContext = preview ? formatImageAsciiContext(preview, displayPath) : void 0;
+        const observedAt = Date.now();
         this.snapshot.video = {
-          updatedAt: Date.now(),
+          updatedAt: observedAt,
           source,
           framePath,
           displayPath,
           asciiContext
         };
+        this.snapshot.events = mergeRecentById(this.snapshot.events, [{
+          id: stableEventId("camera_frame", source, observedAt, framePath),
+          kind: "camera_frame",
+          observedAt,
+          source,
+          title: "Captured camera frame",
+          summary: `frame=${framePath}${displayPath ? ` display=${displayPath}` : ""}`,
+          evidencePath: framePath
+        }], 50);
         this.persist();
         return {
           ok: true,
@@ -602346,12 +602593,18 @@ var init_live_sensors = __esm({
               audio_mode: "none",
               detect_objects: true,
               track_objects: true,
-              crop_faces: true
+              crop_faces: true,
+              recognize_faces: true
             });
+            const sampledAt = Date.now();
             inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
+            const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
+            const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
+            this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
+            this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
             this.snapshot.video = {
               ...this.snapshot.video ?? { updatedAt: Date.now(), source },
-              updatedAt: Date.now(),
+              updatedAt: sampledAt,
               source,
               inference,
               ...preview.ok ? {} : { error: preview.message }
@@ -602420,6 +602673,30 @@ ${inference}` : ""].filter(Boolean).join("\n");
             transcript,
             analysis
           };
+          const audioEvents = [];
+          if (transcript) {
+            audioEvents.push({
+              id: stableEventId("audio_transcript", input, this.snapshot.audio.updatedAt, recordingPath),
+              kind: "audio_transcript",
+              observedAt: this.snapshot.audio.updatedAt,
+              source: input,
+              title: "Live ASR transcript",
+              summary: transcript,
+              evidencePath: recordingPath
+            });
+          }
+          if (analysis) {
+            audioEvents.push({
+              id: stableEventId("audio_sound", input, this.snapshot.audio.updatedAt, recordingPath),
+              kind: "audio_sound",
+              observedAt: this.snapshot.audio.updatedAt,
+              source: input,
+              title: "Live sound analysis",
+              summary: analysis,
+              evidencePath: recordingPath
+            });
+          }
+          this.snapshot.events = mergeRecentById(this.snapshot.events, audioEvents, 50);
           this.persist();
           return [
             `Audio sample captured from ${input}: ${recordingPath}`,
@@ -672519,7 +672796,7 @@ function buildTelegramReflectionExtractionPrompt(options2) {
     episodes || "none"
   ].join("\n");
 }
-function parseJsonObject3(raw) {
+function parseJsonObject4(raw) {
   const text2 = raw.trim();
   if (!text2) return null;
   const fenced = text2.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1]?.trim();
@@ -672533,7 +672810,7 @@ function parseJsonObject3(raw) {
   }
 }
 function parseTelegramReflectionExtraction(raw) {
-  const parsed = parseJsonObject3(raw);
+  const parsed = parseJsonObject4(raw);
   if (!parsed) return null;
   const tags = Array.isArray(parsed.tags) ? parsed.tags.map((item) => {
     const obj = item;

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "omnius",
-  "version": "1.0.405",
+  "version": "1.0.406",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "omnius",
-      "version": "1.0.405",
+      "version": "1.0.406",
       "bundleDependencies": [
         "image-to-ascii"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnius",
-  "version": "1.0.405",
+  "version": "1.0.406",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",