@inetafrica/open-claudia 3.5.33 → 3.5.58

package/channels/kazee/call/camera.js

@@ -0,0 +1,233 @@
+"use strict";
+
+// Stills of the caller's camera, so the full agent can look when they ask
+// ("Can you see me?", "What am I holding?"). Their VP8 video arrives as RTP;
+// whole frames go to a local ffmpeg as an IVF stream, and ffmpeg keeps one
+// JPEG up to date every couple of seconds. Only the latest still is kept, in
+// a temp folder removed when the call ends. 11 Oct: the call took only audio,
+// so a camera switched on mid-call was never seen.
+
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { spawn } = require("child_process");
+const { dePacketizeRtpPackets } = require("werift");
+
+const STILL_EVERY_S = 2;
+const STILL_SIZE = 768;
+// Ask the sender for a keyframe at most this often while there is no
+// picture yet, or after a lost packet left the decoder without one.
+const KEYFRAME_RETRY_MS = 2000;
+// Frames still waiting for packets are dropped after this many newer ones.
+const MAX_OPEN_FRAMES = 8;
+// RFC 7742 / 3GPP TS 26.114 coordination of video orientation.
+const VIDEO_ORIENTATION_URI = "urn:3gpp:video-orientation";
+const clog = (msg) => console.log(`${new Date().toISOString()} Kazee call camera: ${msg}`);
+
+// The 2 low bits of the orientation byte: clockwise turns the receiver must
+// apply (0, 90, 180, 270 degrees).
+function orientationTurns(byte) { return Number(byte) & 0x03; }
+
+// The rotation filter for a still, from the sender's orientation turns.
+function rotationFilter(turns) {
+  return ["", "transpose=1", "hflip,vflip", "transpose=2"][turns & 3];
+}
+
+// A VP8 keyframe's width and height (RFC 6386 9.1), or null.
+function vp8KeyframeSize(frame) {
+  if (!frame || frame.length < 10 || (frame[0] & 0x01) !== 0) return null;
+  if (frame[3] !== 0x9d || frame[4] !== 0x01 || frame[5] !== 0x2a) return null;
+  return { width: frame.readUInt16LE(6) & 0x3fff, height: frame.readUInt16LE(8) & 0x3fff };
+}
+
+function ivfHeader({ width, height }) {
+  const header = Buffer.alloc(32);
+  header.write("DKIF", 0, "ascii");
+  header.writeUInt16LE(0, 4);
+  header.writeUInt16LE(32, 6);
+  header.write("VP80", 8, "ascii");
+  header.writeUInt16LE(width, 12);
+  header.writeUInt16LE(height, 14);
+  header.writeUInt32LE(90000, 16); // time base 1/90000: RTP video clock
+  header.writeUInt32LE(1, 20);
+  return header;
+}
+
+function ivfFrame(data, pts) {
+  const header = Buffer.alloc(12);
+  header.writeUInt32LE(data.length, 0);
+  header.writeBigUInt64LE(BigInt(Math.max(0, pts)), 4);
+  return Buffer.concat([header, data]);
+}
+
+// The extmap id the offer gives video orientation, or null.
+function orientationExtensionId(sdp) {
+  const match = String(sdp || "").match(/a=extmap:(\d+)(?:\/\w+)? urn:3gpp:video-orientation/);
+  return match ? Number(match[1]) : null;
+}
+
+class CameraStills {
+  constructor({ feedId, requestKeyframe = async () => {}, spawnImpl = spawn, dir = null, now = Date.now, log = clog } = {}) {
+    this.feedId = String(feedId);
+    this.requestKeyframe = requestKeyframe;
+    this.spawnImpl = spawnImpl;
+    this.dir = dir || fs.mkdtempSync(path.join(os.tmpdir(), "oc-call-camera-"));
+    this.file = path.join(this.dir, "latest.jpg");
+    this.now = now;
+    this.log = log;
+    this.orientationId = null;
+    this.turns = 0;
+    this._frames = new Map(); // RTP timestamp -> packets
+    this._lastSeq = null;
+    this._needKeyframe = true;
+    this._keyframeAskedAt = 0;
+    this._ffmpeg = null;
+    this._firstTimestamp = null;
+    this._size = null;
+    this._frameCount = 0;
+    this._stillAt = 0;
+    this._lastPacketAt = 0;
+    this.closed = false;
+    this._watch = null;
+  }
+
+  setOrientationExtension(id) { this.orientationId = Number.isInteger(id) ? id : null; }
+
+  // One RTP packet of the caller's video (werift RtpPacket).
+  push(packet) {
+    if (this.closed || !packet?.header) return;
+    this._lastPacketAt = this.now();
+    const { header } = packet;
+    if (this.orientationId != null) {
+      const ext = (header.extensions || []).find((item) => item.id === this.orientationId);
+      if (ext?.payload?.length) this._turn(orientationTurns(ext.payload[0]));
+    }
+    // A gap in sequence numbers: some frame is incomplete, and decoding goes
+    // wrong until the next keyframe.
+    if (this._lastSeq != null) {
+      const step = (header.sequenceNumber - this._lastSeq + 65536) % 65536;
+      if (step > 1 && step < 32768) this._lost();
+    }
+    if (this._lastSeq == null || ((header.sequenceNumber - this._lastSeq + 65536) % 65536) < 32768) this._lastSeq = header.sequenceNumber;
+    const list = this._frames.get(header.timestamp) || [];
+    list.push(packet);
+    this._frames.set(header.timestamp, list);
+    if (header.marker) this._complete(header.timestamp);
+    while (this._frames.size > MAX_OPEN_FRAMES) this._frames.delete(this._frames.keys().next().value);
+    this._maybeAskKeyframe();
+  }
+
+  // The latest still: its path and age, or null before the first one.
+  latest() {
+    if (!this._stillAt) return null;
+    return { path: this.file, ageMs: Math.max(0, this.now() - this._stillAt), receiving: this.now() - this._lastPacketAt < 3000 };
+  }
+
+  _lost() {
+    if (this._needKeyframe) return;
+    this._needKeyframe = true;
+    this._keyframeAskedAt = 0;
+  }
+
+  _maybeAskKeyframe() {
+    if (!this._needKeyframe || this.now() - this._keyframeAskedAt < KEYFRAME_RETRY_MS) return;
+    this._keyframeAskedAt = this.now();
+    Promise.resolve(this.requestKeyframe()).catch(() => {});
+  }
+
+  _turn(turns) {
+    if (turns === this.turns) return;
+    this.log(`orientation ${this.turns * 90} -> ${turns * 90} deg feed=${this.feedId}`);
+    this.turns = turns;
+    // The rotation is a filter setting: start ffmpeg again with the new one.
+    this._stopFfmpeg();
+    this._needKeyframe = true;
+    this._keyframeAskedAt = 0;
+  }
+
+  _complete(timestamp) {
+    const packets = (this._frames.get(timestamp) || []).sort((a, b) => ((a.header.sequenceNumber - b.header.sequenceNumber + 32768 + 65536) % 65536) - 32768);
+    this._frames.delete(timestamp);
+    for (let index = 1; index < packets.length; index += 1) {
+      if ((packets[index].header.sequenceNumber - packets[index - 1].header.sequenceNumber + 65536) % 65536 !== 1) return this._lost();
+    }
+    let frame;
+    try { frame = dePacketizeRtpPackets("VP8", packets); } catch (_) { return this._lost(); }
+    if (!frame?.data?.length) return;
+    const size = frame.isKeyframe ? vp8KeyframeSize(frame.data) : null;
+    if (this._needKeyframe) {
+      if (!size) return;
+      this._needKeyframe = false;
+    }
+    if (!this._ffmpeg) {
+      if (!size) { this._needKeyframe = true; return; }
+      this._startFfmpeg(size, timestamp);
+    }
+    const pts = ((timestamp - this._firstTimestamp) >>> 0);
+    try { this._ffmpeg.stdin.write(ivfFrame(frame.data, pts)); } catch (_) {}
+  }
+
+  _startFfmpeg(size, timestamp) {
+    this._size = size;
+    this._firstTimestamp = timestamp;
+    const rotate = rotationFilter(this.turns);
+    // select, not fps: fps holds each still back until later frames decide
+    // it, so the newest picture would always be one interval late.
+    const filter = [
+      `select='isnan(prev_selected_t)+gte(t-prev_selected_t\\,${STILL_EVERY_S})'`,
+      rotate,
+      `scale=${STILL_SIZE}:${STILL_SIZE}:force_original_aspect_ratio=decrease`,
+    ].filter(Boolean).join(",");
+    // No probing: by default ffmpeg reads seconds of input before its first
+    // picture, and a camera stream never ends. (-fflags nobuffer broke VP8
+    // decoding: "Invalid data found when processing input".)
+    const args = ["-hide_banner", "-loglevel", "error", "-probesize", "32", "-analyzeduration", "0", "-f", "ivf", "-i", "pipe:0",
+      "-vf", filter, "-fps_mode", "passthrough", "-update", "1", "-atomic_writing", "1", "-q:v", "4", "-y", this.file];
+    const proc = this.spawnImpl("ffmpeg", args, { stdio: ["pipe", "ignore", "pipe"] });
+    this._ffmpeg = proc;
+    proc.stdin.on?.("error", () => {});
+    proc.stdin.write(ivfHeader(size));
+    proc.stderr?.on?.("data", (chunk) => this.log(`ffmpeg: ${String(chunk).trim().slice(0, 200)}`));
+    proc.on?.("close", (code) => {
+      if (this._ffmpeg === proc) { this._ffmpeg = null; this._needKeyframe = true; }
+      if (!this.closed && code) this.log(`ffmpeg exited ${code} feed=${this.feedId}`);
+    });
+    proc.on?.("error", (error) => this.log(`ffmpeg failed: ${error.message}`));
+    this.log(`stills on: ${size.width}x${size.height} turn=${this.turns * 90} feed=${this.feedId}`);
+    this._watchStill();
+  }
+
+  // ffmpeg renames each finished still into place: note when one lands.
+  _watchStill() {
+    if (this._watch) return;
+    this._watch = setInterval(() => {
+      try {
+        const at = fs.statSync(this.file).mtimeMs;
+        if (at > this._stillAt) {
+          if (!this._stillAt) this.log(`first still feed=${this.feedId}`);
+          this._stillAt = at;
+        }
+      } catch (_) {}
+    }, 500);
+    this._watch.unref?.();
+  }
+
+  _stopFfmpeg() {
+    const proc = this._ffmpeg;
+    this._ffmpeg = null;
+    if (!proc) return;
+    try { proc.stdin.end(); } catch (_) {}
+    setTimeout(() => { try { proc.kill("SIGKILL"); } catch (_) {} }, 1000).unref?.();
+  }
+
+  close() {
+    if (this.closed) return;
+    this.closed = true;
+    this._stopFfmpeg();
+    if (this._watch) clearInterval(this._watch);
+    this._frames.clear();
+    setTimeout(() => { try { fs.rmSync(this.dir, { recursive: true, force: true }); } catch (_) {} }, 1500).unref?.();
+  }
+}
+
+module.exports = { CameraStills, VIDEO_ORIENTATION_URI, orientationExtensionId, orientationTurns, rotationFilter, vp8KeyframeSize, ivfHeader, ivfFrame };

package/channels/kazee/call/job-steps.js

@@ -0,0 +1,81 @@
+"use strict";
+
+// What the full agent is doing on a call, in plain words, from its tool
+// starts: "reading files", "checking GitLab". Fixed rules, no model; ids,
+// paths and commands are never kept. Feeds the call's job board, so the quick
+// voice can answer "what's the status?" from real work. Ported from the 4 Oct
+// calls build (never-silent.js stepForTool).
+
+const STEPS = Object.freeze({
+  notes: "checking my notes",
+  past: "searching our past conversations",
+  tool: "looking for the right tool",
+  servers: "checking the servers",
+  files: "reading files",
+  web: "looking it up online",
+  change: "making the change",
+  check: "running a check",
+  tasks: "checking the tasks",
+  code: "checking the code history",
+  helper: "getting a helper to dig through it",
+  calendar: "checking the calendar",
+  service: "checking a service",
+});
+
+// `open-claudia tool run <id>`: a plain name only for these; any other tool
+// is "running a check" (its id is never said or kept).
+const TOOL_RUNS = [
+  [/k8s|kube|cluster|server|node|pod/, STEPS.servers],
+  [/gitlab|glab/, "checking GitLab"],
+  [/github/, "checking GitHub"],
+  [/mongo|postgres|mysql|database|(?:^|-)db(?:-|$)/, "checking the database"],
+  [/grafana|prometheus|loki|monitor|metrics|alert/, "checking the monitoring"],
+  [/sentry/, "checking the error reports"],
+  [/calendar|agenda/, STEPS.calendar],
+  [/task|spaces|ticket/, STEPS.tasks],
+  [/gmail|mail/, "checking email"],
+  [/zoho|crm|books/, "checking Zoho"],
+  [/kazee|chat/, "checking Kazee"],
+];
+
+const CLI = String.raw`(?:\bopen-claudia|\bcli\.js)\s+`;
+const CLI_STEPS = [
+  [new RegExp(`${CLI}(?:ts|tw|transcript-search|transcript-window|recent)\\b`), STEPS.past],
+  [new RegExp(`${CLI}(?:entity|entities|pack|packs|people|recall|lessons|ideas|intros)\\b`), STEPS.notes],
+  [new RegExp(`${CLI}tools?\\s+(?:search|show|list)\\b`), STEPS.tool],
+  [new RegExp(`${CLI}(?:task|tasks|kpi)\\b`), STEPS.tasks],
+  [new RegExp(`${CLI}(?:agenda|cron-list|job-list|watch-list|remind)\\b`), STEPS.calendar],
+];
+
+function commandText(detail) {
+  if (typeof detail === "string") return detail;
+  const command = detail?.command ?? detail?.cmd ?? "";
+  return Array.isArray(command) ? command.join(" ") : String(command || "");
+}
+
+const byName = (id) => (TOOL_RUNS.find(([pattern]) => pattern.test(String(id).toLowerCase())) || [null, STEPS.check])[1];
+
+// A tool start as a step, or null for bookkeeping (call-say is speech, not a step).
+function stepForTool(name, detail) {
+  const tool = String(name || "");
+  if (/^(?:Read|Grep|Glob|LS|NotebookRead)$/.test(tool)) return STEPS.files;
+  if (/^(?:WebFetch|WebSearch|Web)$/.test(tool)) return STEPS.web;
+  if (/^(?:Edit|Write|MultiEdit|NotebookEdit)$/.test(tool)) return STEPS.change;
+  if (/^(?:TodoWrite|ToolSearch|ExitPlanMode)$/.test(tool)) return null;
+  if (/^(?:Task|Agent)$/.test(tool)) return STEPS.helper;
+  if (/^mcp__/.test(tool)) return /search|fetch|browse|web/i.test(tool) && !/mail|calendar|drive/i.test(tool) ? STEPS.web : byName(tool.slice(5));
+  if (tool !== "Bash" && tool !== "Shell") return STEPS.check;
+  const command = commandText(detail);
+  if (/\bcall-(?:say|ask)\b/.test(command)) return null;
+  const run = new RegExp(`${CLI}tools?\\s+run\\s+([A-Za-z0-9._-]+)`).exec(command);
+  if (run) return byName(run[1]);
+  const cli = CLI_STEPS.find(([pattern]) => pattern.test(command));
+  if (cli) return cli[1];
+  if (/\b(?:kubectl|helm|k9s)\b/.test(command)) return STEPS.servers;
+  if (/^\s*(?:cd\s+\S+\s*&&\s*)?git\b/.test(command)) return STEPS.code;
+  if (/^\s*(?:cd\s+\S+\s*&&\s*)?(?:cat|head|tail|less|grep|rg|ls|find|wc|sed\s+-n|jq)\b/.test(command)) return STEPS.files;
+  if (/^\s*(?:curl|wget)\b/.test(command)) return STEPS.service;
+  return STEPS.check;
+}
+
+module.exports = { stepForTool, STEPS };