@ozaiya/openclaw-channel 0.10.29 → 0.10.31

package/dist/src/h264Packetizer.js

@@ -0,0 +1,153 @@
+/**
+ * RFC 6184 H.264 RTP packetizer for WebRTC media tracks.
+ *
+ * The native capture addon (ScreenCaptureKit + VideoToolbox) emits one
+ * Annex-B access unit (one encoded frame, start-code delimited NAL units) per
+ * `onFrame` callback. To send it over a werift WebRTC video track we must
+ * convert each access unit into a sequence of RTP payloads:
+ *
+ *   - Single NAL unit packet   — NALU that fits in one RTP packet (type 1-23)
+ *   - FU-A fragmentation        — NALU too large for one RTP packet (type 28)
+ *
+ * We deliberately do NOT emit STAP-A aggregation packets: the consumer is a
+ * browser's native WebRTC depacketizer, which handles single-NAL + FU-A
+ * universally, and keeping SPS/PPS/IDR as separate single-NAL packets is
+ * simpler and equally correct.
+ *
+ * werift's RTCRtpSender rewrites ssrc / payloadType / sequenceNumber on send
+ * (see rtpSender.ts `sendRtp`), so the packetizer only needs to produce correct
+ * `marker` (set on the final packet of an access unit) and `timestamp`
+ * (90 kHz). We still supply a monotonic 16-bit sequence number — werift bases
+ * its sequence offset on the first packet it observes.
+ */
+import { RtpPacket, RtpHeader } from 'werift';
+/** Conservative RTP payload size — keeps packets under the common 1280-byte
+ * path MTU after IP/UDP/SRTP/DTLS overhead, avoiding IP fragmentation. */
+const RTP_MTU_PAYLOAD = 1200;
+/** H.264 90 kHz RTP clock rate. */
+export const H264_CLOCK_RATE = 90000;
+/** NAL unit type for FU-A fragmentation units (RFC 6184 §5.8). */
+const NALU_TYPE_FU_A = 28;
+/** NAL unit type for IDR (instantaneous decoder refresh) slices — a keyframe. */
+const NALU_TYPE_IDR = 5;
+/** NAL unit type for sequence parameter set. */
+const NALU_TYPE_SPS = 7;
+/**
+ * Split an Annex-B byte stream (one access unit) into raw NAL units with the
+ * start codes (00 00 01 or 00 00 00 01) stripped. Each returned buffer begins
+ * with the 1-byte NAL header.
+ */
+export function splitNalUnits(annexB) {
+    const nalus = [];
+    const len = annexB.length;
+    // Find the offset of each start code.
+    let i = 0;
+    let naluStart = -1;
+    const isStartCode3 = (p) => p + 2 < len && annexB[p] === 0 && annexB[p + 1] === 0 && annexB[p + 2] === 1;
+    const isStartCode4 = (p) => p + 3 < len && annexB[p] === 0 && annexB[p + 1] === 0 && annexB[p + 2] === 0 && annexB[p + 3] === 1;
+    while (i < len) {
+        if (isStartCode4(i)) {
+            if (naluStart >= 0 && i > naluStart)
+                nalus.push(annexB.subarray(naluStart, i));
+            i += 4;
+            naluStart = i;
+        }
+        else if (isStartCode3(i)) {
+            if (naluStart >= 0 && i > naluStart)
+                nalus.push(annexB.subarray(naluStart, i));
+            i += 3;
+            naluStart = i;
+        }
+        else {
+            i++;
+        }
+    }
+    if (naluStart >= 0 && naluStart < len)
+        nalus.push(annexB.subarray(naluStart, len));
+    // If no start codes were found, treat the whole buffer as one NAL unit
+    // (some encoders emit AVCC-less raw NALUs). This keeps us robust.
+    if (nalus.length === 0 && len > 0)
+        nalus.push(annexB);
+    return nalus;
+}
+/** True if the access unit contains an IDR slice (i.e. is a keyframe). */
+export function isKeyframeAccessUnit(annexB) {
+    for (const nalu of splitNalUnits(annexB)) {
+        const type = nalu[0] & 0x1f;
+        if (type === NALU_TYPE_IDR || type === NALU_TYPE_SPS)
+            return true;
+    }
+    return false;
+}
+/**
+ * Stateful packetizer — owns the rolling RTP sequence number for one track.
+ */
+export class H264Packetizer {
+    /** 16-bit rolling sequence number; werift offsets from the first value. */
+    sequenceNumber;
+    payloadType;
+    ssrc;
+    constructor(opts) {
+        // werift overrides payloadType/ssrc on send; these are placeholders.
+        this.payloadType = opts?.payloadType ?? 96;
+        this.ssrc = opts?.ssrc ?? 0;
+        // Start from a fixed-but-arbitrary value; varying it is unnecessary since
+        // werift rewrites the on-wire sequence numbers.
+        this.sequenceNumber = (opts?.initialSequenceNumber ?? 0) & 0xffff;
+    }
+    nextSeq() {
+        const s = this.sequenceNumber;
+        this.sequenceNumber = (this.sequenceNumber + 1) & 0xffff;
+        return s;
+    }
+    makePacket(payload, timestamp, marker) {
+        const header = new RtpHeader({
+            version: 2,
+            payloadType: this.payloadType,
+            sequenceNumber: this.nextSeq(),
+            timestamp: timestamp >>> 0,
+            ssrc: this.ssrc,
+            marker,
+        });
+        return new RtpPacket(header, payload);
+    }
+    /**
+     * Packetize one Annex-B access unit into RTP packets.
+     * @param accessUnit Annex-B encoded frame (start-code delimited NALUs).
+     * @param rtpTimestamp 90 kHz RTP timestamp for this frame.
+     */
+    packetize(accessUnit, rtpTimestamp) {
+        const nalus = splitNalUnits(accessUnit);
+        if (nalus.length === 0)
+            return [];
+        // Build the payloads first so we know which is the final one (marker bit).
+        const payloads = [];
+        for (const nalu of nalus) {
+            if (nalu.length <= RTP_MTU_PAYLOAD) {
+                // Single NAL unit packet — payload is the NALU verbatim.
+                payloads.push(nalu);
+            }
+            else {
+                // FU-A fragmentation.
+                const naluHeader = nalu[0];
+                const fIndicator = naluHeader & 0xe0; // F + NRI bits
+                const naluType = naluHeader & 0x1f;
+                const body = nalu.subarray(1); // drop the 1-byte NAL header
+                const chunkSize = RTP_MTU_PAYLOAD - 2; // 2 bytes FU indicator + FU header
+                const total = Math.ceil(body.length / chunkSize);
+                for (let f = 0; f < total; f++) {
+                    const start = f === 0 ? 1 : 0;
+                    const end = f === total - 1 ? 1 : 0;
+                    const fuIndicator = fIndicator | NALU_TYPE_FU_A;
+                    const fuHeader = (start << 7) | (end << 6) | naluType; // R bit = 0
+                    const slice = body.subarray(f * chunkSize, (f + 1) * chunkSize);
+                    payloads.push(Buffer.concat([Buffer.from([fuIndicator, fuHeader]), slice]));
+                }
+            }
+        }
+        const ts = rtpTimestamp >>> 0;
+        const lastIndex = payloads.length - 1;
+        return payloads.map((p, idx) => this.makePacket(p, ts, idx === lastIndex));
+    }
+}
+//# sourceMappingURL=h264Packetizer.js.map

package/dist/src/h264Packetizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"h264Packetizer.js","sourceRoot":"","sources":["../../src/h264Packetizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAE9C;0EAC0E;AAC1E,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,mCAAmC;AACnC,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,CAAC;AAErC,kEAAkE;AAClE,MAAM,cAAc,GAAG,EAAE,CAAC;AAE1B,iFAAiF;AACjF,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,gDAAgD;AAChD,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,MAAc;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAE1B,sCAAsC;IACtC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;IAEnB,MAAM,YAAY,GAAG,CAAC,CAAS,EAAW,EAAE,CACxC,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACjF,MAAM,YAAY,GAAG,CAAC,CAAS,EAAW,EAAE,CACxC,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAExG,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC;QACb,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;YAClB,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/E,CAAC,IAAI,CAAC,CAAC;YACP,SAAS,GAAG,CAAC,CAAC;QAClB,CAAC;aAAM,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;YACzB,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/E,CAAC,IAAI,CAAC,CAAC;YACP,SAAS,GAAG,CAAC,CAAC;QAClB,CAAC;aAAM,CAAC;YACJ,CAAC,EAAE,CAAC;QACR,CAAC;IACL,CAAC;IACD,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,GAAG,GAAG;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;IAEnF,uEAAuE;IACvE,kEAAkE;IAClE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEtD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,oBAAoB,CAAC,MAAc;IAC/C,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAC5B,IAAI,IAAI,KAAK,aAAa,IAAI,IAAI,KAAK,aAAa;YAAE,OAAO,IAAI,CAAC;IACtE,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,cAAc;IACvB,2EAA2E;IACnE,cAAc,CAAS;IACd,WAAW,CAAS;IACpB,IAAI,CAAS;IAE9B,YAAY,IAA8E;QACtF,qEAAqE;QACrE,IAAI,CAAC,WAAW,GAAG,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;QAC5B,0EAA0E;QAC1E,gDAAgD;QAChD,IAAI,CAAC,cAAc,GAAG,CAAC,IAAI,EAAE,qBAAqB,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC;IACtE,CAAC;IAEO,OAAO;QACX,MAAM,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC;QACzD,OAAO,CAAC,CAAC;IACb,CAAC;IAEO,UAAU,CAAC,OAAe,EAAE,SAAiB,EAAE,MAAe;QAClE,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;YACzB,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE;YAC9B,SAAS,EAAE,SAAS,KAAK,CAAC;YAC1B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM;SACT,CAAC,CAAC;QACH,OAAO,IAAI,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,UAAkB,EAAE,YAAoB;QAC9C,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,2EAA2E;QAC3E,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,IAAI,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;gBACjC,yDAAyD;gBACzD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACJ,sBAAsB;gBACtB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,MAAM,UAAU,GAAG,UAAU,GAAG,IAAI,CAAC,CAAC,eAAe;gBACrD,MAAM,QAAQ,GAAG,UAAU,GAAG,IAAI,CAAC;gBACnC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,6BAA6B;gBAC5D,MAAM,SAAS,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,mCAAmC;gBAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;gBAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7B,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC9B,MAAM,GAAG,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACpC,MAAM,WAAW,GAAG,UAAU,GAAG,cAAc,CAAC;oBAChD,MAAM,QAAQ,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,YAAY;oBACnE,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;oBAChE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBAChF,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC;QAC9B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC;IAC/E,CAAC;CACJ"}

package/dist/src/sandboxScreenCdp.d.ts

@@ -15,6 +15,18 @@ export interface CdpScreencastOptions {
         width: number;
         height: number;
     }) => void;
+    /**
+     * Pointer moved/clicked inside the sandbox page (viewport CSS px — 1:1 with
+     * frame pixels at dpr=1). Sourced from a page-injected listener, so it sees
+     * BOTH the agent's synthetic CDP input and viewer takeover input. JPEG/H.264
+     * frames carry no pointer at all (headless screenshots have no OS cursor),
+     * so this is the viewer's only way to draw one. `shape` is the CSS cursor
+     * under the pointer, reported only when it changes (hover affordance).
+     */
+    onCursor?: (x: number, y: number, click: boolean, shape?: string) => void;
+    /** Text copied/cut inside the sandbox page (same probe) — the sandbox
+     *  clipboard is unreachable any other way. */
+    onClipboard?: (text: string) => void;
     quality?: number;
     minIntervalMs?: number;
     /**
@@ -38,3 +50,19 @@ export interface CdpScreencastOptions {
     };
 }
 export declare function createCdpScreencast(opts: CdpScreencastOptions): Promise<CdpScreencast | null>;
+export interface SharedViewerHooks {
+    onFrame: (jpeg: Buffer) => void;
+    onCursor?: (x: number, y: number, click: boolean, shape?: string) => void;
+    onClipboard?: (text: string) => void;
+    getQuality?: () => number;
+    getMinIntervalMs?: () => number;
+    getBackpressure?: () => number;
+}
+export declare function acquireCdpScreencast(opts: {
+    host: string;
+    port: number;
+    quality?: number;
+    minIntervalMs?: number;
+    log?: CdpScreencastOptions["log"];
+}, viewerId: string, hooks: SharedViewerHooks): Promise<CdpScreencast | null>;
+export declare function releaseCdpScreencast(host: string, port: number, viewerId: string): void;