susi-qemu 0.0.3 → 0.0.5

data/lib/novnc/core/decoders/h264.js

@@ -0,0 +1,321 @@
+/*
+ * noVNC: HTML5 VNC client
+ * Copyright (C) 2024 The noVNC Authors
+ * Licensed under MPL 2.0 (see LICENSE.txt)
+ *
+ * See README.md for usage and integration instructions.
+ *
+ */
+
+import * as Log from '../util/logging.js';
+
+export class H264Parser {
+    constructor(data) {
+        this._data = data;
+        this._index = 0;
+        this.profileIdc = null;
+        this.constraintSet = null;
+        this.levelIdc = null;
+    }
+
+    _getStartSequenceLen(index) {
+        let data = this._data;
+        if (data[index + 0] == 0 && data[index + 1] == 0 && data[index + 2] == 0 && data[index + 3] == 1) {
+            return 4;
+        }
+        if (data[index + 0] == 0 && data[index + 1] == 0 && data[index + 2] == 1) {
+            return 3;
+        }
+        return 0;
+    }
+
+    _indexOfNextNalUnit(index) {
+        let data = this._data;
+        for (let i = index; i < data.length; ++i) {
+            if (this._getStartSequenceLen(i) != 0) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    _parseSps(index) {
+        this.profileIdc = this._data[index];
+        this.constraintSet = this._data[index + 1];
+        this.levelIdc = this._data[index + 2];
+    }
+
+    _parseNalUnit(index) {
+        const firstByte = this._data[index];
+        if (firstByte & 0x80) {
+            throw new Error('H264 parsing sanity check failed, forbidden zero bit is set');
+        }
+        const unitType = firstByte & 0x1f;
+
+        switch (unitType) {
+            case 1: // coded slice, non-idr
+                return { slice: true };
+            case 5: // coded slice, idr
+                return { slice: true, key: true };
+            case 6: // sei
+                return {};
+            case 7: // sps
+                this._parseSps(index + 1);
+                return {};
+            case 8: // pps
+                return {};
+            default:
+                Log.Warn("Unhandled unit type: ", unitType);
+                break;
+        }
+        return {};
+    }
+
+    parse() {
+        const startIndex = this._index;
+        let isKey = false;
+
+        while (this._index < this._data.length) {
+            const startSequenceLen = this._getStartSequenceLen(this._index);
+            if (startSequenceLen == 0) {
+                throw new Error('Invalid start sequence in bit stream');
+            }
+
+            const { slice, key } = this._parseNalUnit(this._index + startSequenceLen);
+
+            let nextIndex = this._indexOfNextNalUnit(this._index + startSequenceLen);
+            if (nextIndex == -1) {
+                this._index = this._data.length;
+            } else {
+                this._index = nextIndex;
+            }
+
+            if (key) {
+                isKey = true;
+            }
+            if (slice) {
+                break;
+            }
+        }
+
+        if (startIndex === this._index) {
+            return null;
+        }
+
+        return {
+            frame: this._data.subarray(startIndex, this._index),
+            key: isKey,
+        };
+    }
+}
+
+export class H264Context {
+    constructor(width, height) {
+        this.lastUsed = 0;
+        this._width = width;
+        this._height = height;
+        this._profileIdc = null;
+        this._constraintSet = null;
+        this._levelIdc = null;
+        this._decoder = null;
+        this._pendingFrames = [];
+    }
+
+    _handleFrame(frame) {
+        let pending = this._pendingFrames.shift();
+        if (pending === undefined) {
+            throw new Error("Pending frame queue empty when receiving frame from decoder");
+        }
+
+        if (pending.timestamp != frame.timestamp) {
+            throw new Error("Video frame timestamp mismatch. Expected " +
+                frame.timestamp + " but but got " + pending.timestamp);
+        }
+
+        pending.frame = frame;
+        pending.ready = true;
+        pending.resolve();
+
+        if (!pending.keep) {
+            frame.close();
+        }
+    }
+
+    _handleError(e) {
+        throw new Error("Failed to decode frame: " + e.message);
+    }
+
+    _configureDecoder(profileIdc, constraintSet, levelIdc) {
+        if (this._decoder === null || this._decoder.state === 'closed') {
+            this._decoder = new VideoDecoder({
+                output: frame => this._handleFrame(frame),
+                error: e => this._handleError(e),
+            });
+        }
+        const codec = 'avc1.' +
+            profileIdc.toString(16).padStart(2, '0') +
+            constraintSet.toString(16).padStart(2, '0') +
+            levelIdc.toString(16).padStart(2, '0');
+        this._decoder.configure({
+            codec: codec,
+            codedWidth: this._width,
+            codedHeight: this._height,
+            optimizeForLatency: true,
+        });
+    }
+
+    _preparePendingFrame(timestamp) {
+        let pending = {
+            timestamp: timestamp,
+            promise: null,
+            resolve: null,
+            frame: null,
+            ready: false,
+            keep: false,
+        };
+        pending.promise = new Promise((resolve) => {
+            pending.resolve = resolve;
+        });
+        this._pendingFrames.push(pending);
+
+        return pending;
+    }
+
+    decode(payload) {
+        let parser = new H264Parser(payload);
+        let result = null;
+
+        // Ideally, this timestamp should come from the server, but we'll just
+        // approximate it instead.
+        let timestamp = Math.round(window.performance.now() * 1e3);
+
+        while (true) {
+            let encodedFrame = parser.parse();
+            if (encodedFrame === null) {
+                break;
+            }
+
+            if (parser.profileIdc !== null) {
+                self._profileIdc = parser.profileIdc;
+                self._constraintSet = parser.constraintSet;
+                self._levelIdc = parser.levelIdc;
+            }
+
+            if (this._decoder === null || this._decoder.state !== 'configured') {
+                if (!encodedFrame.key) {
+                    Log.Warn("Missing key frame. Can't decode until one arrives");
+                    continue;
+                }
+                if (self._profileIdc === null) {
+                    Log.Warn('Cannot config decoder. Have not received SPS and PPS yet.');
+                    continue;
+                }
+                this._configureDecoder(self._profileIdc, self._constraintSet,
+                                       self._levelIdc);
+            }
+
+            result = this._preparePendingFrame(timestamp);
+
+            const chunk = new EncodedVideoChunk({
+                timestamp: timestamp,
+                type: encodedFrame.key ? 'key' : 'delta',
+                data: encodedFrame.frame,
+            });
+
+            try {
+                this._decoder.decode(chunk);
+            } catch (e) {
+                Log.Warn("Failed to decode:", e);
+            }
+        }
+
+        // We only keep last frame of each payload
+        if (result !== null) {
+            result.keep = true;
+        }
+
+        return result;
+    }
+}
+
+export default class H264Decoder {
+    constructor() {
+        this._tick = 0;
+        this._contexts = {};
+    }
+
+    _contextId(x, y, width, height) {
+        return [x, y, width, height].join(',');
+    }
+
+    _findOldestContextId() {
+        let oldestTick = Number.MAX_VALUE;
+        let oldestKey = undefined;
+        for (const [key, value] of Object.entries(this._contexts)) {
+            if (value.lastUsed < oldestTick) {
+                oldestTick = value.lastUsed;
+                oldestKey = key;
+            }
+        }
+        return oldestKey;
+    }
+
+    _createContext(x, y, width, height) {
+        const maxContexts = 64;
+        if (Object.keys(this._contexts).length >= maxContexts) {
+            let oldestContextId = this._findOldestContextId();
+            delete this._contexts[oldestContextId];
+        }
+        let context = new H264Context(width, height);
+        this._contexts[this._contextId(x, y, width, height)] = context;
+        return context;
+    }
+
+    _getContext(x, y, width, height) {
+        let context = this._contexts[this._contextId(x, y, width, height)];
+        return context !== undefined ? context : this._createContext(x, y, width, height);
+    }
+
+    _resetContext(x, y, width, height) {
+        delete this._contexts[this._contextId(x, y, width, height)];
+    }
+
+    _resetAllContexts() {
+        this._contexts = {};
+    }
+
+    decodeRect(x, y, width, height, sock, display, depth) {
+        const resetContextFlag = 1;
+        const resetAllContextsFlag = 2;
+
+        if (sock.rQwait("h264 header", 8)) {
+            return false;
+        }
+
+        const length = sock.rQshift32();
+        const flags = sock.rQshift32();
+
+        if (sock.rQwait("h264 payload", length, 8)) {
+            return false;
+        }
+
+        if (flags & resetAllContextsFlag) {
+            this._resetAllContexts();
+        } else if (flags & resetContextFlag) {
+            this._resetContext(x, y, width, height);
+        }
+
+        let context = this._getContext(x, y, width, height);
+        context.lastUsed = this._tick++;
+
+        if (length !== 0) {
+            let payload = sock.rQshiftBytes(length, false);
+            let frame = context.decode(payload);
+            if (frame !== null) {
+                display.videoFrame(x, y, width, height, frame);
+            }
+        }
+
+        return true;
+    }
+}

data/lib/novnc/core/decoders/hextile.js

@@ -0,0 +1,181 @@
+/*
+ * noVNC: HTML5 VNC client
+ * Copyright (C) 2019 The noVNC Authors
+ * Licensed under MPL 2.0 (see LICENSE.txt)
+ *
+ * See README.md for usage and integration instructions.
+ *
+ */
+
+import * as Log from '../util/logging.js';
+
+export default class HextileDecoder {
+    constructor() {
+        this._tiles = 0;
+        this._lastsubencoding = 0;
+        this._tileBuffer = new Uint8Array(16 * 16 * 4);
+    }
+
+    decodeRect(x, y, width, height, sock, display, depth) {
+        if (this._tiles === 0) {
+            this._tilesX = Math.ceil(width / 16);
+            this._tilesY = Math.ceil(height / 16);
+            this._totalTiles = this._tilesX * this._tilesY;
+            this._tiles = this._totalTiles;
+        }
+
+        while (this._tiles > 0) {
+            let bytes = 1;
+
+            if (sock.rQwait("HEXTILE", bytes)) {
+                return false;
+            }
+
+            let subencoding = sock.rQpeek8();
+            if (subencoding > 30) {  // Raw
+                throw new Error("Illegal hextile subencoding (subencoding: " +
+                            subencoding + ")");
+            }
+
+            const currTile = this._totalTiles - this._tiles;
+            const tileX = currTile % this._tilesX;
+            const tileY = Math.floor(currTile / this._tilesX);
+            const tx = x + tileX * 16;
+            const ty = y + tileY * 16;
+            const tw = Math.min(16, (x + width) - tx);
+            const th = Math.min(16, (y + height) - ty);
+
+            // Figure out how much we are expecting
+            if (subencoding & 0x01) {  // Raw
+                bytes += tw * th * 4;
+            } else {
+                if (subencoding & 0x02) {  // Background
+                    bytes += 4;
+                }
+                if (subencoding & 0x04) {  // Foreground
+                    bytes += 4;
+                }
+                if (subencoding & 0x08) {  // AnySubrects
+                    bytes++;  // Since we aren't shifting it off
+
+                    if (sock.rQwait("HEXTILE", bytes)) {
+                        return false;
+                    }
+
+                    let subrects = sock.rQpeekBytes(bytes).at(-1);
+                    if (subencoding & 0x10) {  // SubrectsColoured
+                        bytes += subrects * (4 + 2);
+                    } else {
+                        bytes += subrects * 2;
+                    }
+                }
+            }
+
+            if (sock.rQwait("HEXTILE", bytes)) {
+                return false;
+            }
+
+            // We know the encoding and have a whole tile
+            sock.rQshift8();
+            if (subencoding === 0) {
+                if (this._lastsubencoding & 0x01) {
+                    // Weird: ignore blanks are RAW
+                    Log.Debug("     Ignoring blank after RAW");
+                } else {
+                    display.fillRect(tx, ty, tw, th, this._background);
+                }
+            } else if (subencoding & 0x01) {  // Raw
+                let pixels = tw * th;
+                let data = sock.rQshiftBytes(pixels * 4, false);
+                // Max sure the image is fully opaque
+                for (let i = 0;i <  pixels;i++) {
+                    data[i * 4 + 3] = 255;
+                }
+                display.blitImage(tx, ty, tw, th, data, 0);
+            } else {
+                if (subencoding & 0x02) {  // Background
+                    this._background = new Uint8Array(sock.rQshiftBytes(4));
+                }
+                if (subencoding & 0x04) {  // Foreground
+                    this._foreground = new Uint8Array(sock.rQshiftBytes(4));
+                }
+
+                this._startTile(tx, ty, tw, th, this._background);
+                if (subencoding & 0x08) {  // AnySubrects
+                    let subrects = sock.rQshift8();
+
+                    for (let s = 0; s < subrects; s++) {
+                        let color;
+                        if (subencoding & 0x10) {  // SubrectsColoured
+                            color = sock.rQshiftBytes(4);
+                        } else {
+                            color = this._foreground;
+                        }
+                        const xy = sock.rQshift8();
+                        const sx = (xy >> 4);
+                        const sy = (xy & 0x0f);
+
+                        const wh = sock.rQshift8();
+                        const sw = (wh >> 4) + 1;
+                        const sh = (wh & 0x0f) + 1;
+
+                        this._subTile(sx, sy, sw, sh, color);
+                    }
+                }
+                this._finishTile(display);
+            }
+            this._lastsubencoding = subencoding;
+            this._tiles--;
+        }
+
+        return true;
+    }
+
+    // start updating a tile
+    _startTile(x, y, width, height, color) {
+        this._tileX = x;
+        this._tileY = y;
+        this._tileW = width;
+        this._tileH = height;
+
+        const red = color[0];
+        const green = color[1];
+        const blue = color[2];
+
+        const data = this._tileBuffer;
+        for (let i = 0; i < width * height * 4; i += 4) {
+            data[i]     = red;
+            data[i + 1] = green;
+            data[i + 2] = blue;
+            data[i + 3] = 255;
+        }
+    }
+
+    // update sub-rectangle of the current tile
+    _subTile(x, y, w, h, color) {
+        const red = color[0];
+        const green = color[1];
+        const blue = color[2];
+        const xend = x + w;
+        const yend = y + h;
+
+        const data = this._tileBuffer;
+        const width = this._tileW;
+        for (let j = y; j < yend; j++) {
+            for (let i = x; i < xend; i++) {
+                const p = (i + (j * width)) * 4;
+                data[p]     = red;
+                data[p + 1] = green;
+                data[p + 2] = blue;
+                data[p + 3] = 255;
+            }
+        }
+    }
+
+    // draw the current tile to the screen
+    _finishTile(display) {
+        display.blitImage(this._tileX, this._tileY,
+                          this._tileW, this._tileH,
+                          this._tileBuffer, 0);
+    }
+}

data/lib/novnc/core/decoders/jpeg.js

@@ -0,0 +1,146 @@
+/*
+ * noVNC: HTML5 VNC client
+ * Copyright (C) 2019 The noVNC Authors
+ * Licensed under MPL 2.0 (see LICENSE.txt)
+ *
+ * See README.md for usage and integration instructions.
+ *
+ */
+
+export default class JPEGDecoder {
+    constructor() {
+        // RealVNC will reuse the quantization tables
+        // and Huffman tables, so we need to cache them.
+        this._cachedQuantTables = [];
+        this._cachedHuffmanTables = [];
+
+        this._segments = [];
+    }
+
+    decodeRect(x, y, width, height, sock, display, depth) {
+        // A rect of JPEG encodings is simply a JPEG file
+        while (true) {
+            let segment = this._readSegment(sock);
+            if (segment === null) {
+                return false;
+            }
+            this._segments.push(segment);
+            // End of image?
+            if (segment[1] === 0xD9) {
+                break;
+            }
+        }
+
+        let huffmanTables = [];
+        let quantTables = [];
+        for (let segment of this._segments) {
+            let type = segment[1];
+            if (type === 0xC4) {
+                // Huffman tables
+                huffmanTables.push(segment);
+            } else if (type === 0xDB) {
+                // Quantization tables
+                quantTables.push(segment);
+            }
+        }
+
+        const sofIndex = this._segments.findIndex(
+            x => x[1] == 0xC0 || x[1] == 0xC2
+        );
+        if (sofIndex == -1) {
+            throw new Error("Illegal JPEG image without SOF");
+        }
+
+        if (quantTables.length === 0) {
+            this._segments.splice(sofIndex+1, 0,
+                                  ...this._cachedQuantTables);
+        }
+        if (huffmanTables.length === 0) {
+            this._segments.splice(sofIndex+1, 0,
+                                  ...this._cachedHuffmanTables);
+        }
+
+        let length = 0;
+        for (let segment of this._segments) {
+            length += segment.length;
+        }
+
+        let data = new Uint8Array(length);
+        length = 0;
+        for (let segment of this._segments) {
+            data.set(segment, length);
+            length += segment.length;
+        }
+
+        display.imageRect(x, y, width, height, "image/jpeg", data);
+
+        if (huffmanTables.length !== 0) {
+            this._cachedHuffmanTables = huffmanTables;
+        }
+        if (quantTables.length !== 0) {
+            this._cachedQuantTables = quantTables;
+        }
+
+        this._segments = [];
+
+        return true;
+    }
+
+    _readSegment(sock) {
+        if (sock.rQwait("JPEG", 2)) {
+            return null;
+        }
+
+        let marker = sock.rQshift8();
+        if (marker != 0xFF) {
+            throw new Error("Illegal JPEG marker received (byte: " +
+                               marker + ")");
+        }
+        let type = sock.rQshift8();
+        if (type >= 0xD0 && type <= 0xD9 || type == 0x01) {
+            // No length after marker
+            return new Uint8Array([marker, type]);
+        }
+
+        if (sock.rQwait("JPEG", 2, 2)) {
+            return null;
+        }
+
+        let length = sock.rQshift16();
+        if (length < 2) {
+            throw new Error("Illegal JPEG length received (length: " +
+                               length + ")");
+        }
+
+        if (sock.rQwait("JPEG", length-2, 4)) {
+            return null;
+        }
+
+        let extra = 0;
+        if (type === 0xDA) {
+            // start of scan
+            extra += 2;
+            while (true) {
+                if (sock.rQwait("JPEG", length-2+extra, 4)) {
+                    return null;
+                }
+                let data = sock.rQpeekBytes(length-2+extra, false);
+                if (data.at(-2) === 0xFF && data.at(-1) !== 0x00 &&
+                    !(data.at(-1) >= 0xD0 && data.at(-1) <= 0xD7)) {
+                    extra -= 2;
+                    break;
+                }
+                extra++;
+            }
+        }
+
+        let segment = new Uint8Array(2 + length + extra);
+        segment[0] = marker;
+        segment[1] = type;
+        segment[2] = length >> 8;
+        segment[3] = length;
+        segment.set(sock.rQshiftBytes(length-2+extra, false), 4);
+
+        return segment;
+    }
+}

data/lib/novnc/core/decoders/raw.js

@@ -0,0 +1,59 @@
+/*
+ * noVNC: HTML5 VNC client
+ * Copyright (C) 2019 The noVNC Authors
+ * Licensed under MPL 2.0 (see LICENSE.txt)
+ *
+ * See README.md for usage and integration instructions.
+ *
+ */
+
+export default class RawDecoder {
+    constructor() {
+        this._lines = 0;
+    }
+
+    decodeRect(x, y, width, height, sock, display, depth) {
+        if ((width === 0) || (height === 0)) {
+            return true;
+        }
+
+        if (this._lines === 0) {
+            this._lines = height;
+        }
+
+        const pixelSize = depth == 8 ? 1 : 4;
+        const bytesPerLine = width * pixelSize;
+
+        while (this._lines > 0) {
+            if (sock.rQwait("RAW", bytesPerLine)) {
+                return false;
+            }
+
+            const curY = y + (height - this._lines);
+
+            let data = sock.rQshiftBytes(bytesPerLine, false);
+
+            // Convert data if needed
+            if (depth == 8) {
+                const newdata = new Uint8Array(width * 4);
+                for (let i = 0; i < width; i++) {
+                    newdata[i * 4 + 0] = ((data[i] >> 0) & 0x3) * 255 / 3;
+                    newdata[i * 4 + 1] = ((data[i] >> 2) & 0x3) * 255 / 3;
+                    newdata[i * 4 + 2] = ((data[i] >> 4) & 0x3) * 255 / 3;
+                    newdata[i * 4 + 3] = 255;
+                }
+                data = newdata;
+            }
+
+            // Max sure the image is fully opaque
+            for (let i = 0; i < width; i++) {
+                data[i * 4 + 3] = 255;
+            }
+
+            display.blitImage(x, curY, width, 1, data, 0);
+            this._lines--;
+        }
+
+        return true;
+    }
+}