@nativescript/cloudflare-desktop 0.1.0-alpha.0 → 0.1.0-alpha.2

package/README.md

@@ -30,7 +30,7 @@ AppKit holds control targets and delegates weakly. Keep them referenced from Jav
 - **Code.** The host joins the project's socket as `platform: 'macos'`, walks each new entry's module graph over HTTP and serves it to Node through `module.registerHooks`. Module URLs are versioned, so only changed modules are fetched. Node can't unload old versions; they stay in memory until the host restarts.
 - **Event loop.** Cocoa events are pumped from Node's timers, so sockets and `fetch` keep working.
 - **Memory.** Each frame's native objects are released as soon as it's sent (`interop.releaseNativeObject`, NativeScript/runtimes#103); runtimes without it get a garbage collection every few frames instead.
-- **Stream.** While a viewer watches, ScreenCaptureKit captures the project's window (only that window, through the current-process listing, macOS 14.4+) at up to 30 fps; frames go out as JPEG over `/p/<id>/stream/<stream>`, a relay separate from the project's hub.
+- **Stream.** While a viewer watches, ScreenCaptureKit captures the project's window (only that window, through the current-process listing, macOS 14.4+) at up to 30 fps and at the window's point size (at most 800 px wide); frames go out as JPEG over `/p/<id>/stream/<stream>`, a relay separate from the project's hub, in chunks of at most 16 KB under a budget of about 4.8 Mbps with adaptive quality. AppKit can repaint a still window many times a second, so a frame identical to the last one sent is skipped. Cloudflare cuts a Durable Object's WebSocket when messages are large or one Mac streams two windows at once, so the IDE only watches a window while its pane is on screen.
 - **Input.** Pointer, wheel and key events from the viewer are posted to the app's own event queue, so they reach only this app, never move the real cursor and need no Accessibility permission. A press is held until its release arrives, because a control's tracking loop blocks the JS thread in between: drags land on release, and press-and-hold isn't possible yet.
 
 ## Publishing

package/dist/stream.d.ts

@@ -14,6 +14,12 @@ export declare class WindowStreamer {
     private ci?;
     private input;
     private frames;
+    private quality;
+    private budget;
+    private budgetAt;
+    private framed;
+    /** The last JPEG sent; repeats are skipped until a viewer joins. */
+    private sent?;
     private backoff;
     private closed;
     private timer?;
@@ -29,6 +35,8 @@ export declare class WindowStreamer {
     private frame;
     /** Objects created along the way go into `held`, released once the frame is sent. */
     private encode;
+    /** Steers JPEG quality so frames stay near the size the budget allows at full frame rate. */
+    private adaptQuality;
     private options?;
     private jpegOptions;
     /** Tells viewers the window's size and title when either changed; true if the size did. */

package/dist/stream.js

@@ -1,11 +1,23 @@
 import v8 from 'node:v8';
 import vm from 'node:vm';
+import { FRAME_CHUNK_BYTES, FRAME_END, FRAME_MORE } from '@nativescript/cloudflare-protocol';
 import { InputInjector } from './input.js';
 const FPS = 30;
-const MAX_PIXEL_WIDTH = 1600;
-const JPEG_QUALITY = 0.7;
+const MAX_PIXEL_WIDTH = 800;
+/**
+ * Outbound budget. Cloudflare cuts a Durable Object's WebSocket (1006) once
+ * sustained inbound traffic passes roughly 7 to 11 Mbps, so the stream stays
+ * well under that: busy windows get fewer frames, never a dropped connection.
+ */
+const BYTES_PER_SECOND = 600_000;
+const TARGET_FRAME_BYTES = BYTES_PER_SECOND / FPS;
+/** JPEG quality adapts within this range to keep frames near the target size. */
+const QUALITY = { min: 0.35, max: 0.6 };
 /** Unsent bytes past which frames are dropped rather than queued behind a slow link. */
 const MAX_BUFFERED = 1_500_000;
+/** ScreenCaptureKit occasionally never completes a start; it is retried after this long without a frame. */
+const START_TIMEOUT_MS = 4000;
+const START_ATTEMPTS = 3;
 const MAX_BACKOFF_MS = 8000;
 /** A viewer reloading the IDE or switching panes is back within this; capture keeps running for it. */
 const STOP_DELAY_MS = 4000;
@@ -71,6 +83,12 @@ export class WindowStreamer {
     ci;
     input = new InputInjector();
     frames = 0;
+    quality = QUALITY.max;
+    budget = BYTES_PER_SECOND;
+    budgetAt = 0;
+    framed = false;
+    /** The last JPEG sent; repeats are skipped until a viewer joins. */
+    sent;
     backoff = 500;
     closed = false;
     timer;
@@ -96,6 +114,8 @@ export class WindowStreamer {
                 return;
             }
             if (msg.type === 'viewers') {
+                if (msg.count > this.viewers)
+                    this.sent = undefined;
                 this.viewers = msg.count;
                 void this.sync();
             }
@@ -108,7 +128,9 @@ export class WindowStreamer {
                 }
             }
         };
-        ws.onclose = () => {
+        ws.onclose = (e) => {
+            if (process.env.NS_EDGE_DEBUG)
+                console.error(`[ns-edge desktop] stream socket closed: ${e.code} ${e.reason} (${ws.bufferedAmount} bytes unsent, ${this.frames} frames)`);
             if (this.ws !== ws)
                 return;
             this.viewers = 0;
@@ -148,11 +170,12 @@ export class WindowStreamer {
             }, STOP_DELAY_MS);
         }
     }
-    async start() {
+    async start(attempt = 1) {
         const window = projectWindow();
         if (!window)
             return;
         this.starting = true;
+        let retry = false;
         try {
             const target = await ownWindow(window.windowNumber);
             if (!target || this.viewers === 0)
@@ -172,15 +195,27 @@ export class WindowStreamer {
                 throw new Error('ScreenCaptureKit refused the stream output');
             // Set before starting: the first frame, the only one a still window produces, can arrive before the start callback.
             this.capture = { stream, output, window, size: '' };
+            this.framed = false;
             this.sendSize();
+            const started = new Promise((resolve, reject) => stream.startCaptureWithCompletionHandler((err) => (err ? reject(new Error(err.localizedDescription)) : resolve(true))));
+            let completed;
             try {
-                await new Promise((resolve, reject) => stream.startCaptureWithCompletionHandler((err) => (err ? reject(new Error(err.localizedDescription)) : resolve())));
+                completed = await Promise.race([started, new Promise((r) => setTimeout(() => r(false), START_TIMEOUT_MS))]);
             }
             catch (err) {
                 if (this.capture?.stream === stream)
                     this.capture = undefined;
                 throw err;
             }
+            if (!completed && !this.framed) {
+                if (this.capture?.stream === stream)
+                    this.capture = undefined;
+                stream.stopCaptureWithCompletionHandler(null);
+                retry = attempt < START_ATTEMPTS;
+                if (!retry)
+                    throw new Error(`ScreenCaptureKit did not start after ${START_ATTEMPTS} attempts`);
+                return;
+            }
             this.onState?.(true, this.viewers);
         }
         catch (err) {
@@ -188,6 +223,8 @@ export class WindowStreamer {
         }
         finally {
             this.starting = false;
+            if (retry && this.viewers > 0)
+                await this.start(attempt + 1);
         }
     }
     stop() {
@@ -221,12 +258,16 @@ export class WindowStreamer {
             config.ignoreShadowsSingleWindow = true;
             capture.stream.updateConfigurationCompletionHandler(config, null);
         }
-        if (ws.bufferedAmount > MAX_BUFFERED)
+        const now = performance.now();
+        this.budget = Math.min(BYTES_PER_SECOND, this.budget + ((now - this.budgetAt) * BYTES_PER_SECOND) / 1000);
+        this.budgetAt = now;
+        if (this.budget < 0 || ws.bufferedAmount > MAX_BUFFERED)
             return;
         const pixels = CMSampleBufferGetImageBuffer(sampleBuffer);
         // Frames where nothing changed carry no image.
         if (!pixels)
             return;
+        this.framed = true;
         held.push(pixels);
         this.ci ??= CIContext.context();
         const image = CIImage.imageWithCVPixelBuffer(pixels);
@@ -235,12 +276,38 @@ export class WindowStreamer {
         if (!jpeg)
             return;
         held.push(jpeg);
-        // A copy: the buffer is backed by the NSData, which is released right after.
-        ws.send(new Uint8Array(interop.bufferFromData(jpeg)).slice());
+        // Chunks are copies: the bytes are backed by the NSData, which is released right after.
+        const bytes = new Uint8Array(interop.bufferFromData(jpeg));
+        // AppKit can repaint a still window many times a second, and ScreenCaptureKit
+        // reports each repaint as a complete frame with the whole window dirty.
+        if (this.sent && sameBytes(this.sent, bytes))
+            return;
+        this.sent = bytes;
+        for (let at = 0; at < bytes.byteLength; at += FRAME_CHUNK_BYTES) {
+            const payload = bytes.subarray(at, at + FRAME_CHUNK_BYTES);
+            const chunk = new Uint8Array(payload.byteLength + 1);
+            chunk[0] = at + FRAME_CHUNK_BYTES >= bytes.byteLength ? FRAME_END : FRAME_MORE;
+            chunk.set(payload, 1);
+            ws.send(chunk);
+        }
+        this.budget -= bytes.byteLength;
+        this.adaptQuality(bytes.byteLength);
+    }
+    /** Steers JPEG quality so frames stay near the size the budget allows at full frame rate. */
+    adaptQuality(bytes) {
+        const next = bytes > TARGET_FRAME_BYTES * 1.2
+            ? Math.max(QUALITY.min, this.quality - 0.05)
+            : bytes < TARGET_FRAME_BYTES * 0.6
+                ? Math.min(QUALITY.max, this.quality + 0.02)
+                : this.quality;
+        if (next !== this.quality) {
+            this.quality = next;
+            this.options = undefined;
+        }
     }
     options;
     jpegOptions() {
-        return (this.options ??= NSDictionary.dictionaryWithObjectForKey(NSNumber.numberWithDouble(JPEG_QUALITY), kCGImageDestinationLossyCompressionQuality));
+        return (this.options ??= NSDictionary.dictionaryWithObjectForKey(NSNumber.numberWithDouble(this.quality), kCGImageDestinationLossyCompressionQuality));
     }
     /** Tells viewers the window's size and title when either changed; true if the size did. */
     sendSize() {
@@ -260,10 +327,22 @@ export class WindowStreamer {
         return resized;
     }
 }
-/** Output pixels: the window's backing size, capped so a large Retina window stays a reasonable stream. */
+function sameBytes(a, b) {
+    if (a.byteLength !== b.byteLength)
+        return false;
+    for (let i = 0; i < a.byteLength; i++)
+        if (a[i] !== b[i])
+            return false;
+    return true;
+}
+/**
+ * Output pixels: the window's size in points, not its Retina backing size.
+ * Viewers show it in a side pane, and 2x frames are four times the bytes,
+ * which halves the frame rate on a typical uplink.
+ */
 function applySize(config, window) {
     const { width, height } = window.frame.size;
-    const scale = Math.min(window.backingScaleFactor || 1, MAX_PIXEL_WIDTH / Math.max(width, 1));
+    const scale = Math.min(1, MAX_PIXEL_WIDTH / Math.max(width, 1));
     config.width = Math.max(2, Math.round(width * scale));
     config.height = Math.max(2, Math.round(height * scale));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nativescript/cloudflare-desktop",
-  "version": "0.1.0-alpha.0",
+  "version": "0.1.0-alpha.2",
   "description": "Runs an ns-edge project as a macOS desktop app on the NativeScript Node-API runtime, hot-reloaded from the edge and streamed to the browser IDE",
   "license": "Apache-2.0",
   "type": "module",
@@ -29,7 +29,7 @@
     "metadata": "node scripts/metadata.mjs"
   },
   "dependencies": {
-    "@nativescript/cloudflare-protocol": "0.1.0-beta.1",
+    "@nativescript/cloudflare-protocol": "0.1.0-beta.2",
     "@nativescript/macos-node-api": "0.4.4-next.2026-08-09-31292056208",
     "es-module-lexer": "^3.0.4"
   },