node-mac-recorder 2.22.15 → 2.22.16

package/lib/cursorCapture/polling.js

@@ -0,0 +1,404 @@
+"use strict";
+
+const fs = require("fs");
+const { resolveCursorDisplayInfo } = require("./displayInfo");
+
+const TEXT_INPUT_SAMPLE_MS = 95;
+
+function shouldCaptureCursorSample(lastCapturedData, currentData) {
+	if (!lastCapturedData) {
+		return true;
+	}
+	const last = lastCapturedData;
+	if (currentData.type !== last.type) {
+		return true;
+	}
+	if (
+		Math.abs(currentData.x - last.x) >= 2 ||
+		Math.abs(currentData.y - last.y) >= 2
+	) {
+		return true;
+	}
+	if (currentData.cursorType !== last.cursorType) {
+		return true;
+	}
+	return false;
+}
+
+function transformGlobalToVideo(globalX, globalY, d) {
+	if (!d || !d.videoRelative) {
+		return {
+			x: globalX,
+			y: globalY,
+			coordinateSystem: "global",
+			outsideVideo: false,
+		};
+	}
+	const displayRelativeX = globalX - d.displayX;
+	const displayRelativeY = globalY - d.displayY;
+	const x = displayRelativeX - d.videoOffsetX;
+	const y = displayRelativeY - d.videoOffsetY;
+	const outsideVideo =
+		x < 0 ||
+		y < 0 ||
+		x >= d.videoWidth ||
+		y >= d.videoHeight;
+	return {
+		x,
+		y,
+		coordinateSystem: outsideVideo
+			? "video-relative-outside"
+			: "video-relative",
+		outsideVideo,
+	};
+}
+
+function transformInputFrameGlobal(ifr, d) {
+	if (!ifr || typeof ifr !== "object") {
+		return {};
+	}
+	const ox = Number(ifr.x);
+	const oy = Number(ifr.y);
+	const tw = transformGlobalToVideo(ox, oy, d);
+	return {
+		x: tw.x,
+		y: tw.y,
+		width: Number(ifr.width) || 0,
+		height: Number(ifr.height) || 0,
+	};
+}
+
+function tryAppendTextInput(
+	recorder,
+	nativeBinding,
+	filepath,
+	position,
+	timestamp,
+) {
+	if (typeof nativeBinding.getTextInputSnapshot !== "function") {
+		return;
+	}
+	const wall = Date.now();
+	if (wall - (recorder._tiSampleWallMs || 0) < TEXT_INPUT_SAMPLE_MS) {
+		return;
+	}
+	recorder._tiSampleWallMs = wall;
+
+	let snap = null;
+	try {
+		snap = nativeBinding.getTextInputSnapshot();
+	} catch {
+		return;
+	}
+	if (
+		!snap ||
+		!Number.isFinite(snap.caretX) ||
+		!Number.isFinite(snap.caretY)
+	) {
+		return;
+	}
+
+	const d = recorder.cursorDisplayInfo;
+	const caretT = transformGlobalToVideo(snap.caretX, snap.caretY, d);
+	const mouseGX = position.x;
+	const mouseGY = position.y;
+	const mouseT = transformGlobalToVideo(mouseGX, mouseGY, d);
+
+	const inputFrameVid = transformInputFrameGlobal(snap.inputFrame, d);
+
+	const tiRow = {
+		x: mouseT.x,
+		y: mouseT.y,
+		timestamp,
+		unixTimeMs: wall,
+		cursorType: "text",
+		type: "textInput",
+		caretX: caretT.x,
+		caretY: caretT.y,
+		inputFrame: inputFrameVid,
+		coordinateSystem: caretT.coordinateSystem,
+		recordingType: d?.recordingType || "display",
+		videoInfo: d
+			? {
+					width: d.videoWidth,
+					height: d.videoHeight,
+					offsetX: d.videoOffsetX,
+					offsetY: d.videoOffsetY,
+				}
+			: {},
+		displayInfo: d
+			? {
+					displayId: d.displayId,
+					width: d.displayWidth,
+					height: d.displayHeight,
+				}
+			: {},
+	};
+
+	if (
+		recorder.cursorCaptureFirstWrite &&
+		recorder.cursorCaptureSessionTimestamp
+	) {
+		tiRow._syncMetadata = {
+			videoStartTime: recorder.cursorCaptureSessionTimestamp,
+			cursorStartTime: recorder.cursorCaptureStartTime,
+			offsetMs:
+				recorder.cursorCaptureStartTime -
+				recorder.cursorCaptureSessionTimestamp,
+		};
+	}
+
+	const le = recorder._lastTextInputEmitted;
+	if (
+		le &&
+		Math.abs(le.caretX - tiRow.caretX) < 0.75 &&
+		Math.abs(le.caretY - tiRow.caretY) < 0.75 &&
+		timestamp - le.timestamp < 220
+	) {
+		return;
+	}
+	recorder._lastTextInputEmitted = {
+		caretX: tiRow.caretX,
+		caretY: tiRow.caretY,
+		timestamp,
+	};
+
+	const jsonString = JSON.stringify(tiRow);
+	if (recorder.cursorCaptureFirstWrite) {
+		fs.appendFileSync(filepath, jsonString);
+		recorder.cursorCaptureFirstWrite = false;
+	} else {
+		fs.appendFileSync(filepath, "," + jsonString);
+	}
+}
+
+async function startCursorCapture(recorder, nativeBinding, intervalOrFilepath, options = {}) {
+	let filepath;
+	let interval = 20;
+
+	if (typeof intervalOrFilepath === "number") {
+		interval = Math.max(10, intervalOrFilepath);
+		filepath = `cursor-data-${Date.now()}.json`;
+	} else if (typeof intervalOrFilepath === "string") {
+		filepath = intervalOrFilepath;
+	} else {
+		throw new Error("Parameter must be interval (number) or filepath (string)");
+	}
+
+	if (recorder.cursorCaptureInterval) {
+		throw new Error("Cursor capture is already running");
+	}
+
+	const syncStartTime = options.startTimestamp || Date.now();
+
+	if (options.multiWindowBounds && options.multiWindowBounds.length > 0) {
+		try {
+			const allWindows = await recorder.getWindows();
+			for (const windowInfo of options.multiWindowBounds) {
+				const windowData = allWindows.find(
+					(w) => w.id === windowInfo.windowId,
+				);
+				if (windowData) {
+					windowInfo.bounds = {
+						x: windowData.x || 0,
+						y: windowData.y || 0,
+						width: windowData.width || 0,
+						height: windowData.height || 0,
+					};
+				}
+			}
+		} catch (error) {
+			console.warn(
+				"Failed to fetch window bounds for multi-window cursor tracking:",
+				error.message,
+			);
+		}
+	}
+
+	await resolveCursorDisplayInfo(recorder, options);
+
+	return new Promise((resolve, reject) => {
+		try {
+			fs.writeFileSync(filepath, "[");
+
+			recorder.cursorCaptureFile = filepath;
+			recorder.cursorCaptureStartTime = syncStartTime;
+			recorder.cursorCaptureFirstWrite = true;
+			recorder.lastCapturedData = null;
+			recorder.cursorCaptureSessionTimestamp = recorder.sessionTimestamp;
+			recorder._tiSampleWallMs = 0;
+			recorder._lastTextInputEmitted = null;
+
+			recorder.cursorCaptureInterval = setInterval(() => {
+				try {
+					const position = nativeBinding.getCursorPosition();
+					const timestamp =
+						Date.now() - recorder.cursorCaptureStartTime;
+
+					let x = position.x;
+					let y = position.y;
+					let coordinateSystem = "global";
+
+					const di = recorder.cursorDisplayInfo;
+					if (di && di.videoRelative) {
+						const displayRelativeX = position.x - di.displayX;
+						const displayRelativeY = position.y - di.displayY;
+						x = displayRelativeX - di.videoOffsetX;
+						y = displayRelativeY - di.videoOffsetY;
+						coordinateSystem = "video-relative";
+						const outsideVideo =
+							x < 0 ||
+							y < 0 ||
+							x >= di.videoWidth ||
+							y >= di.videoHeight;
+						if (outsideVideo) {
+							coordinateSystem = "video-relative-outside";
+						}
+					}
+
+					const cursorData = {
+						x,
+						y,
+						timestamp,
+						unixTimeMs: Date.now(),
+						cursorType: position.cursorType,
+						type: position.eventType || "move",
+						coordinateSystem,
+						recordingType: di?.recordingType || "display",
+						videoInfo: di
+							? {
+									width: di.videoWidth,
+									height: di.videoHeight,
+									offsetX: di.videoOffsetX,
+									offsetY: di.videoOffsetY,
+								}
+							: {},
+						displayInfo: di
+							? {
+									displayId: di.displayId,
+									width: di.displayWidth,
+									height: di.displayHeight,
+								}
+							: {},
+					};
+
+					if (di?.multiWindowBounds && di.multiWindowBounds.length > 0) {
+						const location = { hover: null, click: null };
+						let windowRelativeCoords = null;
+						for (const windowInfo of di.multiWindowBounds) {
+							if (windowInfo.bounds) {
+								const { x: wx, y: wy, width: ww, height: wh } =
+									windowInfo.bounds;
+								if (
+									position.x >= wx &&
+									position.x <= wx + ww &&
+									position.y >= wy &&
+									position.y <= wy + wh
+								) {
+									location.hover = windowInfo.windowId;
+									windowRelativeCoords = {
+										windowId: windowInfo.windowId,
+										x: position.x - wx,
+										y: position.y - wy,
+										windowWidth: ww,
+										windowHeight: wh,
+									};
+									const eventType = position.eventType || "";
+									if (
+										eventType === "mousedown" ||
+										eventType === "mouseup" ||
+										eventType === "drag" ||
+										eventType === "rightmousedown" ||
+										eventType === "rightmouseup" ||
+										eventType === "rightdrag"
+									) {
+										location.click = windowInfo.windowId;
+									}
+									break;
+								}
+							}
+						}
+						cursorData.location = location;
+						if (windowRelativeCoords) {
+							cursorData.windowRelative = windowRelativeCoords;
+						}
+					}
+
+					if (
+						recorder.cursorCaptureFirstWrite &&
+						recorder.cursorCaptureSessionTimestamp
+					) {
+						cursorData._syncMetadata = {
+							videoStartTime: recorder.cursorCaptureSessionTimestamp,
+							cursorStartTime: recorder.cursorCaptureStartTime,
+							offsetMs:
+								recorder.cursorCaptureStartTime -
+								recorder.cursorCaptureSessionTimestamp,
+						};
+					}
+
+					if (shouldCaptureCursorSample(recorder.lastCapturedData, cursorData)) {
+						const jsonString = JSON.stringify(cursorData);
+						if (recorder.cursorCaptureFirstWrite) {
+							fs.appendFileSync(filepath, jsonString);
+							recorder.cursorCaptureFirstWrite = false;
+						} else {
+							fs.appendFileSync(filepath, "," + jsonString);
+						}
+						recorder.lastCapturedData = { ...cursorData };
+					}
+
+					tryAppendTextInput(
+						recorder,
+						nativeBinding,
+						filepath,
+						position,
+						timestamp,
+					);
+				} catch (error) {
+					console.error("Cursor capture error:", error);
+				}
+			}, interval);
+
+			recorder.emit("cursorCaptureStarted", filepath);
+			resolve(true);
+		} catch (error) {
+			reject(error);
+		}
+	});
+}
+
+async function stopCursorCapture(recorder) {
+	return new Promise((resolve, reject) => {
+		try {
+			if (!recorder.cursorCaptureInterval) {
+				return resolve(false);
+			}
+			clearInterval(recorder.cursorCaptureInterval);
+			recorder.cursorCaptureInterval = null;
+
+			if (recorder.cursorCaptureFile) {
+				fs.appendFileSync(recorder.cursorCaptureFile, "]");
+				recorder.cursorCaptureFile = null;
+			}
+
+			recorder.lastCapturedData = null;
+			recorder.cursorCaptureStartTime = null;
+			recorder.cursorCaptureFirstWrite = true;
+			recorder.cursorDisplayInfo = null;
+			recorder._tiSampleWallMs = 0;
+			recorder._lastTextInputEmitted = null;
+
+			recorder.emit("cursorCaptureStopped");
+			resolve(true);
+		} catch (error) {
+			reject(error);
+		}
+	});
+}
+
+module.exports = {
+	startCursorCapture,
+	stopCursorCapture,
+	shouldCaptureCursorSample,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "node-mac-recorder",
-	"version": "2.22.15",
+	"version": "2.22.16",
 	"description": "Native macOS screen recording package for Node.js applications",
 	"main": "index.js",
 	"keywords": [

package/src/cursor_tracker.mm

@@ -7,6 +7,7 @@
 #import <Accessibility/Accessibility.h>
 #import <dispatch/dispatch.h>
 #import "logging.h"
+#import "text_input_ax_snapshot.h"
 #include <vector>
 #include <math.h>
 
@@ -1948,125 +1949,25 @@ void writeToFile(NSDictionary *cursorData) {
 
 // Text input event: Klavye basıldığında focused text field'in caret pozisyonunu yakala
 static void emitTextInputEvent(NSTimeInterval timestamp, NSTimeInterval unixTimeMs, CGPoint mouseLocation) {
-    // Throttle: Çok sık emit etme (performans için)
     if (unixTimeMs - g_lastTextInputEmitTime < TEXT_INPUT_THROTTLE_MS) {
         return;
     }
 
-    @autoreleasepool {
-        @try {
-            AXUIElementRef systemWide = AXUIElementCreateSystemWide();
-            if (!systemWide) return;
-
-            AXUIElementRef focusedElement = NULL;
-            AXError focusErr = AXUIElementCopyAttributeValue(
-                systemWide, kAXFocusedUIElementAttribute, (CFTypeRef *)&focusedElement);
-
-            if (focusErr != kAXErrorSuccess || !focusedElement) {
-                CFRelease(systemWide);
-                return;
-            }
-
-            NSString *role = CopyAttributeString(focusedElement, kAXRoleAttribute);
-            BOOL isEditable = NO;
-            CopyAttributeBoolean(focusedElement, CFSTR("AXEditable"), &isEditable);
-
-            CFTypeRef selectedRangeValue = NULL;
-            AXError rangeProbeErr = AXUIElementCopyAttributeValue(
-                focusedElement, CFSTR("AXSelectedTextRange"), (CFTypeRef *)&selectedRangeValue);
-            BOOL hasTextRange = (rangeProbeErr == kAXErrorSuccess && selectedRangeValue != NULL);
-
-            // Electron/Chromium: gerçek yazma genelde AXSelectedTextRange ile gelir; AXWebArea tek başına tüm sayfa olabilir.
-            BOOL isStandardTextRole = StringEqualsAny(role, @[
-                @"AXTextField", @"AXTextArea", @"AXTextView",
-                @"AXTextEditor", @"AXSearchField",
-                @"AXComboBox"
-            ]);
-            BOOL isWebAreaWithCaret = [role isEqualToString:@"AXWebArea"] && hasTextRange;
-            BOOL isTextField = isStandardTextRole || isEditable || isWebAreaWithCaret || hasTextRange;
-
-            if (!isTextField) {
-                if (selectedRangeValue) CFRelease(selectedRangeValue);
-                CFRelease(focusedElement);
-                CFRelease(systemWide);
-                return;
-            }
-
-            // Input frame bilgisini al (AXPosition + AXSize)
-            CGPoint inputOrigin = CGPointZero;
-            CGSize inputSize = CGSizeZero;
-            AXValueRef positionValue = NULL;
-            AXValueRef sizeValue = NULL;
-
-            AXUIElementCopyAttributeValue(focusedElement, kAXPositionAttribute, (CFTypeRef *)&positionValue);
-            AXUIElementCopyAttributeValue(focusedElement, kAXSizeAttribute, (CFTypeRef *)&sizeValue);
-
-            if (positionValue) {
-                AXValueGetValue(positionValue, kAXValueTypeCGPoint, &inputOrigin);
-                CFRelease(positionValue);
-            }
-            if (sizeValue) {
-                AXValueGetValue(sizeValue, kAXValueTypeCGSize, &inputSize);
-                CFRelease(sizeValue);
-            }
-
-            // Caret pozisyonunu al (AXSelectedTextRange → AXBoundsForRange)
-            CGPoint caretPos = CGPointMake(inputOrigin.x, inputOrigin.y);
-            BOOL hasCaretPos = NO;
-
-            if (selectedRangeValue) {
-                CFTypeRef boundsValue = NULL;
-                AXError boundsErr = AXUIElementCopyParameterizedAttributeValue(
-                    focusedElement, CFSTR("AXBoundsForRange"),
-                    selectedRangeValue, &boundsValue);
-
-                if (boundsErr == kAXErrorSuccess && boundsValue) {
-                    CGRect caretBounds = CGRectZero;
-                    if (AXValueGetValue((AXValueRef)boundsValue, kAXValueTypeCGRect, &caretBounds)) {
-                        caretPos = CGPointMake(
-                            caretBounds.origin.x,
-                            caretBounds.origin.y + caretBounds.size.height / 2.0
-                        );
-                        hasCaretPos = YES;
-                    }
-                    CFRelease(boundsValue);
-                }
-                CFRelease(selectedRangeValue);
-                selectedRangeValue = NULL;
-            }
-
-            // Caret alınamazsa input frame'in sol ortasını kullan
-            if (!hasCaretPos) {
-                caretPos = CGPointMake(inputOrigin.x + 4, inputOrigin.y + inputSize.height / 2.0);
-            }
-
-            // textInput event'i oluştur ve dosyaya yaz
-            NSDictionary *textInputInfo = @{
-                @"x": @((int)mouseLocation.x),
-                @"y": @((int)mouseLocation.y),
-                @"timestamp": @(timestamp),
-                @"unixTimeMs": @(unixTimeMs),
-                @"cursorType": @"text",
-                @"type": @"textInput",
-                @"caretX": @((int)caretPos.x),
-                @"caretY": @((int)caretPos.y),
-                @"inputFrame": @{
-                    @"x": @((int)inputOrigin.x),
-                    @"y": @((int)inputOrigin.y),
-                    @"width": @((int)inputSize.width),
-                    @"height": @((int)inputSize.height)
-                }
-            };
+    NSDictionary *snap = MRTextInputSnapshotDictionary();
+    if (!snap) {
+        return;
+    }
 
-            writeToFile(textInputInfo);
-            g_lastTextInputEmitTime = unixTimeMs;
+    NSMutableDictionary *textInputInfo = [NSMutableDictionary dictionaryWithDictionary:snap];
+    textInputInfo[@"x"] = @((int)mouseLocation.x);
+    textInputInfo[@"y"] = @((int)mouseLocation.y);
+    textInputInfo[@"timestamp"] = @(timestamp);
+    textInputInfo[@"unixTimeMs"] = @(unixTimeMs);
+    textInputInfo[@"cursorType"] = @"text";
+    textInputInfo[@"type"] = @"textInput";
 
-            CFRelease(focusedElement);
-            CFRelease(systemWide);
-        } @catch (NSException *exception) {
-            // Accessibility hata verirse sessizce devam et
-        }
-    }
+    writeToFile(textInputInfo);
+    g_lastTextInputEmitTime = unixTimeMs;
 }
 
 // Event callback for mouse events
@@ -2677,6 +2578,33 @@ Napi::Value GetCursorDebugInfo(const Napi::CallbackInfo& info) {
     }
 }
 
+static Napi::Value DictToNapiTextInputSnapshot(Napi::Env env, NSDictionary *snap) {
+    Napi::Object o = Napi::Object::New(env);
+    NSNumber *cx = snap[@"caretX"];
+    NSNumber *cy = snap[@"caretY"];
+    o.Set("caretX", Napi::Number::New(env, cx ? [cx doubleValue] : 0));
+    o.Set("caretY", Napi::Number::New(env, cy ? [cy doubleValue] : 0));
+    NSDictionary *frame = snap[@"inputFrame"];
+    Napi::Object fo = Napi::Object::New(env);
+    if ([frame isKindOfClass:[NSDictionary class]]) {
+        fo.Set("x", Napi::Number::New(env, [frame[@"x"] doubleValue]));
+        fo.Set("y", Napi::Number::New(env, [frame[@"y"] doubleValue]));
+        fo.Set("width", Napi::Number::New(env, [frame[@"width"] doubleValue]));
+        fo.Set("height", Napi::Number::New(env, [frame[@"height"] doubleValue]));
+    }
+    o.Set("inputFrame", fo);
+    return o;
+}
+
+static Napi::Value GetTextInputSnapshot(const Napi::CallbackInfo& info) {
+    Napi::Env env = info.Env();
+    NSDictionary *snap = MRTextInputSnapshotDictionary();
+    if (!snap) {
+        return env.Null();
+    }
+    return DictToNapiTextInputSnapshot(env, snap);
+}
+
 // Export functions
 Napi::Object InitCursorTracker(Napi::Env env, Napi::Object exports) {
     exports.Set("startCursorTracking", Napi::Function::New(env, StartCursorTracking));
@@ -2684,6 +2612,7 @@ Napi::Object InitCursorTracker(Napi::Env env, Napi::Object exports) {
     exports.Set("getCursorPosition", Napi::Function::New(env, GetCursorPosition));
     exports.Set("getCursorTrackingStatus", Napi::Function::New(env, GetCursorTrackingStatus));
     exports.Set("getCursorDebugInfo", Napi::Function::New(env, GetCursorDebugInfo));
+    exports.Set("getTextInputSnapshot", Napi::Function::New(env, GetTextInputSnapshot));
 
     return exports;
 } 

package/src/electron_safe/cursor_tracker_electron.mm

@@ -2,6 +2,7 @@
 #import <CoreGraphics/CoreGraphics.h>
 #import <AppKit/AppKit.h>
 #import "../logging.h"
+#import "../text_input_ax_snapshot.h"
 
 // Thread-safe cursor tracking for Electron
 static dispatch_queue_t g_cursorQueue = nil;
@@ -92,12 +93,45 @@ Napi::Value GetCursorPositionElectronSafe(const Napi::CallbackInfo& info) {
     }
 }
 
+static Napi::Value DictToNapiTextInputSnapshotElectron(Napi::Env env, NSDictionary *snap) {
+    Napi::Object o = Napi::Object::New(env);
+    NSNumber *cx = snap[@"caretX"];
+    NSNumber *cy = snap[@"caretY"];
+    o.Set("caretX", Napi::Number::New(env, cx ? [cx doubleValue] : 0));
+    o.Set("caretY", Napi::Number::New(env, cy ? [cy doubleValue] : 0));
+    NSDictionary *frame = snap[@"inputFrame"];
+    Napi::Object fo = Napi::Object::New(env);
+    if ([frame isKindOfClass:[NSDictionary class]]) {
+        fo.Set("x", Napi::Number::New(env, [frame[@"x"] doubleValue]));
+        fo.Set("y", Napi::Number::New(env, [frame[@"y"] doubleValue]));
+        fo.Set("width", Napi::Number::New(env, [frame[@"width"] doubleValue]));
+        fo.Set("height", Napi::Number::New(env, [frame[@"height"] doubleValue]));
+    }
+    o.Set("inputFrame", fo);
+    return o;
+}
+
+Napi::Value GetTextInputSnapshotElectronSafe(const Napi::CallbackInfo& info) {
+    Napi::Env env = info.Env();
+    @try {
+        NSDictionary *snap = MRTextInputSnapshotDictionary();
+        if (!snap) {
+            return env.Null();
+        }
+        return DictToNapiTextInputSnapshotElectron(env, snap);
+    } @catch (NSException *e) {
+        NSLog(@"❌ getTextInputSnapshot: %@", e.reason);
+        return env.Null();
+    }
+}
+
 // Initialize cursor tracker module
 Napi::Object InitCursorTrackerElectron(Napi::Env env, Napi::Object exports) {
     @try {
         initializeCursorQueue();
         
         exports.Set("getCursorPosition", Napi::Function::New(env, GetCursorPositionElectronSafe));
+        exports.Set("getTextInputSnapshot", Napi::Function::New(env, GetTextInputSnapshotElectronSafe));
         
         MRLog(@"✅ Electron-safe cursor tracker initialized");
         return exports;

package/src/text_input_ax_snapshot.h

@@ -0,0 +1,3 @@
+#import <Foundation/Foundation.h>
+
+NSDictionary *_Nullable MRTextInputSnapshotDictionary(void);