@momasiku/dsh-pilot 0.5.12 → 0.5.14

package/README.md

@@ -232,6 +232,18 @@ and what `dsh plugin add` checks before it installs anything. Requirements:
 
 ### Recent changes
 
+**0.5.12** — a cold tree, and an empty list that explained nothing.
+
+- **Chromium builds its accessibility tree lazily**, and the first caller pays for it:
+  measured on the harness window right after a restart, 3 elements at 1.2 s and 77
+  elements at 0.3 s on the next call. A caller that trusts the first answer concludes
+  the window has no tree and falls back to pixels for nothing. `controls` now retries
+  once, 300 ms later, when the worker has not yet seen a real tree and the answer is
+  suspiciously small.
+- **A filtered listing that legitimately matches nothing** now says so, instead of
+  returning an empty list that reads as "this window has no tree":
+  `(no operable controls here; the window has 2 elements - pass meaningfulOnly:false to see them)`.
+
 **0.5.11** — the control tree, and two fixes that make a batch honest.
 
 - **`controls` reads the window's UI Automation tree**, and `target` clicks a listed
@@ -616,6 +628,11 @@ dsh plugin --profile <你的profile> add "file:E:\path\to\dsh-pilot"
 
 ### 最近改动
 
+**0.5.12** —— 冷树，以及一个什么都没说的空列表。
+
+- **Chromium 的无障碍树是懒加载的**，而且由第一个来问的人付账：实测重启后在 1.2 秒时只有 3 个元素，下一次调用 0.3 秒时是 77 个。信任第一个答案的人会认定「这窗口没有树」，白白退回像素。现在 `controls` 在「worker 还没见过完整树、且结果异常少」时会等 300 ms 重试一次。
+- **过滤后确实什么都没匹配到时，现在会明说**，而不是返回一个读起来像「这里没有树」的空列表：`(no operable controls here; the window has 2 elements - pass meaningfulOnly:false to see them)`。
+
 **0.5.11** —— 控件树，外加两个让批量操作「不说谎」的修复。
 
 - **`controls` 读窗口的 UI Automation 树**，`target` 点列表里那个控件的精确矩形，于是有名字的控件不必再去像素里找。它不截图，所以也不付 settle。在 673 个元素的窗口上实测：过滤后 **0.65 秒、约 850 token**（77 个可操作元素），不过滤约 9k。暖 worker 记着它发出去的那份列表，所以某一行换了身份会报错，而不是点到搬进来的那个。

package/lib/index.js

@@ -654,6 +654,11 @@ async function capture(ctx, exec, config, args) {
 			...(ref.name === undefined ? {} : { name: ref.name }),
 			...(ref.originalDimensions === undefined ? {} : { originalDimensions: { ...ref.originalDimensions } })
 		};
+		// Predict where the request pipeline will put this frame, from the same budget
+		// it uses. A prediction, always paired with the deferral to the preview size the
+		// model is actually shown.
+		const projected = projectedRequestSize(Number(result.image.width), Number(result.image.height), limits);
+		if (projected !== undefined) result.requestPreview = projected;
 	} catch (error) {
 		result.imageError = `the frame could not be attached: ${error instanceof Error ? error.message : String(error)}`;
 	}
@@ -736,25 +741,83 @@ function captureContent(value) {
  * @param value - a capture, or a tool result carrying one under `capture`.
  * @returns the contract text for that frame.
  */
+/**
+ * Where the request pipeline will put this frame.
+ *
+ * The store hands back a normalized image, and then the request pipeline shrinks it
+ * again to fit the model's image budget - after this plugin has let go of it, which
+ * is why a frame that really was 1:1 at capture can still reach the model smaller.
+ * The geometry is a pure function of the budget the deployment publishes, so the
+ * pipeline can be predicted instead of described. Mirrors requestImageDimensions()
+ * in @deepseek-ai/dsh-attachment, and kept local so a drift between the two can only
+ * cost accuracy: the contract always defers to the preview size the message states.
+ *
+ * @param width - normalized image width.
+ * @param height - normalized image height.
+ * @param limits - the deployment's ImageAttachmentLimits.
+ * @returns predicted request dimensions plus the budget they came from, or undefined.
+ */
+function projectedRequestSize(width, height, limits) {
+	const maxPixels = Number(limits?.maxImagePixels);
+	const maxDimension = Number(limits?.maxImageDimension);
+	if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return undefined;
+	let w = width;
+	let h = height;
+	if (Number.isFinite(maxDimension) && maxDimension > 0 && Math.max(w, h) > maxDimension) {
+		if (w >= h) { h = Math.max(1, Math.round(maxDimension * h / w)); w = maxDimension; }
+		else { w = Math.max(1, Math.round(maxDimension * w / h)); h = maxDimension; }
+	}
+	if (Number.isFinite(maxPixels) && maxPixels > 0 && w * h > maxPixels) {
+		const s = Math.min(1, Math.sqrt(maxPixels / (w * h)));
+		if (w >= h) {
+			w = Math.max(1, Math.floor(w * s));
+			h = Math.max(1, Math.round(w * height / width));
+			while (w * h > maxPixels && w > 1) { w -= 1; h = Math.max(1, Math.round(w * height / width)); }
+		} else {
+			h = Math.max(1, Math.floor(h * s));
+			w = Math.max(1, Math.round(h * width / height));
+			while (w * h > maxPixels && h > 1) { h -= 1; w = Math.max(1, Math.round(h * width / height)); }
+		}
+	}
+	const out = { width: w, height: h };
+	if (Number.isFinite(maxPixels) && maxPixels > 0) out.maxPixels = maxPixels;
+	return out;
+}
 function coordinateContract(value) {
 	const capture = value?.capture ?? value;
-	const scale = deliveredScale(capture);
-	if (scaleIsOne(scale)) {
+	const width = Number(capture?.imageWidth);
+	const height = Number(capture?.imageHeight);
+	const sizeKnown = Number.isFinite(width) && width > 0 && Number.isFinite(height) && height > 0;
+	const frame = sizeKnown ? `${width}x${height}` : 'the captured frame';
+	const delivered = capture?.image;
+	const preview = capture?.requestPreview;
+	const predicted = Number.isFinite(Number(preview?.width)) && Number(preview.width) > 0 && Number.isFinite(Number(preview?.height)) && Number(preview.height) > 0
+		? { width: Number(preview.width), height: Number(preview.height), maxPixels: Number(preview.maxPixels) }
+		: undefined;
+	const final = predicted ?? (delivered === undefined || delivered === null ? undefined : { width: Number(delivered.width), height: Number(delivered.height) });
+	const kx = final !== undefined && sizeKnown && final.width > 0 ? width / final.width : 1;
+	const ky = final !== undefined && sizeKnown && final.height > 0 ? height / final.height : 1;
+	const oneToOne = Math.abs(kx - 1) < 0.002 && Math.abs(ky - 1) < 0.002;
+	const budgetLine = predicted?.maxPixels > 0 ? `This deployment's image budget is ${predicted.maxPixels} px. ` : '';
+	const guard = 'The message carrying this frame states its own preview size; if it differs from the size named here, multiply every reading by capture_width / preview_width before passing it to desktop_control, because a step after capture decides it.';
+	if (oneToOne) {
 		return [
-			'Image pixel (0,0) is the top-left of this frame. Both halves of this device are per-monitor DPI aware and this frame is delivered at capture size, so one image pixel is one real screen pixel:',
-			'to act on a feature at image (x,y), call desktop_control with exactly x and y. Do not rescale for display scaling.'
-		].join(' ');
-	}
-	if (axesAgree(scale)) {
-		const ratio = (scale.x + scale.y) / 2;
-		return [
-			`This frame was delivered at ${printed(ratio)}x the capture, so image pixels are NOT screen pixels:`,
-			`to act on a feature at image (x,y), multiply both readings by ${printed(1 / ratio)} and pass those physical screen pixels to desktop_control.`
+			`Image pixel (0,0) is the top-left of this frame, which was captured at ${frame} physical screen pixels. Both halves of this device are per-monitor DPI aware.`,
+			`${budgetLine}At that size one image pixel is one real screen pixel: to act on a feature at image (x,y), call desktop_control with exactly x and y.`,
+			guard
 		].join(' ');
 	}
+	if (!sizeKnown || final === undefined) return guard;
+	const axes = Math.abs(kx - ky) <= 0.005 * Math.max(kx, ky);
+	const multiply = axes
+		? `multiply every reading by ${printed(kx)}`
+		: `multiply x readings by ${printed(kx)} and y readings by ${printed(ky)}`;
+	const shown = predicted === undefined ? 'delivered' : 'the request pipeline will show it to you';
 	return [
-		`This frame was delivered at ${printed(scale.x)}x horizontally and ${printed(scale.y)}x vertically of the capture, so image pixels are NOT screen pixels:`,
-		`to act on a feature at image (x,y), multiply the readings by ${printed(1 / scale.x)} and ${printed(1 / scale.y)} (x*${printed(1 / scale.x)}, y*${printed(1 / scale.y)}) and pass those physical screen pixels to desktop_control.`
+		`This frame was captured at ${frame} physical screen pixels and ${shown} at ${final.width}x${final.height} (${printed(final.width / width)}x the capture), so image pixels are NOT screen pixels.`,
+		`${budgetLine}To act on a feature at image (x,y), ${multiply} and pass those physical screen pixels to desktop_control, or read the ruler labels, which are already screen coordinates.`,
+		'A capture whose area is at or under the budget arrives at capture size: framing the window or a region instead of the whole screen gives a true 1:1 frame.',
+		guard
 	].join(' ');
 }
 
@@ -941,6 +1004,17 @@ function windowInfoOf(window) {
 	};
 }
 
+/** Where the request pipeline is predicted to put a frame, and the budget used. */
+const REQUEST_PREVIEW_SCHEMA = {
+	type: 'object',
+	additionalProperties: false,
+	properties: {
+		width: { type: 'integer', required: true },
+		height: { type: 'integer', required: true },
+		maxPixels: { type: 'number' }
+	}
+};
+
 const CAPTURE_SCHEMA = {
 	type: 'object',
 	additionalProperties: false,
@@ -969,6 +1043,7 @@ const CAPTURE_SCHEMA = {
 		frameHash: { type: 'string' },
 		unchanged: { type: 'boolean' },
 		rulers: { type: 'boolean' },
+		requestPreview: REQUEST_PREVIEW_SCHEMA,
 		capturedAt: { type: 'string' }
 	}
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@momasiku/dsh-pilot",
-  "version": "0.5.12",
+  "version": "0.5.14",
   "description": "Desktop vision and control for DeepSeek Harness - the whole Windows machine, any application, not browser automation: screen_view captures the live screen as an image the model can see, desktop_control drives the mouse, keyboard, and windows against the exact same pixel space; desktop_sequence runs a whole batch of actions in one call and looks once at the end.",
   "license": "MIT",
   "author": "momasiku",