dsh-click 0.1.4 → 0.3.0

package/lib/types/index.d.ts

@@ -27,13 +27,17 @@ export { HelperBackend } from './platform/runner.js';
 export { allTools, type ToolServices } from './tools.js';
 export { OBSERVED_EVENT, ACTION_EVENT, type ObservedEvent, type ActionEvent, type ProcessFacts } from './events.js';
 export { sanitizeText, sanitizePath, redactSensitive, sanitizeVisible } from './sanitize.js';
+export { TesseractOcrProvider, UnavailableOcrProvider, parseTesseractTsv, probeTesseract, tesseractArgv, type OcrBox, type OcrProvider, type OcrWord, } from './vision/ocr.js';
+export { UnavailableGroundingProvider, type GroundingBox, type GroundingTarget, type VisualGroundingProvider, } from './vision/grounding.js';
+export { centerOf, screenBoxFor } from './vision/index.js';
 /**
- * Mount the plugin: resolve config, select the platform backend, and register
+ * Mount the plugin: resolve config, select the platform backend, probe the
+ * optional OCR provider, read the optional visual-grounding seam, and register
  * every tool through `ctx.tools.register` (each registration is an effect
  * whose disposer removes exactly that tool on stop/HMR).
  *
  * @param ctx - context carrying the tools registry.
  * @param config - raw loader config; defaults applied through {@link resolveConfig}.
  */
-export declare function apply(ctx: Context, config: Config): void;
+export declare function apply(ctx: Context, config: Config): Promise<void>;
 //# sourceMappingURL=index.d.ts.map

package/lib/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAIlD,OAAO,EAAE,MAAM,EAAiB,MAAM,aAAa,CAAA;AAMnD,eAAO,MAAM,IAAI,cAAc,CAAA;AAE/B,oEAAoE;AACpE,eAAO,MAAM,MAAM,UAAY,CAAA;AAE/B,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,KAAK,iBAAiB,EAAE,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC/G,OAAO,EAAE,cAAc,EAAE,KAAK,kBAAkB,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAA;AACzF,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,KAAK,aAAa,EAAE,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAA;AACnH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAE5F;;;;;;;GAOG;AACH,wBAAgB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAYxD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAKlD,OAAO,EAAE,MAAM,EAAiB,MAAM,aAAa,CAAA;AAQnD,eAAO,MAAM,IAAI,cAAc,CAAA;AAE/B,oEAAoE;AACpE,eAAO,MAAM,MAAM,UAAY,CAAA;AAE/B,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,KAAK,iBAAiB,EAAE,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC/G,OAAO,EAAE,cAAc,EAAE,KAAK,kBAAkB,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAA;AACzF,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,KAAK,aAAa,EAAE,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAA;AACnH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC5F,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,iBAAiB,EACjB,cAAc,EACd,aAAa,EACb,KAAK,MAAM,EACX,KAAK,WAAW,EAChB,KAAK,OAAO,GACb,MAAM,iBAAiB,CAAA;AACxB,OAAO,EACL,4BAA4B,EAC5B,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,uBAAuB,GAC7B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAE1D;;;;;;;;GAQG;AACH,wBAAsB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA4BvE"}

package/lib/types/tools.d.ts

@@ -14,6 +14,7 @@ import type { Context } from '@deepseek-ai/cordis';
 import { ActionExecutor } from './actions.js';
 import type { ResolvedConfig } from './config.js';
 import { ObservationStore } from './observe.js';
+import { type OcrProvider, type VisualGroundingProvider } from './vision/index.js';
 import type { DesktopBackend, Rect } from './platform/types.js';
 /** Everything one tool needs at runtime; injected by `src/index.ts`. */
 export interface ToolServices {
@@ -27,6 +28,10 @@ export interface ToolServices {
     readonly observations: ObservationStore;
     /** The action executor. */
     readonly actions: ActionExecutor;
+    /** The probed OCR provider (optional). */
+    readonly ocr: OcrProvider;
+    /** The optional visual grounding provider. */
+    readonly grounding: VisualGroundingProvider;
 }
 /** The sanitized, model-visible window facts shared by observer outputs. */
 export interface ObservedWindowValue {
@@ -57,6 +62,18 @@ export declare function screenShotTool(services: ToolServices): import("@deepsee
  * @returns the tool definition.
  */
 export declare function screenReadTool(services: ToolServices): import("@deepseek-ai/dsh-tools").ToolDefinition;
+/**
+ * `screen_find` — find clickable text/visual targets in a window whose
+ * accessibility tree is empty (no UIA elements). It captures the window,
+ * runs OCR (when the probed provider is available) and, when a grounding
+ * provider is mounted and a query is given, visual grounding, then returns
+ * screen-coordinate targets plus an observationId for the coordinate-based
+ * `click` path. Read-only: never needs approval.
+ *
+ * @param services - runtime services.
+ * @returns the tool definition.
+ */
+export declare function screenFindTool(services: ToolServices): import("@deepseek-ai/dsh-tools").ToolDefinition;
 /**
  * `click` — click an element (by elementId) or a coordinate inside the
  * observed window. Never steals foreground focus; delivery is UIA invoke

package/lib/types/tools.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/tools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAKlD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAG/C,OAAO,KAAK,EAAE,cAAc,EAA0B,IAAI,EAAyC,MAAM,qBAAqB,CAAA;AAE9H,wEAAwE;AACxE,MAAM,WAAW,YAAY;IAC3B,4BAA4B;IAC5B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,8BAA8B;IAC9B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;IAC/B,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,6BAA6B;IAC7B,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAA;IACvC,2BAA2B;IAC3B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;CACjC;AAED,4EAA4E;AAC5E,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,IAAI,EAAE,IAAI,CAAA;IACV,UAAU,EAAE,OAAO,CAAA;CACpB;AAgFD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,YAAY,mDAoIpD;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,YAAY,mDA+IpD;AAiED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,YAAY,mDAoD/C;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,YAAY,mDAyC9C;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,YAAY,mDA0ChD;AAED;;;;;;;GAOG;AACH,wBAAgB,OAAO,CAAC,QAAQ,EAAE,YAAY,mDAkC7C;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,YAAY,mDA+EjD;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,YAAY,mDAwCnD;AAED,oDAAoD;AACpD,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,YAAY,qDAW9C"}
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/tools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAKlD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAG/C,OAAO,EAA0B,KAAK,WAAW,EAAE,KAAK,uBAAuB,EAAE,MAAM,mBAAmB,CAAA;AAC1G,OAAO,KAAK,EAAE,cAAc,EAA0B,IAAI,EAAyC,MAAM,qBAAqB,CAAA;AAE9H,wEAAwE;AACxE,MAAM,WAAW,YAAY;IAC3B,4BAA4B;IAC5B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,8BAA8B;IAC9B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;IAC/B,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,6BAA6B;IAC7B,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAA;IACvC,2BAA2B;IAC3B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,0CAA0C;IAC1C,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,8CAA8C;IAC9C,QAAQ,CAAC,SAAS,EAAE,uBAAuB,CAAA;CAC5C;AAED,4EAA4E;AAC5E,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,IAAI,EAAE,IAAI,CAAA;IACV,UAAU,EAAE,OAAO,CAAA;CACpB;AAgFD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,YAAY,mDAoIpD;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,YAAY,mDA+IpD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,YAAY,mDAsJpD;AAiED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,YAAY,mDAoD/C;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,YAAY,mDAyC9C;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,YAAY,mDA0ChD;AAED;;;;;;;GAOG;AACH,wBAAgB,OAAO,CAAC,QAAQ,EAAE,YAAY,mDAkC7C;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,YAAY,mDA+EjD;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,YAAY,mDAwCnD;AAED,oDAAoD;AACpD,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,YAAY,qDAY9C"}

package/lib/types/version.d.ts

@@ -7,7 +7,7 @@
  * @module dsh-click/version
  */
 /** Plugin version advertised in the helper-protocol handshake. */
-export declare const VERSION = "0.1.4";
+export declare const VERSION = "0.3.0";
 /** The helper protocol revision; bump only on incompatible wire changes. */
 export declare const HELPER_PROTOCOL_VERSION = 1;
 //# sourceMappingURL=version.d.ts.map

package/lib/types/vision/grounding.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Visual grounding seam: turning a screenshot plus a natural-language prompt
+ * into labeled bounding boxes. Kept as an independent provider so a vision
+ * model (or an external grounding service) can plug in later without touching
+ * the desktop backend or the tool surface. The default is unavailable — the
+ * OCR path (`screen_find`) already yields text targets, and a grounding
+ * provider is optional and probed through the `dsh-click/grounding` seam.
+ *
+ * @module dsh-click/vision/grounding
+ */
+/** One grounded box in screenshot pixel coordinates. */
+export interface GroundingBox {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+/** One grounded target: a label plus its box and confidence. */
+export interface GroundingTarget {
+    label: string;
+    box: GroundingBox;
+    /** 0..1 confidence. */
+    confidence: number;
+}
+/** One visual grounding provider. */
+export interface VisualGroundingProvider {
+    /** Stable provider id. */
+    readonly id: string;
+    /** Whether the provider can run. */
+    readonly available: boolean;
+    /** Why the provider is unavailable, when it is. */
+    readonly unavailableReason?: string;
+    /** Ground one screenshot for the given prompt. */
+    ground(pngBase64: string, width: number, height: number, prompt: string, signal?: AbortSignal): Promise<GroundingTarget[]>;
+}
+/** A provider that cannot run: `ground` fails closed with the given reason. */
+export declare class UnavailableGroundingProvider implements VisualGroundingProvider {
+    readonly id: string;
+    readonly reason: string;
+    readonly available = false;
+    /** @param id - provider id. @param reason - model-readable unavailability reason. */
+    constructor(id?: string, reason?: string);
+    get unavailableReason(): string;
+    ground(_pngBase64: string, _width: number, _height: number, _prompt: string, _signal?: AbortSignal): Promise<GroundingTarget[]>;
+}
+//# sourceMappingURL=grounding.d.ts.map

package/lib/types/vision/grounding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grounding.d.ts","sourceRoot":"","sources":["../../../src/vision/grounding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,wDAAwD;AACxD,MAAM,WAAW,YAAY;IAC3B,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;IACT,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAED,gEAAgE;AAChE,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,YAAY,CAAA;IACjB,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,qCAAqC;AACrC,MAAM,WAAW,uBAAuB;IACtC,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,oCAAoC;IACpC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;IAC3B,mDAAmD;IACnD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,kDAAkD;IAClD,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAA;CAC3H;AAED,+EAA+E;AAC/E,qBAAa,4BAA6B,YAAW,uBAAuB;IAI9D,QAAQ,CAAC,EAAE;IAAkB,QAAQ,CAAC,MAAM;IAHxD,QAAQ,CAAC,SAAS,SAAQ;IAE1B,qFAAqF;gBAChE,EAAE,SAAgB,EAAW,MAAM,SAA4C;IAEpG,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAGhI"}

package/lib/types/vision/index.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Vision seam index: re-exports the OCR and visual-grounding providers and the
+ * shared pixel→screen coordinate mapping the `screen_find` tool uses to turn
+ * screenshot boxes into clickable screen coordinates.
+ *
+ * @module dsh-click/vision
+ */
+import type { Rect } from '../platform/types.js';
+export { TesseractOcrProvider, UnavailableOcrProvider, parseTesseractTsv, probeTesseract, tesseractArgv, type OcrBox, type OcrProvider, type OcrWord, } from './ocr.js';
+export { UnavailableGroundingProvider, type GroundingBox, type GroundingTarget, type VisualGroundingProvider, } from './grounding.js';
+/**
+ * Map one screenshot box (possibly downscaled by `maxScreenshotSide`) to its
+ * screen-space box, centering on the window's screen rect origin.
+ * @param rect - the window's screen rectangle.
+ * @param imageWidth - screenshot width in pixels.
+ * @param imageHeight - screenshot height in pixels.
+ * @param box - the screenshot-space box.
+ * @returns the screen-space box.
+ */
+export declare function screenBoxFor(rect: Rect, imageWidth: number, imageHeight: number, box: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}): {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+};
+/** Center point of a screen-space box (the coordinate `click` addresses). */
+export declare function centerOf(box: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}): {
+    x: number;
+    y: number;
+};
+//# sourceMappingURL=index.d.ts.map

package/lib/types/vision/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/vision/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAA;AAEhD,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,iBAAiB,EACjB,cAAc,EACd,aAAa,EACb,KAAK,MAAM,EACX,KAAK,WAAW,EAChB,KAAK,OAAO,GACb,MAAM,UAAU,CAAA;AACjB,OAAO,EACL,4BAA4B,EAC5B,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,uBAAuB,GAC7B,MAAM,gBAAgB,CAAA;AAEvB;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CASzD;AAED,6EAA6E;AAC7E,wBAAgB,QAAQ,CAAC,GAAG,EAAE;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAE/G"}

package/lib/types/vision/ocr.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Optical character recognition seam for windows without a UIA tree. OCR is
+ * optional and probed at mount: the `tesseract` provider is used only when the
+ * binary resolves through the subprocess service, otherwise an unavailable
+ * provider fails closed with a model-readable reason. Recognized words carry
+ * their pixel boxes, which `screen_find` maps to screen coordinates for the
+ * coordinate-based `click` path.
+ *
+ * @module dsh-click/vision/ocr
+ */
+import type { SubprocessRuntime } from '@deepseek-ai/dsh-subprocess';
+/** One OCR word box, in screenshot pixel coordinates. */
+export interface OcrBox {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+/** One recognized word plus its box and confidence. */
+export interface OcrWord {
+    text: string;
+    box: OcrBox;
+    /** 0..100 tesseract confidence, or 0 when unknown. */
+    confidence: number;
+}
+/** One OCR provider (probed at mount). */
+export interface OcrProvider {
+    /** Stable provider id. */
+    readonly id: string;
+    /** Whether the provider can run. */
+    readonly available: boolean;
+    /** Why the provider is unavailable, when it is. */
+    readonly unavailableReason?: string;
+    /** Recognize text in one PNG screenshot. */
+    recognize(pngBase64: string, signal?: AbortSignal): Promise<OcrWord[]>;
+}
+/**
+ * Build the tesseract command line: write words as TSV to stdout.
+ * @param command - tesseract executable.
+ * @param imagePath - absolute PNG path.
+ * @param language - tesseract language tag.
+ * @returns the argv (argv[0] is the executable).
+ */
+export declare function tesseractArgv(command: string, imagePath: string, language: string): readonly string[];
+/**
+ * Parse tesseract TSV output into word boxes. Non-word rows, empty text, and
+ * malformed numeric fields are skipped; a hostile or truncated stdout cannot
+ * produce bogus coordinates.
+ * @param tsv - tesseract TSV text from stdout.
+ * @returns the recognized words.
+ */
+export declare function parseTesseractTsv(tsv: string): OcrWord[];
+/** A provider that cannot run: `recognize` fails closed with the given reason. */
+export declare class UnavailableOcrProvider implements OcrProvider {
+    readonly id: string;
+    readonly reason: string;
+    readonly available = false;
+    /** @param id - provider id. @param reason - model-readable unavailability reason. */
+    constructor(id: string, reason: string);
+    get unavailableReason(): string;
+    recognize(_pngBase64: string, _signal?: AbortSignal): Promise<OcrWord[]>;
+}
+/** The tesseract OCR provider over `ctx.subprocess`. */
+export declare class TesseractOcrProvider implements OcrProvider {
+    private readonly subprocess;
+    private readonly command;
+    private readonly language;
+    readonly id = "tesseract";
+    readonly available = true;
+    /**
+     * @param subprocess - the subprocess service that runs tesseract.
+     * @param command - tesseract executable name.
+     * @param language - tesseract language tag.
+     */
+    constructor(subprocess: SubprocessRuntime, command: string, language: string);
+    recognize(pngBase64: string, signal?: AbortSignal): Promise<OcrWord[]>;
+}
+/**
+ * Probe tesseract through the subprocess service and return a provider.
+ * A missing binary degrades to an unavailable provider (fail closed), never a
+ * crash, so the zero-dependency desktop path keeps working.
+ *
+ * @param subprocess - the subprocess service.
+ * @param command - tesseract executable name.
+ * @param language - tesseract language tag.
+ * @returns the probed OCR provider.
+ */
+export declare function probeTesseract(subprocess: SubprocessRuntime, command: string, language: string): Promise<OcrProvider>;
+//# sourceMappingURL=ocr.d.ts.map

package/lib/types/vision/ocr.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocr.d.ts","sourceRoot":"","sources":["../../../src/vision/ocr.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAA;AAIpE,yDAAyD;AACzD,MAAM,WAAW,MAAM;IACrB,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;IACT,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAED,uDAAuD;AACvD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,0CAA0C;AAC1C,MAAM,WAAW,WAAW;IAC1B,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,oCAAoC;IACpC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;IAC3B,mDAAmD;IACnD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,4CAA4C;IAC5C,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CACvE;AAKD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAErG;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,EAAE,CAqBxD;AAED,kFAAkF;AAClF,qBAAa,sBAAuB,YAAW,WAAW;IAI5C,QAAQ,CAAC,EAAE,EAAE,MAAM;IAAE,QAAQ,CAAC,MAAM,EAAE,MAAM;IAHxD,QAAQ,CAAC,SAAS,SAAQ;IAE1B,qFAAqF;gBAChE,EAAE,EAAE,MAAM,EAAW,MAAM,EAAE,MAAM;IAExD,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAGzE;AAED,wDAAwD;AACxD,qBAAa,oBAAqB,YAAW,WAAW;IAUpD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAX3B,QAAQ,CAAC,EAAE,eAAc;IACzB,QAAQ,CAAC,SAAS,QAAO;IAEzB;;;;OAIG;gBAEgB,UAAU,EAAE,iBAAiB,EAC7B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM;IAG7B,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CA2B7E;AAED;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAAC,UAAU,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAO3H"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-click",
-  "version": "0.1.4",
+  "version": "0.3.0",
   "description": "Cross-platform native desktop control tools for DeepSeek Harness (Windows first): screen_shot, screen_read (accessibility tree + pixel description for text-only models), click/type/scroll/key, and app_list/app_launch — every mutating action gated by approval, never stealing foreground focus, with stale-observation rejection and process-identity checks",
   "repository": {
     "type": "git",
@@ -146,5 +146,9 @@
     "verify:self-contained": "node scripts/verify-self-contained.mjs",
     "verify:artifacts": "node scripts/verify-artifacts.mjs"
   },
-  "license": "Apache-2.0"
+  "license": "Apache-2.0",
+  "funding": {
+    "type": "individual",
+    "url": "https://github.com/sponsors/PerryLink"
+  }
 }