@predicatelabs/sdk 0.99.9

package/spec/sdk-types.md

@@ -0,0 +1,259 @@
+# SDK-Level Type Definitions
+
+**Purpose**: Define SDK-level types that extend the snapshot contract with action results, wait results, and trace steps.
+
+## Core Types
+
+### Snapshot
+```typescript
+interface Snapshot {
+  status: "success" | "error";
+  timestamp?: string;
+  url: string;
+  viewport?: { width: number; height: number };
+  elements: Element[];
+  screenshot?: string;
+  screenshot_format?: "png" | "jpeg";
+  error?: string;
+  requires_license?: boolean;
+}
+```
+
+### Element
+```typescript
+interface Element {
+  id: number;
+  role: string;
+  text: string | null;
+  importance: number;
+  bbox: BBox;
+  visual_cues: VisualCues;
+  in_viewport: boolean;
+  is_occluded: boolean;
+  z_index: number;
+}
+```
+
+### BBox
+```typescript
+interface BBox {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+```
+
+### Viewport
+```typescript
+interface Viewport {
+  width: number;
+  height: number;
+}
+```
+
+### VisualCues
+```typescript
+interface VisualCues {
+  is_primary: boolean;
+  background_color_name: string | null;
+  is_clickable: boolean;
+}
+```
+
+## Action Types
+
+### ActionResult
+```typescript
+interface ActionResult {
+  success: boolean;
+  duration_ms: number;
+  outcome?: "navigated" | "dom_updated" | "no_change" | "error";
+  url_changed?: boolean;
+  snapshot_after?: Snapshot;  // Optional in Week 1, required in Week 2
+  error?: {
+    code: string;
+    reason: string;
+    recovery_hint?: string;
+  };
+}
+```
+
+**Fields**:
+- `success`: True if action completed successfully
+- `duration_ms`: Time taken in milliseconds
+- `outcome`: What happened after action (navigation, DOM update, no change, error)
+- `url_changed`: True if URL changed (for navigation detection)
+- `snapshot_after`: Post-action snapshot (optional in MVP, required for recorder)
+- `error`: Error details if action failed
+
+## Wait & Assert Types
+
+### WaitResult
+```typescript
+interface WaitResult {
+  found: boolean;
+  element?: Element;
+  duration_ms: number;
+  timeout: boolean;
+}
+```
+
+**Fields**:
+- `found`: True if element was found
+- `element`: Found element (if found)
+- `duration_ms`: Time taken to find (or timeout)
+- `timeout`: True if wait timed out
+
+### AssertionError
+```typescript
+class AssertionError extends Error {
+  predicate: string | object;
+  timeout: number;
+  element?: Element;
+}
+```
+
+## Trace Types (for Recorder)
+
+### Trace
+```typescript
+interface Trace {
+  version: string;           // "1.0.0"
+  created_at: string;        // ISO 8601
+  start_url: string;
+  steps: TraceStep[];
+}
+```
+
+### TraceStep
+```typescript
+interface TraceStep {
+  ts: number;                // Timestamp (milliseconds since start)
+  type: "navigation" | "click" | "type" | "press" | "wait" | "assert";
+  selector?: string;         // Semantic selector (inferred)
+  element_id?: number;       // Element ID
+  text?: string;             // For type actions (may be masked)
+  key?: string;             // For press actions
+  url?: string;             // For navigation
+  snapshot?: Snapshot;       // Optional: snapshot at this step
+}
+```
+
+**Step Types**:
+- `navigation`: `goto(url)`
+- `click`: Click on element
+- `type`: Type text into element
+- `press`: Press keyboard key
+- `wait`: Wait for element/predicate
+- `assert`: Assertion check
+
+## Query Types
+
+### QuerySelector
+```typescript
+// String DSL
+type QuerySelectorString = string;  // e.g., "role=button text~'Sign in'"
+
+// Structured object
+interface QuerySelectorObject {
+  role?: string;
+  text?: string | RegExp;
+  name?: string | RegExp;
+  clickable?: boolean;
+  isPrimary?: boolean;
+  importance?: number | { min?: number; max?: number };
+}
+
+type QuerySelector = QuerySelectorString | QuerySelectorObject;
+```
+
+## Python Equivalents (Pydantic)
+
+```python
+from pydantic import BaseModel
+from typing import Optional, List, Union
+from datetime import datetime
+
+class BBox(BaseModel):
+    x: float
+    y: float
+    width: float
+    height: float
+
+class Viewport(BaseModel):
+    width: float
+    height: float
+
+class VisualCues(BaseModel):
+    is_primary: bool
+    background_color_name: Optional[str]
+    is_clickable: bool
+
+class Element(BaseModel):
+    id: int
+    role: str
+    text: Optional[str]
+    importance: int
+    bbox: BBox
+    visual_cues: VisualCues
+    in_viewport: bool = True
+    is_occluded: bool = False
+    z_index: int = 0
+
+class Snapshot(BaseModel):
+    status: str  # "success" | "error"
+    timestamp: Optional[str] = None
+    url: str
+    viewport: Optional[Viewport] = None
+    elements: List[Element]
+    screenshot: Optional[str] = None
+    screenshot_format: Optional[str] = None
+    error: Optional[str] = None
+    requires_license: Optional[bool] = None
+
+class ActionResult(BaseModel):
+    success: bool
+    duration_ms: int
+    outcome: Optional[str] = None
+    url_changed: Optional[bool] = None
+    snapshot_after: Optional[Snapshot] = None
+    error: Optional[dict] = None
+
+class WaitResult(BaseModel):
+    found: bool
+    element: Optional[Element] = None
+    duration_ms: int
+    timeout: bool
+
+class TraceStep(BaseModel):
+    ts: int
+    type: str
+    selector: Optional[str] = None
+    element_id: Optional[int] = None
+    text: Optional[str] = None
+    key: Optional[str] = None
+    url: Optional[str] = None
+    snapshot: Optional[Snapshot] = None
+
+class Trace(BaseModel):
+    version: str
+    created_at: str
+    start_url: str
+    steps: List[TraceStep]
+```
+
+## Type Validation Rules
+
+1. **Required Fields**: Must be present and non-null
+2. **Optional Fields**: May be omitted or null
+3. **Type Coercion**: Numbers should be validated (no NaN, Infinity)
+4. **Enum Values**: String enums must match exactly
+5. **Array Bounds**: Elements array should be validated (not empty for success status)
+
+## Compatibility Notes
+
+- SDKs should handle missing optional fields gracefully
+- Default values should match extension behavior
+- Type coercion should be minimal (prefer validation errors)
+

package/spec/snapshot.schema.json

@@ -0,0 +1,148 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Sentience Snapshot Schema v1",
+  "description": "Single source of truth for snapshot data structure between extension and SDKs",
+  "version": "1.0.0",
+  "type": "object",
+  "required": ["status", "url", "elements"],
+  "properties": {
+    "status": {
+      "type": "string",
+      "enum": ["success", "error"],
+      "description": "Operation status"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp of snapshot"
+    },
+    "url": {
+      "type": "string",
+      "format": "uri",
+      "description": "Current page URL"
+    },
+    "viewport": {
+      "type": "object",
+      "required": ["width", "height"],
+      "properties": {
+        "width": {
+          "type": "number",
+          "description": "Viewport width in pixels"
+        },
+        "height": {
+          "type": "number",
+          "description": "Viewport height in pixels"
+        }
+      }
+    },
+    "elements": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/Element"
+      },
+      "description": "Array of analyzed elements, sorted by importance (descending)"
+    },
+    "screenshot": {
+      "type": "string",
+      "description": "Base64-encoded screenshot data URL (optional)",
+      "pattern": "^data:image/(png|jpeg);base64,"
+    },
+    "screenshot_format": {
+      "type": "string",
+      "enum": ["png", "jpeg"],
+      "description": "Screenshot format (optional)"
+    },
+    "error": {
+      "type": "string",
+      "description": "Error message if status is 'error'"
+    },
+    "requires_license": {
+      "type": "boolean",
+      "description": "True if operation requires license key (headless mode)"
+    }
+  },
+  "definitions": {
+    "Element": {
+      "type": "object",
+      "required": ["id", "role", "importance", "bbox", "visual_cues"],
+      "properties": {
+        "id": {
+          "type": "integer",
+          "description": "Unique element identifier (index in registry)"
+        },
+        "role": {
+          "type": "string",
+          "enum": ["button", "link", "textbox", "searchbox", "checkbox", "radio", "combobox", "image", "generic"],
+          "description": "Semantic role of the element"
+        },
+        "text": {
+          "type": ["string", "null"],
+          "description": "Text content, aria-label, or placeholder (max 100 chars)"
+        },
+        "importance": {
+          "type": "integer",
+          "description": "Importance score (-300 to ~1800), higher = more important"
+        },
+        "bbox": {
+          "$ref": "#/definitions/BBox"
+        },
+        "visual_cues": {
+          "$ref": "#/definitions/VisualCues"
+        },
+        "in_viewport": {
+          "type": "boolean",
+          "description": "True if element is visible in viewport"
+        },
+        "is_occluded": {
+          "type": "boolean",
+          "description": "True if element is covered by another element"
+        },
+        "z_index": {
+          "type": "integer",
+          "description": "CSS z-index value (0 if auto or not set)"
+        }
+      }
+    },
+    "BBox": {
+      "type": "object",
+      "required": ["x", "y", "width", "height"],
+      "properties": {
+        "x": {
+          "type": "number",
+          "description": "X coordinate (left edge) in pixels"
+        },
+        "y": {
+          "type": "number",
+          "description": "Y coordinate (top edge) in pixels"
+        },
+        "width": {
+          "type": "number",
+          "description": "Width in pixels"
+        },
+        "height": {
+          "type": "number",
+          "description": "Height in pixels"
+        }
+      }
+    },
+    "VisualCues": {
+      "type": "object",
+      "required": ["is_primary", "is_clickable"],
+      "properties": {
+        "is_primary": {
+          "type": "boolean",
+          "description": "True if element is visually prominent primary action"
+        },
+        "background_color_name": {
+          "type": ["string", "null"],
+          "description": "Nearest named color from 32-color palette (e.g., 'blue', 'red')"
+        },
+        "is_clickable": {
+          "type": "boolean",
+          "description": "True if element has pointer cursor or actionable role"
+        }
+      }
+    }
+  }
+}
+

package/src/extension/background.js

@@ -0,0 +1,104 @@
+import init, { analyze_page_with_options, analyze_page, prune_for_api } from "./pkg/sentience_core.js";
+
+let wasmReady = !1, wasmInitPromise = null;
+
+async function initWASM() {
+    if (!wasmReady) return wasmInitPromise || (wasmInitPromise = (async () => {
+        try {
+            globalThis.js_click_element = () => {}, await init(), wasmReady = !0;
+        } catch (error) {
+            throw error;
+        }
+    })(), wasmInitPromise);
+}
+
+async function handleScreenshotCapture(_tabId, options = {}) {
+    try {
+        const {format: format = "png", quality: quality = 90} = options;
+        return await chrome.tabs.captureVisibleTab(null, {
+            format: format,
+            quality: quality
+        });
+    } catch (error) {
+        throw new Error(`Failed to capture screenshot: ${error.message}`);
+    }
+}
+
+async function handleSnapshotProcessing(rawData, options = {}) {
+    const startTime = performance.now();
+    try {
+        if (!Array.isArray(rawData)) throw new Error("rawData must be an array");
+        if (rawData.length > 1e4 && (rawData = rawData.slice(0, 1e4)), await initWASM(), 
+        !wasmReady) throw new Error("WASM module not initialized");
+        let analyzedElements, prunedRawData;
+        try {
+            const wasmPromise = new Promise((resolve, reject) => {
+                try {
+                    let result;
+                    result = options.limit || options.filter ? analyze_page_with_options(rawData, options) : analyze_page(rawData), 
+                    resolve(result);
+                } catch (e) {
+                    reject(e);
+                }
+            });
+            analyzedElements = await Promise.race([ wasmPromise, new Promise((_, reject) => setTimeout(() => reject(new Error("WASM processing timeout (>18s)")), 18e3)) ]);
+        } catch (e) {
+            const errorMsg = e.message || "Unknown WASM error";
+            throw new Error(`WASM analyze_page failed: ${errorMsg}`);
+        }
+        try {
+            prunedRawData = prune_for_api(rawData);
+        } catch (e) {
+            prunedRawData = rawData;
+        }
+        performance.now();
+        return {
+            elements: analyzedElements,
+            raw_elements: prunedRawData
+        };
+    } catch (error) {
+        performance.now();
+        throw error;
+    }
+}
+
+initWASM().catch(err => {}), chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
+    try {
+        return "captureScreenshot" === request.action ? (handleScreenshotCapture(sender.tab.id, request.options).then(screenshot => {
+            sendResponse({
+                success: !0,
+                screenshot: screenshot
+            });
+        }).catch(error => {
+            sendResponse({
+                success: !1,
+                error: error.message || "Screenshot capture failed"
+            });
+        }), !0) : "processSnapshot" === request.action ? (handleSnapshotProcessing(request.rawData, request.options).then(result => {
+            sendResponse({
+                success: !0,
+                result: result
+            });
+        }).catch(error => {
+            sendResponse({
+                success: !1,
+                error: error.message || "Snapshot processing failed"
+            });
+        }), !0) : (sendResponse({
+            success: !1,
+            error: "Unknown action"
+        }), !1);
+    } catch (error) {
+        try {
+            sendResponse({
+                success: !1,
+                error: `Fatal error: ${error.message || "Unknown error"}`
+            });
+        } catch (e) {}
+        return !1;
+    }
+}), self.addEventListener("error", event => {
+    event.preventDefault();
+}), self.addEventListener("unhandledrejection", event => {
+    event.preventDefault();
+});

package/src/extension/content.js

@@ -0,0 +1,162 @@
+!function() {
+    "use strict";
+    window, window.top;
+    document.documentElement.dataset.sentienceExtensionId = chrome.runtime.id, window.addEventListener("message", event => {
+        var data;
+        if (event.source === window) switch (event.data.type) {
+          case "SENTIENCE_SCREENSHOT_REQUEST":
+            data = event.data, chrome.runtime.sendMessage({
+                action: "captureScreenshot",
+                options: data.options
+            }, response => {
+                window.postMessage({
+                    type: "SENTIENCE_SCREENSHOT_RESULT",
+                    requestId: data.requestId,
+                    screenshot: response?.success ? response.screenshot : null,
+                    error: response?.error
+                }, "*");
+            });
+            break;
+
+          case "SENTIENCE_SNAPSHOT_REQUEST":
+            !function(data) {
+                const startTime = performance.now();
+                let responded = !1;
+                const timeoutId = setTimeout(() => {
+                    if (!responded) {
+                        responded = !0;
+                        const duration = performance.now() - startTime;
+                        window.postMessage({
+                            type: "SENTIENCE_SNAPSHOT_RESULT",
+                            requestId: data.requestId,
+                            error: "WASM processing timeout - background script may be unresponsive",
+                            duration: duration
+                        }, "*");
+                    }
+                }, 2e4);
+                try {
+                    chrome.runtime.sendMessage({
+                        action: "processSnapshot",
+                        rawData: data.rawData,
+                        options: data.options
+                    }, response => {
+                        if (responded) return;
+                        responded = !0, clearTimeout(timeoutId);
+                        const duration = performance.now() - startTime;
+                        chrome.runtime.lastError ? window.postMessage({
+                            type: "SENTIENCE_SNAPSHOT_RESULT",
+                            requestId: data.requestId,
+                            error: `Chrome runtime error: ${chrome.runtime.lastError.message}`,
+                            duration: duration
+                        }, "*") : response?.success ? window.postMessage({
+                            type: "SENTIENCE_SNAPSHOT_RESULT",
+                            requestId: data.requestId,
+                            elements: response.result.elements,
+                            raw_elements: response.result.raw_elements,
+                            duration: duration
+                        }, "*") : window.postMessage({
+                            type: "SENTIENCE_SNAPSHOT_RESULT",
+                            requestId: data.requestId,
+                            error: response?.error || "Processing failed",
+                            duration: duration
+                        }, "*");
+                    });
+                } catch (error) {
+                    if (!responded) {
+                        responded = !0, clearTimeout(timeoutId);
+                        const duration = performance.now() - startTime;
+                        window.postMessage({
+                            type: "SENTIENCE_SNAPSHOT_RESULT",
+                            requestId: data.requestId,
+                            error: `Failed to send message: ${error.message}`,
+                            duration: duration
+                        }, "*");
+                    }
+                }
+            }(event.data);
+            break;
+
+          case "SENTIENCE_SHOW_OVERLAY":
+            !function(data) {
+                const {elements: elements, targetElementId: targetElementId} = data;
+                if (!elements || "object" != typeof elements || "number" != typeof elements.length) return;
+                const elementsArray = Array.isArray(elements) ? elements : Array.from(elements);
+                removeOverlay();
+                const host = document.createElement("div");
+                host.id = OVERLAY_HOST_ID, host.style.cssText = "\n        position: fixed !important;\n        top: 0 !important;\n        left: 0 !important;\n        width: 100vw !important;\n        height: 100vh !important;\n        pointer-events: none !important;\n        z-index: 2147483647 !important;\n        margin: 0 !important;\n        padding: 0 !important;\n    ", 
+                document.body.appendChild(host);
+                const shadow = host.attachShadow({
+                    mode: "closed"
+                }), maxImportance = Math.max(...elementsArray.map(e => e.importance || 0), 1);
+                elementsArray.forEach(element => {
+                    const bbox = element.bbox;
+                    if (!bbox) return;
+                    const isTarget = element.id === targetElementId, isPrimary = element.visual_cues?.is_primary || !1, importance = element.importance || 0;
+                    let color;
+                    color = isTarget ? "#FF0000" : isPrimary ? "#0066FF" : "#00FF00";
+                    const importanceRatio = maxImportance > 0 ? importance / maxImportance : .5, borderOpacity = isTarget ? 1 : isPrimary ? .9 : Math.max(.4, .5 + .5 * importanceRatio), fillOpacity = .2 * borderOpacity, borderWidth = isTarget ? 2 : isPrimary ? 1.5 : Math.max(.5, Math.round(2 * importanceRatio)), hexOpacity = Math.round(255 * fillOpacity).toString(16).padStart(2, "0"), box = document.createElement("div");
+                    if (box.style.cssText = `\n            position: absolute;\n            left: ${bbox.x}px;\n            top: ${bbox.y}px;\n            width: ${bbox.width}px;\n            height: ${bbox.height}px;\n            border: ${borderWidth}px solid ${color};\n            background-color: ${color}${hexOpacity};\n            box-sizing: border-box;\n            opacity: ${borderOpacity};\n            pointer-events: none;\n        `, 
+                    importance > 0 || isPrimary) {
+                        const badge = document.createElement("span");
+                        badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`, badge.style.cssText = `\n                position: absolute;\n                top: -18px;\n                left: 0;\n                background: ${color};\n                color: white;\n                font-size: 11px;\n                font-weight: bold;\n                padding: 2px 6px;\n                font-family: Arial, sans-serif;\n                border-radius: 3px;\n                opacity: 0.95;\n                white-space: nowrap;\n                pointer-events: none;\n            `, 
+                        box.appendChild(badge);
+                    }
+                    if (isTarget) {
+                        const targetIndicator = document.createElement("span");
+                        targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n                position: absolute;\n                top: -18px;\n                right: 0;\n                font-size: 16px;\n                pointer-events: none;\n            ", 
+                        box.appendChild(targetIndicator);
+                    }
+                    shadow.appendChild(box);
+                }), overlayTimeout = setTimeout(() => {
+                    removeOverlay();
+                }, 5e3);
+            }(event.data);
+            break;
+
+          case "SENTIENCE_CLEAR_OVERLAY":
+            removeOverlay();
+            break;
+
+          case "SENTIENCE_SHOW_GRID_OVERLAY":
+            !function(data) {
+                const {grids: grids, targetGridId: targetGridId} = data;
+                if (!grids || !Array.isArray(grids)) return;
+                removeOverlay();
+                const host = document.createElement("div");
+                host.id = OVERLAY_HOST_ID, host.style.cssText = "\n        position: fixed !important;\n        top: 0 !important;\n        left: 0 !important;\n        width: 100vw !important;\n        height: 100vh !important;\n        pointer-events: none !important;\n        z-index: 2147483647 !important;\n        margin: 0 !important;\n        padding: 0 !important;\n    ", 
+                document.body.appendChild(host);
+                const shadow = host.attachShadow({
+                    mode: "closed"
+                });
+                grids.forEach(grid => {
+                    const bbox = grid.bbox;
+                    if (!bbox) return;
+                    const isTarget = grid.grid_id === targetGridId, isDominant = !0 === grid.is_dominant;
+                    let color = "#9B59B6";
+                    isTarget ? color = "#FF0000" : isDominant && (color = "#FF8C00");
+                    const borderStyle = isTarget ? "solid" : "dashed", borderWidth = isTarget ? 3 : isDominant ? 2.5 : 2, opacity = isTarget ? 1 : isDominant ? .9 : .8, fillOpacity = .1 * opacity, hexOpacity = Math.round(255 * fillOpacity).toString(16).padStart(2, "0"), box = document.createElement("div");
+                    box.style.cssText = `\n            position: absolute;\n            left: ${bbox.x}px;\n            top: ${bbox.y}px;\n            width: ${bbox.width}px;\n            height: ${bbox.height}px;\n            border: ${borderWidth}px ${borderStyle} ${color};\n            background-color: ${color}${hexOpacity};\n            box-sizing: border-box;\n            opacity: ${opacity};\n            pointer-events: none;\n        `;
+                    let labelText = grid.label ? `Grid ${grid.grid_id}: ${grid.label}` : `Grid ${grid.grid_id}`;
+                    grid.is_dominant && (labelText = `⭐ ${labelText} (dominant)`);
+                    const badge = document.createElement("span");
+                    if (badge.textContent = labelText, badge.style.cssText = `\n                position: absolute;\n                top: -18px;\n                left: 0;\n                background: ${color};\n                color: white;\n                font-size: 11px;\n                font-weight: bold;\n                padding: 2px 6px;\n                font-family: Arial, sans-serif;\n                border-radius: 3px;\n                opacity: 0.95;\n                white-space: nowrap;\n                pointer-events: none;\n            `, 
+                    box.appendChild(badge), isTarget) {
+                        const targetIndicator = document.createElement("span");
+                        targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n                position: absolute;\n                top: -18px;\n                right: 0;\n                font-size: 16px;\n                pointer-events: none;\n            ", 
+                        box.appendChild(targetIndicator);
+                    }
+                    shadow.appendChild(box);
+                }), overlayTimeout = setTimeout(() => {
+                    removeOverlay();
+                }, 5e3);
+            }(event.data);
+        }
+    });
+    const OVERLAY_HOST_ID = "sentience-overlay-host";
+    let overlayTimeout = null;
+    function removeOverlay() {
+        const existing = document.getElementById(OVERLAY_HOST_ID);
+        existing && existing.remove(), overlayTimeout && (clearTimeout(overlayTimeout), 
+        overlayTimeout = null);
+    }
+}();