@weavetab/plugin-realtime 0.0.0-stage → 0.3.0

package/dist/reflex.js

@@ -0,0 +1,171 @@
+import { run, arm } from "./page.js";
+import { canvasTool } from "./canvas.js";
+// one rAF loop per page, only alive while a rule is active.
+// it grabs the un-spied rAF so reflex ticks never show up as canvas frames.
+const RUNTIME = `(() => {
+  if (window.__wt_rf) return 1;
+  const nativeRaf = (window.__wt_cv && window.__wt_cv.raf) || window.requestAnimationFrame;
+  const rf = window.__wt_rf = { rules: {}, on: false };
+
+  const CODES = { ' ': ['Space', 32], Enter: ['Enter', 13], Escape: ['Escape', 27], Tab: ['Tab', 9],
+    ArrowLeft: ['ArrowLeft', 37], ArrowUp: ['ArrowUp', 38], ArrowRight: ['ArrowRight', 39], ArrowDown: ['ArrowDown', 40] };
+  const press = (k, hold) => {
+    if (k === 'Space') k = ' ';
+    const c = CODES[k] || (/^[a-z]$/i.test(k) ? ['Key' + k.toUpperCase(), k.toUpperCase().charCodeAt(0)]
+      : /^[0-9]$/.test(k) ? ['Digit' + k, k.charCodeAt(0)] : [k, 0]);
+    const o = { key: k, code: c[0], keyCode: c[1], which: c[1], bubbles: true, cancelable: true };
+    const tgt = document.activeElement || document.body;
+    tgt.dispatchEvent(new KeyboardEvent('keydown', o));
+    if (tgt !== document) document.dispatchEvent(new KeyboardEvent('keydown', o));
+    setTimeout(() => {
+      tgt.dispatchEvent(new KeyboardEvent('keyup', o));
+      if (tgt !== document) document.dispatchEvent(new KeyboardEvent('keyup', o));
+    }, hold);
+  };
+  const click = (el, x, y) => {
+    if (x == null) { const b = el.getBoundingClientRect(); x = b.left + b.width / 2; y = b.top + b.height / 2; }
+    const o = { bubbles: true, cancelable: true, composed: true, clientX: x, clientY: y, view: window, button: 0 };
+    for (const t of ['pointerdown', 'mousedown', 'pointerup', 'mouseup', 'click'])
+      el.dispatchEvent(t.startsWith('pointer') ? new PointerEvent(t, o) : new MouseEvent(t, o));
+  };
+  const shown = (el) => {
+    const r = el.getBoundingClientRect();
+    if (!r.width || !r.height) return false;
+    const s = getComputedStyle(el);
+    return s.visibility !== 'hidden' && s.display !== 'none';
+  };
+  const isNew = (el, since) => {
+    const rt = window.__wt_rt;
+    if (!rt) return false;
+    for (let n = el; n; n = n.parentElement) { const b = rt.born.get(n); if (b !== undefined) return b >= since; }
+    return false;
+  };
+  const find = (r) => {
+    for (const el of document.querySelectorAll(r.sel)) {
+      if (r.enabled && (el.disabled || el.getAttribute('aria-disabled') === 'true')) continue;
+      if (r.text && !((el.getAttribute('aria-label') || '') + ' ' + (el.value || '') + ' ' + el.textContent).toLowerCase().includes(r.text)) continue;
+      if (r.onlyNew && !isNew(el, r.t0)) continue;
+      if (shown(el)) return el;
+    }
+    return null;
+  };
+  const dist = (a, b) => (a && b) ? Math.hypot(a.x + a.w / 2 - b.x - b.w / 2, a.y + a.h / 2 - b.y - b.h / 2) : Infinity;
+  const gap = (a, b) => (a && b) ? b.x - (a.x + a.w) : Infinity;
+  const E = new Proxy({}, { get: (_, k) => {
+    const cv = window.__wt_cv;
+    if (!cv) return [];
+    if (k === 'all') return cv.track('*', 2);
+    if (k === 'moving') return cv.track('*', 2).filter((e) => Math.abs(e.vx) > 0.1 || Math.abs(e.vy) > 0.1);
+    const sig = rf.names && rf.names[k];
+    return sig ? cv.track(sig, 2) : [];
+  } });
+
+  const tick = () => {
+    const now = performance.now();
+    let live = 0;
+    for (const id in rf.rules) {
+      const r = rf.rules[id];
+      if (r.done) continue;
+      try {
+        if (now - r.t0 > r.ttl) { r.done = 'ttl'; continue; }
+        if (r.stop && document.querySelector(r.stop)) { r.done = 'stop_selector'; continue; }
+        live++;
+        if (now - r.last < r.cooldown) continue;
+        rf.names = r.names;
+        let el = null;
+        if (r.sel) { el = find(r); if (!el) continue; }
+        else if (!r.whenFn(E, dist, gap)) continue;
+        const a = r.act;
+        if (a.type === 'click') click(el);
+        else if (a.type === 'click_xy') click(document.elementFromPoint(a.x, a.y) || document.body, a.x, a.y);
+        else if (a.type === 'key') press(a.key, a.hold);
+        else r.jsFn();
+        r.hits++; r.last = now;
+        if (r.hits >= r.max) { r.done = 'max_hits'; live--; }
+      } catch (e) { r.done = 'error'; r.err = String((e && e.message) || e); }
+    }
+    if (live) nativeRaf.call(window, tick); else rf.on = false;
+  };
+  rf.add = (rule, whenFn, jsFn) => {
+    rf.rules[rule.id] = Object.assign({}, rule, { t0: performance.now(), last: -1e9, hits: 0, done: null, err: null, whenFn, jsFn });
+    if (!rf.on) { rf.on = true; nativeRaf.call(window, tick); }
+  };
+  rf.snap = () => {
+    const now = performance.now();
+    return Object.keys(rf.rules).map((id) => {
+      const r = rf.rules[id];
+      return { id, kind: r.sel ? 'dom' : 'canvas', hits: r.hits, done: r.done, error: r.err,
+        last_hit_ms_ago: r.hits ? Math.round(now - r.last) : null, age_ms: Math.round(now - r.t0) };
+    });
+  };
+  return 0;
+})()`;
+let seq = 0;
+export async function reflexTool(sess, a) {
+    await arm(sess);
+    await run(sess, RUNTIME);
+    if (a.action === "list")
+        return run(sess, "__wt_rf.snap()");
+    if (a.action === "clear") {
+        await run(sess, "(() => { for (const k in __wt_rf.rules) delete __wt_rf.rules[k]; })()");
+        return { cleared: true };
+    }
+    if (a.action === "remove") {
+        if (!a.id)
+            throw new Error("id required");
+        const had = await run(sess, `(() => { const had = ${JSON.stringify(a.id)} in __wt_rf.rules; delete __wt_rf.rules[${JSON.stringify(a.id)}]; return had; })()`);
+        return { removed: had };
+    }
+    if (!a.when || !a.do)
+        throw new Error("add needs both when and do");
+    const w = a.when, d = a.do;
+    const cv = w.canvas;
+    if (!!w.selector === !!cv)
+        throw new Error("when needs exactly one of selector or canvas");
+    if (d.type === "click" && !w.selector)
+        throw new Error("do.click only works with a selector trigger, use click_xy");
+    if (d.type === "click_xy" && (d.x == null || d.y == null))
+        throw new Error("click_xy needs x and y");
+    if (d.type === "key" && !d.key)
+        throw new Error("key needs a key");
+    if (d.type === "js" && !d.code)
+        throw new Error("js needs code");
+    const id = a.id || `r${++seq}`;
+    const rule = {
+        id,
+        sel: w.selector || null,
+        text: w.text ? w.text.toLowerCase() : null,
+        onlyNew: Boolean(w.only_new),
+        enabled: w.enabled !== false,
+        names: cv?.names ?? null,
+        act: { type: d.type, x: d.x, y: d.y, key: d.key, hold: d.hold_ms ?? 40 },
+        cooldown: a.cooldown_ms ?? 100,
+        max: a.max_hits ?? 50,
+        ttl: Math.min(a.ttl_ms ?? 30000, 600000),
+        stop: a.stop_selector || null,
+    };
+    if (cv)
+        await canvasTool(sess, { action: "attach" });
+    // fns go in as source so page CSP (no unsafe-eval) can't block them
+    const whenFn = cv ? `(E, dist, gap) => (\n${cv.js}\n)` : "null";
+    const jsFn = d.type === "js" ? `() => {\n${d.code}\n}` : "null";
+    await run(sess, `__wt_rf.add(${JSON.stringify(rule)}, ${whenFn}, ${jsFn})`);
+    return { id, armed: true };
+}
+export function statusTool(sess, waitMs = 0) {
+    const ms = Math.min(Math.max(waitMs, 0), 30000);
+    return run(sess, `new Promise((res) => {
+      const t0 = performance.now();
+      const snap = () => ({
+        probe: Boolean(window.__wt_rt),
+        canvas: window.__wt_cv ? { attached: true, frames_seen: window.__wt_cv.n } : { attached: false },
+        reflexes: window.__wt_rf ? window.__wt_rf.snap() : []
+      });
+      const tick = () => {
+        const s = snap();
+        if (!${ms} || !s.reflexes.some((r) => !r.done) || performance.now() - t0 > ${ms}) res(s);
+        else setTimeout(tick, 50);
+      };
+      tick();
+    })`, true);
+}

package/dist/surface.d.ts

@@ -0,0 +1,19 @@
+export declare function readFeed(sess: any, winMs: number): Promise<{
+    active_tab: boolean;
+    url?: undefined;
+    probe?: undefined;
+    last_click?: undefined;
+    last_mut_ms_ago?: undefined;
+    recent_elements?: undefined;
+    reflexes?: undefined;
+    canvas?: undefined;
+} | {
+    url: any;
+    probe: any;
+    last_click: any;
+    last_mut_ms_ago: any;
+    recent_elements: any;
+    reflexes: any;
+    canvas: any;
+    active_tab?: undefined;
+}>;

package/dist/surface.js

@@ -0,0 +1,59 @@
+import { run, arm } from "./page.js";
+import { canvasSnap } from "./canvas.js";
+import { statusTool } from "./reflex.js";
+const RECENT = `((win) => {
+  const rt = window.__wt_rt;
+  if (!rt) return { click: null, last_mut_ms_ago: null, fresh: [] };
+  const now = performance.now();
+  const clk = rt.click && now - rt.click.t < 15000
+    ? { x: Math.round(rt.click.x), y: Math.round(rt.click.y), age_ms: Math.round(now - rt.click.t) }
+    : null;
+  const sel = 'button, a[href], input, select, textarea, summary, dialog[open], [role=button], [role=link], [role=menuitem], [role=tab], [role=dialog], [role=alertdialog], [onclick]';
+  const fresh = [];
+  for (const el of document.querySelectorAll(sel)) {
+    let b;
+    for (let n = el; n && b === undefined; n = n.parentElement) b = rt.born.get(n);
+    if (b === undefined || now - b > win) continue;
+    const r = el.getBoundingClientRect();
+    if (r.width < 2 || r.height < 2) continue;
+    const s = getComputedStyle(el);
+    if (s.visibility === 'hidden' || s.display === 'none') continue;
+    const label = (el.getAttribute('aria-label') || el.innerText || el.value || el.title || '').trim().replace(/\\s+/g, ' ').slice(0, 60);
+    const dist = clk ? Math.round(Math.hypot(r.left + r.width / 2 - clk.x, r.top + r.height / 2 - clk.y)) : null;
+    fresh.push({
+      tag: el.getAttribute('role') || el.tagName.toLowerCase(),
+      id: el.id || null,
+      wt_id: el.dataset && el.dataset.wtId || null,
+      label,
+      age_ms: Math.round(now - b),
+      dist_from_click_px: dist
+    });
+    if (fresh.length >= 25) break;
+  }
+  fresh.sort((a, b) => a.age_ms - b.age_ms);
+  return {
+    url: location.href,
+    click: clk,
+    last_mut_ms_ago: rt.lastMut ? Math.round(now - rt.lastMut) : null,
+    fresh
+  };
+})`;
+export async function readFeed(sess, winMs) {
+    if (!sess?.Runtime)
+        return { active_tab: false };
+    await arm(sess);
+    const [st, dom, cv] = await Promise.all([
+        statusTool(sess, 0),
+        run(sess, `${RECENT}(${winMs})`),
+        canvasSnap(sess),
+    ]);
+    return {
+        url: dom.url,
+        probe: st.probe,
+        last_click: dom.click,
+        last_mut_ms_ago: dom.last_mut_ms_ago,
+        recent_elements: dom.fresh,
+        reflexes: st.reflexes,
+        canvas: cv || st.canvas,
+    };
+}

package/package.json

@@ -1,6 +1,54 @@
 {
   "name": "@weavetab/plugin-realtime",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.3.0",
+  "description": "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop for Weavetab.",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "weavetab.json",
+    "README.md",
+    "AGENTS.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "preflight": "weavetab preflight",
+    "test": "node --test test/*.test.mjs",
+    "prepublishOnly": "npm run build"
+  },
+  "dependencies": {
+    "@weavetab/sdk": "^2.6.2"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.0",
+    "chrome-remote-interface": "^0.33.2",
+    "typescript": "^5.5.0"
+  },
+  "keywords": [
+    "weavetab",
+    "weavetab-plugin",
+    "cdp",
+    "canvas",
+    "browser-automation"
+  ],
+  "author": {
+    "name": "fy2ne",
+    "url": "https://fy2ne.me"
+  },
+  "license": "Apache-2.0",
+  "repository": { "type": "git", "url": "git+https://github.com/Weavetab/Realtime.git" },
+  "bugs": { "url": "https://github.com/Weavetab/Realtime/issues" },
+  "homepage": "https://github.com/Weavetab/Realtime#readme",
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/weavetab.json

@@ -0,0 +1,270 @@
+{
+  "$schema": "https://weavetab.dev/schema/plugin.json",
+  "name": "@weavetab/plugin-realtime",
+  "displayName": "Realtime",
+  "version": "0.3.0",
+  "description": "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop.",
+  "entry": "dist/index.js",
+  "agentsMd": "AGENTS.md",
+  "author": {
+    "name": "fy2ne",
+    "url": "https://fy2ne.me"
+  },
+  "license": "Apache-2.0",
+  "homepage": "https://github.com/Weavetab/Realtime#readme",
+  "repository": "https://github.com/Weavetab/Realtime",
+  "engines": {
+    "weavetab": "^2.6.2",
+    "node": ">=18.0.0"
+  },
+  "environments": [
+    "mcp",
+    "cli",
+    "browser"
+  ],
+  "keywords": [
+    "weavetab",
+    "weavetab-plugin",
+    "cdp",
+    "canvas",
+    "browser-automation"
+  ],
+  "defaultConfig": {
+    "causalWindowMs": 4000
+  },
+  "permissions": {
+    "cdp": {
+      "evaluate": true,
+      "input": true,
+      "reason": "Installs the in-page MutationObserver, hooks Canvas2D/rAF, and dispatches click/key events."
+    },
+    "tools": {
+      "add": true,
+      "reason": "Registers realtime_canvas, realtime_reflex, realtime_autopilot, and realtime_status."
+    },
+    "commands": {
+      "add": true,
+      "reason": "Registers status and reflex-clear CLI commands."
+    },
+    "prompts": {
+      "add": true,
+      "reason": "Registers the /realtime_watch prompt utility."
+    },
+    "resources": {
+      "add": true,
+      "reason": "Registers the @realtime_feed resource utility."
+    },
+    "hooks": {
+      "onBeforeAction": true,
+      "onAfterAction": true,
+      "reason": "Arms the in-page mutation probe before actions and tags newly added nodes in browser_map output."
+    },
+    "network": {
+      "domains": [
+        "127.0.0.1",
+        "localhost"
+      ],
+      "reason": "Allows realtime_autopilot to query an optional local OpenAI-compatible model endpoint."
+    }
+  },
+  "commands": [
+    {
+      "name": "status",
+      "description": "Read the in-page realtime probe state, canvas frame counter, and active 60Hz reflex rules.",
+      "usage": "weavetab plugin use realtime status [--wait_ms <ms>]",
+      "requiresSession": true,
+      "options": [
+        {
+          "name": "wait_ms",
+          "type": "number",
+          "description": "Block up to wait_ms until active reflexes finish",
+          "required": false
+        }
+      ]
+    },
+    {
+      "name": "reflex-clear",
+      "description": "Clear all active 60Hz reflex rules on the active browser tab via CLI.",
+      "usage": "weavetab plugin use realtime reflex-clear",
+      "requiresSession": true,
+      "options": []
+    },
+    {
+      "name": "autopilot",
+      "description": "Run bounded in-page autopilot toward a goal directly via CLI.",
+      "usage": "weavetab plugin use realtime autopilot --goal <text> [--max_steps <n>]",
+      "requiresSession": true,
+      "options": [
+        {
+          "name": "goal",
+          "type": "string",
+          "description": "Natural language click goal",
+          "required": true
+        },
+        {
+          "name": "max_steps",
+          "type": "number",
+          "description": "Maximum click steps (default 5)",
+          "required": false
+        },
+        {
+          "name": "stop_when_text",
+          "type": "string",
+          "description": "Stop when this text appears",
+          "required": false
+        }
+      ]
+    }
+  ],
+  "prompts": [
+    {
+      "name": "realtime_watch",
+      "description": "Snapshot live DOM mutations, active reflexes, and Canvas2D entities, then guide the agent to wire a 60Hz reflex rule.",
+      "arguments": [
+        {
+          "name": "goal",
+          "description": "What the reflex should watch for and do",
+          "required": true
+        },
+        {
+          "name": "mode",
+          "description": "Trigger mode (auto, dom, canvas)",
+          "required": false
+        }
+      ]
+    }
+  ],
+  "resources": [
+    {
+      "name": "realtime_canvas_feed",
+      "uri": "realtime://canvas",
+      "description": "Live snapshot of Canvas2D draw counts, rAF frame rate, and active moving entity bounding boxes.",
+      "mimeType": "application/json"
+    },
+    {
+      "name": "realtime_feed",
+      "uri": "realtime://feed",
+      "description": "Live snapshot of newly mutated DOM elements (with click distance), active 60Hz reflex rules, and tracked Canvas2D entities.",
+      "mimeType": "application/json"
+    }
+  ],
+  "tools": [
+    {
+      "name": "realtime_canvas",
+      "description": "Records Canvas2D draw calls across requestAnimationFrame ticks and returns moving/static entities with bounding boxes and px/frame velocity.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "action": {
+            "type": "string",
+            "enum": [
+              "sample",
+              "attach",
+              "detach"
+            ],
+            "description": "sample (default), attach, or detach"
+          },
+          "frames": {
+            "type": "number",
+            "description": "Frames to sample (2-15, default 6)"
+          }
+        }
+      }
+    },
+    {
+      "name": "realtime_reflex",
+      "description": "Registers, removes, lists, or clears in-page 60Hz requestAnimationFrame rules that click or press keys on DOM or Canvas2D triggers without an LLM round-trip.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "action": {
+            "type": "string",
+            "enum": [
+              "add",
+              "remove",
+              "list",
+              "clear"
+            ],
+            "description": "Operation to perform"
+          },
+          "id": {
+            "type": "string",
+            "description": "Rule identifier"
+          },
+          "when": {
+            "type": "object",
+            "description": "DOM selector or Canvas2D expression trigger"
+          },
+          "do": {
+            "type": "object",
+            "description": "Response action (click, click_xy, key, js)"
+          },
+          "cooldown_ms": {
+            "type": "number",
+            "description": "Minimum ms between firings (default 100)"
+          },
+          "max_hits": {
+            "type": "number",
+            "description": "Auto-stop after N hits (default 50)"
+          },
+          "ttl_ms": {
+            "type": "number",
+            "description": "Rule lifetime in ms (default 30000)"
+          },
+          "stop_selector": {
+            "type": "string",
+            "description": "Selector that halts the rule when present"
+          }
+        },
+        "required": [
+          "action"
+        ]
+      }
+    },
+    {
+      "name": "realtime_autopilot",
+      "description": "Runs a bounded in-page click loop toward a short goal ('accept cookies', 'continue') with causal age/dialog scoring and optional local model pick.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "goal": {
+            "type": "string",
+            "description": "Short natural-language click goal"
+          },
+          "max_steps": {
+            "type": "number",
+            "description": "Maximum clicks (default 8, max 25)"
+          },
+          "stop_when_selector": {
+            "type": "string",
+            "description": "Stop when selector becomes visible"
+          },
+          "stop_when_text": {
+            "type": "string",
+            "description": "Stop when body text contains string"
+          },
+          "stop_when_url": {
+            "type": "string",
+            "description": "Stop when URL matches regex"
+          }
+        },
+        "required": [
+          "goal"
+        ]
+      }
+    },
+    {
+      "name": "realtime_status",
+      "description": "Returns the live state of the in-page probe, Canvas2D spy, and active 60Hz reflex rules. Optionally waits until reflexes complete.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "wait_ms": {
+            "type": "number",
+            "description": "Wait up to wait_ms (max 30000) for active reflexes to finish"
+          }
+        }
+      }
+    }
+  ]
+}