@weavetab/plugin-realtime 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/AGENTS.md ADDED
@@ -0,0 +1,19 @@
1
+ # Realtime Plugin — Agent Playbook
2
+
3
+ 60Hz in-page reflex engine, causal DOM age/proximity tagging, Canvas2D entity tracking, and bounded click autopilot (`@weavetab/plugin-realtime`).
4
+
5
+ ## MCP Tools
6
+ - `realtime_canvas`: Sample Canvas2D draw calls (`sample`, `attach`, `detach`, `frames`) to get entities and px/frame velocity.
7
+ - `realtime_reflex`: Arm, remove, list, or clear 60Hz `requestAnimationFrame` DOM or Canvas2D reflex rules (`action`, `when`, `do`).
8
+ - `realtime_autopilot`: Run a bounded click loop toward a short goal (`goal`, `stop_when_selector`, `stop_when_text`).
9
+ - `realtime_status`: Inspect probe status, canvas frame count, and reflex hits (`wait_ms`).
10
+
11
+ ## CLI Commands (`weavetab plugin use realtime <cmd>`)
12
+ - `weavetab plugin use realtime status [--wait_ms 2000]`: Inspect live probe and reflex rules.
13
+ - `weavetab plugin use realtime reflex-clear`: Clear all active 60Hz reflex rules on the active tab.
14
+ - `weavetab plugin use realtime autopilot --goal <text>`: Run bounded click autopilot via CLI.
15
+
16
+ ## Utilities (`/` Prompts & `@` Resources)
17
+ - `/realtime_watch`: Snapshot live DOM/canvas state and wire a 60Hz reflex rule.
18
+ - `@realtime://feed`: Live JSON feed of newly added DOM elements, reflexes, and canvas entities.
19
+ - `@realtime://canvas`: Live inspection of Canvas2D draw count, rAF FPS, and moving bounding boxes.
package/dist/canvas.js CHANGED
@@ -69,12 +69,32 @@ const SPY = `(() => {
69
69
  });
70
70
  };
71
71
 
72
+ const matchSig = (pattern, s) => {
73
+ if (!pattern) return false;
74
+ if (pattern === '*' || pattern === true) return true;
75
+ if (Array.isArray(pattern)) return pattern.some((p) => matchSig(p, s));
76
+ if (typeof pattern !== 'string') return false;
77
+ if (pattern.indexOf('*') < 0) return s === pattern;
78
+ const parts = pattern.split('*');
79
+ if (!s.startsWith(parts[0])) return false;
80
+ if (!s.endsWith(parts[parts.length - 1])) return false;
81
+ let idx = 0;
82
+ for (let i = 0; i < parts.length; i++) {
83
+ const part = parts[i];
84
+ if (!part) continue;
85
+ const found = s.indexOf(part, idx);
86
+ if (found < 0) return false;
87
+ idx = found + part.length;
88
+ }
89
+ return true;
90
+ };
91
+
72
92
  cv.track = (sig, depth) => {
73
93
  const fr = cv.frames;
74
94
  if (!fr.length) return [];
75
- const last = fr[fr.length - 1].d.filter((d) => d.sig === sig);
95
+ const last = fr[fr.length - 1].d.filter((d) => matchSig(sig, d.sig));
76
96
  if (last.length > 200) return last.map((d) => Object.assign({ vx: 0, vy: 0 }, d));
77
- const prevs = fr.slice(Math.max(0, fr.length - 1 - depth), -1).reverse().map((f) => f.d.filter((d) => d.sig === sig));
97
+ const prevs = fr.slice(Math.max(0, fr.length - 1 - (depth || 2)), -1).reverse().map((f) => f.d.filter((d) => matchSig(sig, d.sig)));
78
98
  return last.map((d) => {
79
99
  let cur = d, steps = 0;
80
100
  for (const p of prevs) {
package/dist/index.d.ts CHANGED
@@ -1,3 +1,3 @@
1
- declare const plugin: import("@weavetab/sdk").WeavetabPluginDefinition;
2
- export default plugin;
3
- export { plugin };
1
+ export declare const realtimePlugin: import("@weavetab/sdk").WeavetabPluginDefinition;
2
+ export default realtimePlugin;
3
+ export declare const plugin: import("@weavetab/sdk").WeavetabPluginDefinition;
package/dist/index.js CHANGED
@@ -1,61 +1,133 @@
1
- import { definePlugin, defineTool, definePrompt, defineResource, z } from "@weavetab/sdk";
2
- import { arm, sessOf } from "./page.js";
1
+ import { definePlugin, defineTool, defineCommand, definePrompt, defineResource, z, } from "@weavetab/sdk";
2
+ import { sessOf, arm } from "./page.js";
3
3
  import { tagMap } from "./causal.js";
4
4
  import { canvasTool } from "./canvas.js";
5
5
  import { reflexTool, statusTool } from "./reflex.js";
6
6
  import { autopilotTool } from "./autopilot.js";
7
7
  import { readFeed } from "./surface.js";
8
- let winMs = 4000;
9
- const asText = (v) => ({ content: [{ type: "text", text: JSON.stringify(v, null, 2) }] });
10
- const plugin = definePlugin({
11
- name: "@weavetab/plugin-realtime",
12
- version: "0.2.0",
13
- description: "Causal tags for browser_map, in-page reflex rules, canvas entity sampling, bounded click loop.",
14
- defaultConfig: { causalWindowMs: 4000 },
15
- permissions: {
16
- cdp: { evaluate: true, input: true, reason: "Page probe, canvas spy, 60Hz reflex loop, and trusted CDP mouse clicks." },
17
- tools: { add: true, reason: "Registers realtime_canvas, realtime_reflex, realtime_autopilot and realtime_status." },
18
- prompts: { add: true, reason: "Registers /realtime_watch to guide setting up 60Hz DOM or canvas reflexes." },
19
- resources: { add: true, reason: "Registers @realtime_feed (realtime://feed) for live reflex, DOM, and canvas telemetry." },
20
- hooks: { onBeforeAction: true, onAfterAction: true, reason: "Arms page probe and appends {new ...} causal tags to browser_map." },
21
- network: { hosts: ["127.0.0.1", "localhost"], reason: "Optional local model_url in realtime_autopilot." },
8
+ const statusCommand = defineCommand({
9
+ name: "status",
10
+ description: "Read the in-page realtime probe state, canvas frame counter, and active 60Hz reflex rules.",
11
+ usage: "weavetab plugin use realtime status [--wait_ms <ms>]",
12
+ requiresSession: true,
13
+ schema: {
14
+ wait_ms: z.number().optional().describe("Block up to wait_ms until active reflexes finish"),
22
15
  },
23
- async onBeforeAction(_tool, _args, sess) {
24
- if (sess?.Runtime)
25
- await arm(sess).catch(() => { }); // page may be mid-navigation, next call retries
16
+ handler: async (args, ctx) => {
17
+ const sess = sessOf(ctx);
18
+ return await statusTool(sess, Number(args.wait_ms || 0));
26
19
  },
27
- async onAfterAction(tool, res, sess) {
28
- if (tool !== "browser_map" || !sess?.Runtime)
29
- return;
30
- return tagMap(sess, res, winMs).catch(() => undefined);
20
+ });
21
+ const reflexClearCommand = defineCommand({
22
+ name: "reflex-clear",
23
+ description: "Clear all active 60Hz reflex rules on the active browser tab via CLI.",
24
+ usage: "weavetab plugin use realtime reflex-clear",
25
+ requiresSession: true,
26
+ handler: async (_args, ctx) => {
27
+ const sess = sessOf(ctx);
28
+ return await reflexTool(sess, { action: "clear" });
29
+ },
30
+ });
31
+ const autopilotCommand = defineCommand({
32
+ name: "autopilot",
33
+ description: "Run bounded in-page autopilot toward a goal directly via CLI.",
34
+ usage: "weavetab plugin use realtime autopilot --goal <text> [--max_steps <n>]",
35
+ requiresSession: true,
36
+ schema: {
37
+ goal: z.string().describe("Natural language click goal"),
38
+ max_steps: z.number().optional().describe("Maximum click steps (default 5)"),
39
+ stop_when_text: z.string().optional().describe("Stop when this text appears"),
40
+ },
41
+ handler: async (args, ctx) => {
42
+ const sess = sessOf(ctx);
43
+ return await autopilotTool(sess, {
44
+ goal: args.goal,
45
+ max_steps: args.max_steps ?? 5,
46
+ stop_when_text: args.stop_when_text,
47
+ });
31
48
  },
49
+ });
50
+ export const realtimePlugin = definePlugin({
51
+ name: "@weavetab/plugin-realtime",
52
+ version: "0.3.0",
53
+ description: "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop.",
54
+ agentsMd: "AGENTS.md",
55
+ defaultConfig: {
56
+ causalWindowMs: 4000,
57
+ },
58
+ permissions: {
59
+ cdp: {
60
+ evaluate: true,
61
+ input: true,
62
+ reason: "Installs the in-page MutationObserver, hooks Canvas2D/rAF, and dispatches click/key events.",
63
+ },
64
+ tools: {
65
+ add: true,
66
+ reason: "Registers realtime_canvas, realtime_reflex, realtime_autopilot, and realtime_status.",
67
+ },
68
+ commands: {
69
+ add: true,
70
+ reason: "Registers status and reflex-clear CLI commands.",
71
+ },
72
+ prompts: {
73
+ add: true,
74
+ reason: "Registers the /realtime_watch prompt utility.",
75
+ },
76
+ resources: {
77
+ add: true,
78
+ reason: "Registers the @realtime_feed resource utility.",
79
+ },
80
+ hooks: {
81
+ onBeforeAction: true,
82
+ onAfterAction: true,
83
+ reason: "Arms the in-page mutation probe before actions and tags newly added nodes in browser_map output.",
84
+ },
85
+ network: {
86
+ domains: ["127.0.0.1", "localhost"],
87
+ reason: "Allows realtime_autopilot to query an optional local OpenAI-compatible model endpoint.",
88
+ },
89
+ },
90
+ commands: [statusCommand, reflexClearCommand, autopilotCommand],
32
91
  async onLoad(ctx) {
33
- winMs = Number(ctx.config?.causalWindowMs) || 4000;
34
- const reg = ctx.mcp.registerTool;
35
- reg(defineTool({
92
+ const winMs = typeof ctx.config?.causalWindowMs === "number" ? ctx.config.causalWindowMs : 4000;
93
+ ctx.cli?.registerCommand?.(statusCommand);
94
+ ctx.cli?.registerCommand?.(reflexClearCommand);
95
+ ctx.cli?.registerCommand?.(autopilotCommand);
96
+ ctx.mcp.registerTool(defineTool({
36
97
  name: "realtime_canvas",
37
- description: "Sample what a 2D <canvas> game draws. 'sample' watches a few frames and returns entities (signature, bbox, px/frame velocity) in canvas pixels, plus canvas position/scale to convert to viewport coords. Only sees Canvas2D draw calls made inside requestAnimationFrame; WebGL is invisible to it. 'attach' starts recording early, 'detach' restores the page.",
98
+ description: "Records Canvas2D draw calls across requestAnimationFrame ticks and returns moving/static entities with bounding boxes and px/frame velocity.",
38
99
  requiresSession: true,
39
100
  schema: {
40
- action: z.enum(["sample", "attach", "detach"]).default("sample"),
41
- frames: z.number().int().min(2).max(15).optional().describe("frames to watch, default 6"),
101
+ action: z.enum(["sample", "attach", "detach"]).optional().describe("sample (default), attach, or detach"),
102
+ frames: z.number().optional().describe("Frames to sample (2-15, default 6)"),
103
+ },
104
+ handler: async (args, rawCtx) => {
105
+ const out = await canvasTool(sessOf(rawCtx), {
106
+ action: args.action || "sample",
107
+ frames: args.frames,
108
+ });
109
+ return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
42
110
  },
43
- handler: async (a, ctx) => asText(await canvasTool(sessOf(ctx), a)),
44
111
  }));
45
- reg(defineTool({
112
+ ctx.mcp.registerTool(defineTool({
46
113
  name: "realtime_reflex",
47
- description: "Register a rule that runs inside the page every animation frame, no LLM round-trip. Trigger is either a DOM selector (optionally text / only_new) or a canvas expression over named entity signatures from realtime_canvas: names {player:'rect|20x20|#f00'}, js \"gap(E.player[0], E.cactus[0]) < 50\" (E.name = array of {x,y,w,h,vx,vy}; dist() and gap() return Infinity when an entity is missing). Action: click, click_xy, key, or js. Events are synthetic, sites that check isTrusted ignore them. Needs a visible tab.",
114
+ description: "Registers, removes, lists, or clears in-page 60Hz requestAnimationFrame rules that click or press keys on DOM or Canvas2D triggers without an LLM round-trip.",
48
115
  requiresSession: true,
49
116
  schema: {
50
- action: z.enum(["add", "remove", "list", "clear"]).default("add"),
51
- id: z.string().optional(),
117
+ action: z.enum(["add", "remove", "list", "clear"]).describe("Operation to perform"),
118
+ id: z.string().optional().describe("Rule identifier"),
52
119
  when: z
53
120
  .object({
54
121
  selector: z.string().optional(),
55
- text: z.string().optional().describe("case-insensitive substring of label/text"),
56
- only_new: z.boolean().optional().describe("only elements added after this rule was created"),
57
- enabled: z.boolean().optional().describe("skip disabled elements, default true"),
58
- canvas: z.object({ names: z.record(z.string()), js: z.string() }).optional(),
122
+ text: z.string().optional(),
123
+ only_new: z.boolean().optional(),
124
+ enabled: z.boolean().optional(),
125
+ canvas: z
126
+ .object({
127
+ names: z.record(z.string(), z.union([z.string(), z.array(z.string())])),
128
+ js: z.string(),
129
+ })
130
+ .optional(),
59
131
  })
60
132
  .optional(),
61
133
  do: z
@@ -64,72 +136,107 @@ const plugin = definePlugin({
64
136
  x: z.number().optional(),
65
137
  y: z.number().optional(),
66
138
  key: z.string().optional(),
67
- hold_ms: z.number().optional().describe("key hold time, default 40"),
139
+ hold_ms: z.number().optional(),
68
140
  code: z.string().optional(),
69
141
  })
70
142
  .optional(),
71
- cooldown_ms: z.number().optional().describe("default 100"),
72
- max_hits: z.number().optional().describe("default 50"),
73
- ttl_ms: z.number().optional().describe("default 30000, max 600000"),
143
+ cooldown_ms: z.number().optional(),
144
+ max_hits: z.number().optional(),
145
+ ttl_ms: z.number().optional(),
74
146
  stop_selector: z.string().optional(),
75
147
  },
76
- handler: async (a, ctx) => asText(await reflexTool(sessOf(ctx), a)),
148
+ handler: async (args, rawCtx) => {
149
+ const out = await reflexTool(sessOf(rawCtx), args);
150
+ return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
151
+ },
77
152
  }));
78
- reg(defineTool({
153
+ ctx.mcp.registerTool(defineTool({
79
154
  name: "realtime_autopilot",
80
- description: "Bounded click loop for short UI sub-goals ('accept cookies', 'click next until Order summary shows'). Ranks visible, unoccluded clickable elements by word overlap with the goal, boosts ones that just appeared and ones inside an open dialog, and never re-clicks a button whose last click changed nothing. Trusted CDP mouse clicks. Skips delete/pay/sign-out style labels unless the goal says so. Not a planner: stops with no_match when no label overlaps the goal. Optional model_url = OpenAI-compatible chat endpoint to pick instead; falls back to the ranker on any failure.",
155
+ description: "Runs a bounded in-page click loop toward a short goal ('accept cookies', 'continue') with causal age/dialog scoring and optional local model pick.",
81
156
  requiresSession: true,
82
157
  schema: {
83
- goal: z.string().min(3),
84
- max_steps: z.number().int().min(1).max(25).optional().describe("default 8"),
158
+ goal: z.string().describe("Short natural-language click goal"),
159
+ max_steps: z.number().optional(),
85
160
  stop_when_selector: z.string().optional(),
86
161
  stop_when_text: z.string().optional(),
87
- stop_when_url: z.string().optional().describe("regex tested against location.href"),
88
- avoid_labels: z.string().optional().describe("regex of extra labels never to click"),
89
- model_url: z.string().url().optional(),
162
+ stop_when_url: z.string().optional(),
163
+ avoid_labels: z.string().optional(),
164
+ model_url: z.string().optional(),
90
165
  model: z.string().optional(),
91
- model_timeout_ms: z.number().optional().describe("default 1500"),
166
+ model_timeout_ms: z.number().optional(),
167
+ },
168
+ handler: async (args, rawCtx) => {
169
+ const out = await autopilotTool(sessOf(rawCtx), args);
170
+ return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
92
171
  },
93
- handler: async (a, ctx) => asText(await autopilotTool(sessOf(ctx), a)),
94
172
  }));
95
- reg(defineTool({
173
+ ctx.mcp.registerTool(defineTool({
96
174
  name: "realtime_status",
97
- description: "Probe/canvas/reflex state for the current page. wait_ms (max 30000) blocks until every active reflex has finished, so the agent can sleep through a game or a drop-watch.",
175
+ description: "Returns the live state of the in-page probe, Canvas2D spy, and active 60Hz reflex rules. Optionally waits until reflexes complete.",
98
176
  requiresSession: true,
99
- schema: { wait_ms: z.number().optional() },
100
- handler: async (a, ctx) => asText(await statusTool(sessOf(ctx), a.wait_ms)),
177
+ schema: {
178
+ wait_ms: z.number().optional().describe("Wait up to wait_ms (max 30000) for active reflexes to finish"),
179
+ },
180
+ handler: async (args, rawCtx) => {
181
+ const out = await statusTool(sessOf(rawCtx), args.wait_ms ?? 0);
182
+ return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
183
+ },
101
184
  }));
102
- ctx.mcp.registerPrompt?.(definePrompt({
185
+ ctx.mcp.registerPrompt(definePrompt({
103
186
  name: "realtime_watch",
104
- title: "Watch & React (60Hz Reflex)",
105
- description: "Set up an in-page 60Hz reflex rule or canvas game loop and wait until it finishes.",
187
+ description: "Snapshot live DOM mutations, active reflexes, and Canvas2D entities, then guide the agent to wire a 60Hz reflex rule.",
106
188
  arguments: {
107
- goal: z.string().describe("What to watch for and trigger (e.g. 'click Claim as soon as it appears' or 'jump over green obstacles in the canvas game')"),
108
- mode: z.string().optional().describe("'dom', 'canvas', or 'auto' (default 'auto')"),
189
+ goal: z.string().describe("What the reflex should watch for and do"),
190
+ mode: z.enum(["auto", "dom", "canvas"]).optional().describe("Trigger mode (default: auto)"),
109
191
  },
110
- handler: async ({ goal, mode }, sctx) => {
111
- const feed = await readFeed(sessOf(sctx), winMs).catch(() => ({ active_tab: false }));
112
- const m = (mode || "auto").toLowerCase();
192
+ handler: async (args, rawCtx) => {
193
+ const feed = await readFeed(sessOf(rawCtx), winMs);
113
194
  return [
114
- `Goal: ${goal}`,
115
- `Mode: ${m}`,
116
- `Live state: ${JSON.stringify(feed)}`,
117
- m === "canvas"
118
- ? "1. Call realtime_canvas with action='sample' to inspect moving entity signatures and velocities.\n2. Arm realtime_reflex with when.canvas ({ names, js }) using gap(E.a[0], E.b[0]) or dist(E.a[0], E.b[0]).\n3. Call realtime_status with wait_ms to block until the reflex completes."
119
- : "1. If targeting a 2D <canvas>, run realtime_canvas (action='sample') first to grab entity signatures; if targeting DOM changes, pick a selector/text filter (set only_new=true to ignore elements already on screen).\n2. Arm the rule with realtime_reflex (action='add').\n3. Call realtime_status with wait_ms (up to 30000) to sleep until the reflex hits max_hits or stop_selector.",
120
- ].join("\n\n");
195
+ `Goal: ${args.goal}`,
196
+ `Mode: ${args.mode || "auto"}`,
197
+ `Live feed: ${JSON.stringify(feed)}`,
198
+ `Use realtime_canvas, realtime_reflex, and realtime_status to arm and verify the rule.`,
199
+ ].join("\n");
121
200
  },
122
201
  }));
123
- ctx.mcp.registerResource?.(defineResource({
202
+ ctx.mcp.registerResource(defineResource({
203
+ name: "realtime_canvas_feed",
204
+ uri: "realtime://canvas",
205
+ description: "Live snapshot of Canvas2D draw counts, rAF frame rate, and active moving entity bounding boxes.",
206
+ mimeType: "application/json",
207
+ read: async (_uri, _vars, rawCtx) => {
208
+ const sess = sessOf(rawCtx);
209
+ const canvasData = await canvasTool(sess, { action: "sample", frames: 3 });
210
+ return JSON.stringify(canvasData, null, 2);
211
+ },
212
+ }));
213
+ ctx.mcp.registerResource(defineResource({
124
214
  name: "realtime_feed",
125
- title: "Realtime Page Feed",
126
215
  uri: "realtime://feed",
216
+ description: "Live snapshot of newly mutated DOM elements (with click distance), active 60Hz reflex rules, and tracked Canvas2D entities.",
127
217
  mimeType: "application/json",
128
- description: "Live snapshot of active 60Hz reflex rules, recently mutated DOM elements with click distance, and tracked Canvas2D entities.",
129
- read: async (_uri, _vars, sctx) => JSON.stringify(await readFeed(sessOf(sctx), winMs), null, 2),
218
+ read: async (_uri, _vars, rawCtx) => {
219
+ const feed = await readFeed(sessOf(rawCtx), winMs);
220
+ return JSON.stringify(feed, null, 2);
221
+ },
130
222
  }));
131
- ctx.logger?.info("realtime up");
223
+ ctx.logger?.info("@weavetab/plugin-realtime loaded");
224
+ },
225
+ async onBeforeAction(_action, _args, rawCtx) {
226
+ const sess = sessOf(rawCtx);
227
+ if (sess?.Runtime) {
228
+ await arm(sess).catch(() => { });
229
+ }
230
+ },
231
+ async onAfterAction(action, result, rawCtx) {
232
+ if (action !== "browser_map")
233
+ return;
234
+ const sess = sessOf(rawCtx);
235
+ if (!sess?.Runtime)
236
+ return;
237
+ const winMs = typeof rawCtx?.config?.causalWindowMs === "number" ? rawCtx.config.causalWindowMs : 4000;
238
+ return await tagMap(sess, result, winMs).catch(() => undefined);
132
239
  },
133
240
  });
134
- export default plugin;
135
- export { plugin };
241
+ export default realtimePlugin;
242
+ export const plugin = realtimePlugin;
package/dist/reflex.d.ts CHANGED
@@ -7,7 +7,7 @@ type Rule = {
7
7
  only_new?: boolean;
8
8
  enabled?: boolean;
9
9
  canvas?: {
10
- names: Record<string, string>;
10
+ names: Record<string, string | string[]>;
11
11
  js: string;
12
12
  };
13
13
  };
package/dist/reflex.js CHANGED
@@ -16,7 +16,11 @@ const RUNTIME = `(() => {
16
16
  const o = { key: k, code: c[0], keyCode: c[1], which: c[1], bubbles: true, cancelable: true };
17
17
  const tgt = document.activeElement || document.body;
18
18
  tgt.dispatchEvent(new KeyboardEvent('keydown', o));
19
- setTimeout(() => tgt.dispatchEvent(new KeyboardEvent('keyup', o)), hold);
19
+ if (tgt !== document) document.dispatchEvent(new KeyboardEvent('keydown', o));
20
+ setTimeout(() => {
21
+ tgt.dispatchEvent(new KeyboardEvent('keyup', o));
22
+ if (tgt !== document) document.dispatchEvent(new KeyboardEvent('keyup', o));
23
+ }, hold);
20
24
  };
21
25
  const click = (el, x, y) => {
22
26
  if (x == null) { const b = el.getBoundingClientRect(); x = b.left + b.width / 2; y = b.top + b.height / 2; }
@@ -48,8 +52,12 @@ const RUNTIME = `(() => {
48
52
  const dist = (a, b) => (a && b) ? Math.hypot(a.x + a.w / 2 - b.x - b.w / 2, a.y + a.h / 2 - b.y - b.h / 2) : Infinity;
49
53
  const gap = (a, b) => (a && b) ? b.x - (a.x + a.w) : Infinity;
50
54
  const E = new Proxy({}, { get: (_, k) => {
51
- const cv = window.__wt_cv, sig = rf.names && rf.names[k];
52
- return cv && sig ? cv.track(sig, 2) : [];
55
+ const cv = window.__wt_cv;
56
+ if (!cv) return [];
57
+ if (k === 'all') return cv.track('*', 2);
58
+ if (k === 'moving') return cv.track('*', 2).filter((e) => Math.abs(e.vx) > 0.1 || Math.abs(e.vy) > 0.1);
59
+ const sig = rf.names && rf.names[k];
60
+ return sig ? cv.track(sig, 2) : [];
53
61
  } });
54
62
 
55
63
  const tick = () => {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@weavetab/plugin-realtime",
3
- "version": "0.2.0",
3
+ "version": "0.3.0",
4
4
  "description": "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop for Weavetab.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -15,6 +15,7 @@
15
15
  "dist",
16
16
  "weavetab.json",
17
17
  "README.md",
18
+ "AGENTS.md",
18
19
  "LICENSE"
19
20
  ],
20
21
  "scripts": {
@@ -25,7 +26,7 @@
25
26
  "prepublishOnly": "npm run build"
26
27
  },
27
28
  "dependencies": {
28
- "@weavetab/sdk": "^2.6.1"
29
+ "@weavetab/sdk": "^2.6.2"
29
30
  },
30
31
  "devDependencies": {
31
32
  "@types/node": "^22.10.0",
package/weavetab.json CHANGED
@@ -2,157 +2,269 @@
2
2
  "$schema": "https://weavetab.dev/schema/plugin.json",
3
3
  "name": "@weavetab/plugin-realtime",
4
4
  "displayName": "Realtime",
5
- "version": "0.2.0",
6
- "description": "Causal tags for browser_map, in-page reflex rules, canvas entity sampling and a bounded click loop for Weavetab.",
5
+ "version": "0.3.0",
6
+ "description": "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop.",
7
7
  "entry": "dist/index.js",
8
+ "agentsMd": "AGENTS.md",
9
+ "author": {
10
+ "name": "fy2ne",
11
+ "url": "https://fy2ne.me"
12
+ },
13
+ "license": "Apache-2.0",
14
+ "homepage": "https://github.com/Weavetab/Realtime#readme",
15
+ "repository": "https://github.com/Weavetab/Realtime",
8
16
  "engines": {
9
- "weavetab": ">=2.6.1",
17
+ "weavetab": "^2.6.2",
10
18
  "node": ">=18.0.0"
11
19
  },
12
- "environments": ["mcp"],
13
- "authors": [{ "name": "fy2ne", "role": "Author", "url": "https://fy2ne.me" }],
14
- "license": "Apache-2.0",
15
- "tags": ["Canvas", "Games", "CDP", "Automation", "Community"],
16
- "defaultConfig": { "causalWindowMs": 4000 },
17
- "links": { "github": "https://github.com/Weavetab/Realtime", "npm": "https://www.npmjs.com/package/@weavetab/plugin-realtime" },
20
+ "environments": [
21
+ "mcp",
22
+ "cli",
23
+ "browser"
24
+ ],
25
+ "keywords": [
26
+ "weavetab",
27
+ "weavetab-plugin",
28
+ "cdp",
29
+ "canvas",
30
+ "browser-automation"
31
+ ],
32
+ "defaultConfig": {
33
+ "causalWindowMs": 4000
34
+ },
18
35
  "permissions": {
19
36
  "cdp": {
20
37
  "evaluate": true,
21
38
  "input": true,
22
- "reason": "Injects a MutationObserver, canvas draw recorder and reflex loop into the page, and sends real mouse clicks for realtime_autopilot."
39
+ "reason": "Installs the in-page MutationObserver, hooks Canvas2D/rAF, and dispatches click/key events."
23
40
  },
24
41
  "tools": {
25
42
  "add": true,
26
- "reason": "Registers realtime_canvas, realtime_reflex, realtime_autopilot and realtime_status."
43
+ "reason": "Registers realtime_canvas, realtime_reflex, realtime_autopilot, and realtime_status."
44
+ },
45
+ "commands": {
46
+ "add": true,
47
+ "reason": "Registers status and reflex-clear CLI commands."
27
48
  },
28
49
  "prompts": {
29
50
  "add": true,
30
- "reason": "Registers /realtime_watch to guide setting up 60Hz DOM or canvas reflexes with live page state."
51
+ "reason": "Registers the /realtime_watch prompt utility."
31
52
  },
32
53
  "resources": {
33
54
  "add": true,
34
- "reason": "Registers @realtime_feed (realtime://feed) for live reflex, causal DOM, and canvas entity telemetry."
55
+ "reason": "Registers the @realtime_feed resource utility."
35
56
  },
36
57
  "hooks": {
37
58
  "onBeforeAction": true,
38
59
  "onAfterAction": true,
39
- "reason": "Installs the page probe before any tool runs and appends {new ...} tags to browser_map output."
60
+ "reason": "Arms the in-page mutation probe before actions and tags newly added nodes in browser_map output."
40
61
  },
41
62
  "network": {
42
- "hosts": ["127.0.0.1", "localhost"],
43
- "reason": "realtime_autopilot can POST candidates to a model_url the caller supplies (meant for a local OpenAI-compatible server)."
63
+ "domains": [
64
+ "127.0.0.1",
65
+ "localhost"
66
+ ],
67
+ "reason": "Allows realtime_autopilot to query an optional local OpenAI-compatible model endpoint."
44
68
  }
45
69
  },
70
+ "commands": [
71
+ {
72
+ "name": "status",
73
+ "description": "Read the in-page realtime probe state, canvas frame counter, and active 60Hz reflex rules.",
74
+ "usage": "weavetab plugin use realtime status [--wait_ms <ms>]",
75
+ "requiresSession": true,
76
+ "options": [
77
+ {
78
+ "name": "wait_ms",
79
+ "type": "number",
80
+ "description": "Block up to wait_ms until active reflexes finish",
81
+ "required": false
82
+ }
83
+ ]
84
+ },
85
+ {
86
+ "name": "reflex-clear",
87
+ "description": "Clear all active 60Hz reflex rules on the active browser tab via CLI.",
88
+ "usage": "weavetab plugin use realtime reflex-clear",
89
+ "requiresSession": true,
90
+ "options": []
91
+ },
92
+ {
93
+ "name": "autopilot",
94
+ "description": "Run bounded in-page autopilot toward a goal directly via CLI.",
95
+ "usage": "weavetab plugin use realtime autopilot --goal <text> [--max_steps <n>]",
96
+ "requiresSession": true,
97
+ "options": [
98
+ {
99
+ "name": "goal",
100
+ "type": "string",
101
+ "description": "Natural language click goal",
102
+ "required": true
103
+ },
104
+ {
105
+ "name": "max_steps",
106
+ "type": "number",
107
+ "description": "Maximum click steps (default 5)",
108
+ "required": false
109
+ },
110
+ {
111
+ "name": "stop_when_text",
112
+ "type": "string",
113
+ "description": "Stop when this text appears",
114
+ "required": false
115
+ }
116
+ ]
117
+ }
118
+ ],
119
+ "prompts": [
120
+ {
121
+ "name": "realtime_watch",
122
+ "description": "Snapshot live DOM mutations, active reflexes, and Canvas2D entities, then guide the agent to wire a 60Hz reflex rule.",
123
+ "arguments": [
124
+ {
125
+ "name": "goal",
126
+ "description": "What the reflex should watch for and do",
127
+ "required": true
128
+ },
129
+ {
130
+ "name": "mode",
131
+ "description": "Trigger mode (auto, dom, canvas)",
132
+ "required": false
133
+ }
134
+ ]
135
+ }
136
+ ],
137
+ "resources": [
138
+ {
139
+ "name": "realtime_canvas_feed",
140
+ "uri": "realtime://canvas",
141
+ "description": "Live snapshot of Canvas2D draw counts, rAF frame rate, and active moving entity bounding boxes.",
142
+ "mimeType": "application/json"
143
+ },
144
+ {
145
+ "name": "realtime_feed",
146
+ "uri": "realtime://feed",
147
+ "description": "Live snapshot of newly mutated DOM elements (with click distance), active 60Hz reflex rules, and tracked Canvas2D entities.",
148
+ "mimeType": "application/json"
149
+ }
150
+ ],
46
151
  "tools": [
47
152
  {
48
153
  "name": "realtime_canvas",
49
- "description": "Sample what a 2D canvas game draws: entities with signature, bbox and px/frame velocity.",
154
+ "description": "Records Canvas2D draw calls across requestAnimationFrame ticks and returns moving/static entities with bounding boxes and px/frame velocity.",
50
155
  "inputSchema": {
51
156
  "type": "object",
52
157
  "properties": {
53
- "action": { "type": "string", "enum": ["sample", "attach", "detach"], "default": "sample" },
54
- "frames": { "type": "integer", "minimum": 2, "maximum": 15, "description": "frames to watch, default 6" }
158
+ "action": {
159
+ "type": "string",
160
+ "enum": [
161
+ "sample",
162
+ "attach",
163
+ "detach"
164
+ ],
165
+ "description": "sample (default), attach, or detach"
166
+ },
167
+ "frames": {
168
+ "type": "number",
169
+ "description": "Frames to sample (2-15, default 6)"
170
+ }
55
171
  }
56
172
  }
57
173
  },
58
174
  {
59
175
  "name": "realtime_reflex",
60
- "description": "Register, list or remove rules that run inside the page every animation frame.",
176
+ "description": "Registers, removes, lists, or clears in-page 60Hz requestAnimationFrame rules that click or press keys on DOM or Canvas2D triggers without an LLM round-trip.",
61
177
  "inputSchema": {
62
178
  "type": "object",
63
179
  "properties": {
64
- "action": { "type": "string", "enum": ["add", "remove", "list", "clear"], "default": "add" },
65
- "id": { "type": "string" },
180
+ "action": {
181
+ "type": "string",
182
+ "enum": [
183
+ "add",
184
+ "remove",
185
+ "list",
186
+ "clear"
187
+ ],
188
+ "description": "Operation to perform"
189
+ },
190
+ "id": {
191
+ "type": "string",
192
+ "description": "Rule identifier"
193
+ },
66
194
  "when": {
67
195
  "type": "object",
68
- "properties": {
69
- "selector": { "type": "string" },
70
- "text": { "type": "string" },
71
- "only_new": { "type": "boolean" },
72
- "enabled": { "type": "boolean" },
73
- "canvas": {
74
- "type": "object",
75
- "properties": {
76
- "names": { "type": "object", "additionalProperties": { "type": "string" } },
77
- "js": { "type": "string" }
78
- },
79
- "required": ["names", "js"]
80
- }
81
- }
196
+ "description": "DOM selector or Canvas2D expression trigger"
82
197
  },
83
198
  "do": {
84
199
  "type": "object",
85
- "properties": {
86
- "type": { "type": "string", "enum": ["click", "click_xy", "key", "js"] },
87
- "x": { "type": "number" },
88
- "y": { "type": "number" },
89
- "key": { "type": "string" },
90
- "hold_ms": { "type": "number" },
91
- "code": { "type": "string" }
92
- },
93
- "required": ["type"]
200
+ "description": "Response action (click, click_xy, key, js)"
94
201
  },
95
- "cooldown_ms": { "type": "number" },
96
- "max_hits": { "type": "number" },
97
- "ttl_ms": { "type": "number" },
98
- "stop_selector": { "type": "string" }
99
- }
202
+ "cooldown_ms": {
203
+ "type": "number",
204
+ "description": "Minimum ms between firings (default 100)"
205
+ },
206
+ "max_hits": {
207
+ "type": "number",
208
+ "description": "Auto-stop after N hits (default 50)"
209
+ },
210
+ "ttl_ms": {
211
+ "type": "number",
212
+ "description": "Rule lifetime in ms (default 30000)"
213
+ },
214
+ "stop_selector": {
215
+ "type": "string",
216
+ "description": "Selector that halts the rule when present"
217
+ }
218
+ },
219
+ "required": [
220
+ "action"
221
+ ]
100
222
  }
101
223
  },
102
224
  {
103
225
  "name": "realtime_autopilot",
104
- "description": "Bounded click loop for short UI sub-goals, ranked by label/goal word overlap and element recency.",
226
+ "description": "Runs a bounded in-page click loop toward a short goal ('accept cookies', 'continue') with causal age/dialog scoring and optional local model pick.",
105
227
  "inputSchema": {
106
228
  "type": "object",
107
229
  "properties": {
108
- "goal": { "type": "string", "minLength": 3 },
109
- "max_steps": { "type": "integer", "minimum": 1, "maximum": 25 },
110
- "stop_when_selector": { "type": "string" },
111
- "stop_when_text": { "type": "string" },
112
- "stop_when_url": { "type": "string" },
113
- "avoid_labels": { "type": "string" },
114
- "model_url": { "type": "string", "format": "uri" },
115
- "model": { "type": "string" },
116
- "model_timeout_ms": { "type": "number" }
230
+ "goal": {
231
+ "type": "string",
232
+ "description": "Short natural-language click goal"
233
+ },
234
+ "max_steps": {
235
+ "type": "number",
236
+ "description": "Maximum clicks (default 8, max 25)"
237
+ },
238
+ "stop_when_selector": {
239
+ "type": "string",
240
+ "description": "Stop when selector becomes visible"
241
+ },
242
+ "stop_when_text": {
243
+ "type": "string",
244
+ "description": "Stop when body text contains string"
245
+ },
246
+ "stop_when_url": {
247
+ "type": "string",
248
+ "description": "Stop when URL matches regex"
249
+ }
117
250
  },
118
- "required": ["goal"]
251
+ "required": [
252
+ "goal"
253
+ ]
119
254
  }
120
255
  },
121
256
  {
122
257
  "name": "realtime_status",
123
- "description": "Probe, canvas recorder and reflex state for the current page; wait_ms blocks until active reflexes finish.",
258
+ "description": "Returns the live state of the in-page probe, Canvas2D spy, and active 60Hz reflex rules. Optionally waits until reflexes complete.",
124
259
  "inputSchema": {
125
260
  "type": "object",
126
- "properties": { "wait_ms": { "type": "number" } }
127
- }
128
- }
129
- ],
130
- "prompts": [
131
- {
132
- "name": "realtime_watch",
133
- "title": "Watch & React (60Hz Reflex)",
134
- "description": "Set up an in-page 60Hz reflex rule or canvas game loop and wait until it finishes.",
135
- "arguments": [
136
- {
137
- "name": "goal",
138
- "description": "What to watch for and trigger (e.g. 'click Claim as soon as it appears' or 'jump over green obstacles in the canvas game')",
139
- "required": true
140
- },
141
- {
142
- "name": "mode",
143
- "description": "'dom', 'canvas', or 'auto' (default 'auto')",
144
- "required": false
261
+ "properties": {
262
+ "wait_ms": {
263
+ "type": "number",
264
+ "description": "Wait up to wait_ms (max 30000) for active reflexes to finish"
265
+ }
145
266
  }
146
- ]
147
- }
148
- ],
149
- "resources": [
150
- {
151
- "name": "realtime_feed",
152
- "title": "Realtime Page Feed",
153
- "uri": "realtime://feed",
154
- "mimeType": "application/json",
155
- "description": "Live snapshot of active 60Hz reflex rules, recently mutated DOM elements with click distance, and tracked Canvas2D entities."
267
+ }
156
268
  }
157
269
  ]
158
270
  }