@weavetab/plugin-realtime 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +19 -0
- package/dist/canvas.js +22 -2
- package/dist/index.d.ts +3 -3
- package/dist/index.js +185 -78
- package/dist/reflex.d.ts +1 -1
- package/dist/reflex.js +11 -3
- package/package.json +3 -2
- package/weavetab.json +203 -91
package/AGENTS.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Realtime Plugin — Agent Playbook
|
|
2
|
+
|
|
3
|
+
60Hz in-page reflex engine, causal DOM age/proximity tagging, Canvas2D entity tracking, and bounded click autopilot (`@weavetab/plugin-realtime`).
|
|
4
|
+
|
|
5
|
+
## MCP Tools
|
|
6
|
+
- `realtime_canvas`: Sample Canvas2D draw calls (`sample`, `attach`, `detach`, `frames`) to get entities and px/frame velocity.
|
|
7
|
+
- `realtime_reflex`: Arm, remove, list, or clear 60Hz `requestAnimationFrame` DOM or Canvas2D reflex rules (`action`, `when`, `do`).
|
|
8
|
+
- `realtime_autopilot`: Run a bounded click loop toward a short goal (`goal`, `stop_when_selector`, `stop_when_text`).
|
|
9
|
+
- `realtime_status`: Inspect probe status, canvas frame count, and reflex hits (`wait_ms`).
|
|
10
|
+
|
|
11
|
+
## CLI Commands (`weavetab plugin use realtime <cmd>`)
|
|
12
|
+
- `weavetab plugin use realtime status [--wait_ms 2000]`: Inspect live probe and reflex rules.
|
|
13
|
+
- `weavetab plugin use realtime reflex-clear`: Clear all active 60Hz reflex rules on the active tab.
|
|
14
|
+
- `weavetab plugin use realtime autopilot --goal <text>`: Run bounded click autopilot via CLI.
|
|
15
|
+
|
|
16
|
+
## Utilities (`/` Prompts & `@` Resources)
|
|
17
|
+
- `/realtime_watch`: Snapshot live DOM/canvas state and wire a 60Hz reflex rule.
|
|
18
|
+
- `@realtime://feed`: Live JSON feed of newly added DOM elements, reflexes, and canvas entities.
|
|
19
|
+
- `@realtime://canvas`: Live inspection of Canvas2D draw count, rAF FPS, and moving bounding boxes.
|
package/dist/canvas.js
CHANGED
|
@@ -69,12 +69,32 @@ const SPY = `(() => {
|
|
|
69
69
|
});
|
|
70
70
|
};
|
|
71
71
|
|
|
72
|
+
const matchSig = (pattern, s) => {
|
|
73
|
+
if (!pattern) return false;
|
|
74
|
+
if (pattern === '*' || pattern === true) return true;
|
|
75
|
+
if (Array.isArray(pattern)) return pattern.some((p) => matchSig(p, s));
|
|
76
|
+
if (typeof pattern !== 'string') return false;
|
|
77
|
+
if (pattern.indexOf('*') < 0) return s === pattern;
|
|
78
|
+
const parts = pattern.split('*');
|
|
79
|
+
if (!s.startsWith(parts[0])) return false;
|
|
80
|
+
if (!s.endsWith(parts[parts.length - 1])) return false;
|
|
81
|
+
let idx = 0;
|
|
82
|
+
for (let i = 0; i < parts.length; i++) {
|
|
83
|
+
const part = parts[i];
|
|
84
|
+
if (!part) continue;
|
|
85
|
+
const found = s.indexOf(part, idx);
|
|
86
|
+
if (found < 0) return false;
|
|
87
|
+
idx = found + part.length;
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
90
|
+
};
|
|
91
|
+
|
|
72
92
|
cv.track = (sig, depth) => {
|
|
73
93
|
const fr = cv.frames;
|
|
74
94
|
if (!fr.length) return [];
|
|
75
|
-
const last = fr[fr.length - 1].d.filter((d) => d.sig
|
|
95
|
+
const last = fr[fr.length - 1].d.filter((d) => matchSig(sig, d.sig));
|
|
76
96
|
if (last.length > 200) return last.map((d) => Object.assign({ vx: 0, vy: 0 }, d));
|
|
77
|
-
const prevs = fr.slice(Math.max(0, fr.length - 1 - depth), -1).reverse().map((f) => f.d.filter((d) => d.sig
|
|
97
|
+
const prevs = fr.slice(Math.max(0, fr.length - 1 - (depth || 2)), -1).reverse().map((f) => f.d.filter((d) => matchSig(sig, d.sig)));
|
|
78
98
|
return last.map((d) => {
|
|
79
99
|
let cur = d, steps = 0;
|
|
80
100
|
for (const p of prevs) {
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
declare const
|
|
2
|
-
export default
|
|
3
|
-
export
|
|
1
|
+
export declare const realtimePlugin: import("@weavetab/sdk").WeavetabPluginDefinition;
|
|
2
|
+
export default realtimePlugin;
|
|
3
|
+
export declare const plugin: import("@weavetab/sdk").WeavetabPluginDefinition;
|
package/dist/index.js
CHANGED
|
@@ -1,61 +1,133 @@
|
|
|
1
|
-
import { definePlugin, defineTool, definePrompt, defineResource, z } from "@weavetab/sdk";
|
|
2
|
-
import {
|
|
1
|
+
import { definePlugin, defineTool, defineCommand, definePrompt, defineResource, z, } from "@weavetab/sdk";
|
|
2
|
+
import { sessOf, arm } from "./page.js";
|
|
3
3
|
import { tagMap } from "./causal.js";
|
|
4
4
|
import { canvasTool } from "./canvas.js";
|
|
5
5
|
import { reflexTool, statusTool } from "./reflex.js";
|
|
6
6
|
import { autopilotTool } from "./autopilot.js";
|
|
7
7
|
import { readFeed } from "./surface.js";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
permissions: {
|
|
16
|
-
cdp: { evaluate: true, input: true, reason: "Page probe, canvas spy, 60Hz reflex loop, and trusted CDP mouse clicks." },
|
|
17
|
-
tools: { add: true, reason: "Registers realtime_canvas, realtime_reflex, realtime_autopilot and realtime_status." },
|
|
18
|
-
prompts: { add: true, reason: "Registers /realtime_watch to guide setting up 60Hz DOM or canvas reflexes." },
|
|
19
|
-
resources: { add: true, reason: "Registers @realtime_feed (realtime://feed) for live reflex, DOM, and canvas telemetry." },
|
|
20
|
-
hooks: { onBeforeAction: true, onAfterAction: true, reason: "Arms page probe and appends {new ...} causal tags to browser_map." },
|
|
21
|
-
network: { hosts: ["127.0.0.1", "localhost"], reason: "Optional local model_url in realtime_autopilot." },
|
|
8
|
+
const statusCommand = defineCommand({
|
|
9
|
+
name: "status",
|
|
10
|
+
description: "Read the in-page realtime probe state, canvas frame counter, and active 60Hz reflex rules.",
|
|
11
|
+
usage: "weavetab plugin use realtime status [--wait_ms <ms>]",
|
|
12
|
+
requiresSession: true,
|
|
13
|
+
schema: {
|
|
14
|
+
wait_ms: z.number().optional().describe("Block up to wait_ms until active reflexes finish"),
|
|
22
15
|
},
|
|
23
|
-
async
|
|
24
|
-
|
|
25
|
-
|
|
16
|
+
handler: async (args, ctx) => {
|
|
17
|
+
const sess = sessOf(ctx);
|
|
18
|
+
return await statusTool(sess, Number(args.wait_ms || 0));
|
|
26
19
|
},
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
20
|
+
});
|
|
21
|
+
const reflexClearCommand = defineCommand({
|
|
22
|
+
name: "reflex-clear",
|
|
23
|
+
description: "Clear all active 60Hz reflex rules on the active browser tab via CLI.",
|
|
24
|
+
usage: "weavetab plugin use realtime reflex-clear",
|
|
25
|
+
requiresSession: true,
|
|
26
|
+
handler: async (_args, ctx) => {
|
|
27
|
+
const sess = sessOf(ctx);
|
|
28
|
+
return await reflexTool(sess, { action: "clear" });
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
const autopilotCommand = defineCommand({
|
|
32
|
+
name: "autopilot",
|
|
33
|
+
description: "Run bounded in-page autopilot toward a goal directly via CLI.",
|
|
34
|
+
usage: "weavetab plugin use realtime autopilot --goal <text> [--max_steps <n>]",
|
|
35
|
+
requiresSession: true,
|
|
36
|
+
schema: {
|
|
37
|
+
goal: z.string().describe("Natural language click goal"),
|
|
38
|
+
max_steps: z.number().optional().describe("Maximum click steps (default 5)"),
|
|
39
|
+
stop_when_text: z.string().optional().describe("Stop when this text appears"),
|
|
40
|
+
},
|
|
41
|
+
handler: async (args, ctx) => {
|
|
42
|
+
const sess = sessOf(ctx);
|
|
43
|
+
return await autopilotTool(sess, {
|
|
44
|
+
goal: args.goal,
|
|
45
|
+
max_steps: args.max_steps ?? 5,
|
|
46
|
+
stop_when_text: args.stop_when_text,
|
|
47
|
+
});
|
|
31
48
|
},
|
|
49
|
+
});
|
|
50
|
+
export const realtimePlugin = definePlugin({
|
|
51
|
+
name: "@weavetab/plugin-realtime",
|
|
52
|
+
version: "0.3.0",
|
|
53
|
+
description: "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop.",
|
|
54
|
+
agentsMd: "AGENTS.md",
|
|
55
|
+
defaultConfig: {
|
|
56
|
+
causalWindowMs: 4000,
|
|
57
|
+
},
|
|
58
|
+
permissions: {
|
|
59
|
+
cdp: {
|
|
60
|
+
evaluate: true,
|
|
61
|
+
input: true,
|
|
62
|
+
reason: "Installs the in-page MutationObserver, hooks Canvas2D/rAF, and dispatches click/key events.",
|
|
63
|
+
},
|
|
64
|
+
tools: {
|
|
65
|
+
add: true,
|
|
66
|
+
reason: "Registers realtime_canvas, realtime_reflex, realtime_autopilot, and realtime_status.",
|
|
67
|
+
},
|
|
68
|
+
commands: {
|
|
69
|
+
add: true,
|
|
70
|
+
reason: "Registers status and reflex-clear CLI commands.",
|
|
71
|
+
},
|
|
72
|
+
prompts: {
|
|
73
|
+
add: true,
|
|
74
|
+
reason: "Registers the /realtime_watch prompt utility.",
|
|
75
|
+
},
|
|
76
|
+
resources: {
|
|
77
|
+
add: true,
|
|
78
|
+
reason: "Registers the @realtime_feed resource utility.",
|
|
79
|
+
},
|
|
80
|
+
hooks: {
|
|
81
|
+
onBeforeAction: true,
|
|
82
|
+
onAfterAction: true,
|
|
83
|
+
reason: "Arms the in-page mutation probe before actions and tags newly added nodes in browser_map output.",
|
|
84
|
+
},
|
|
85
|
+
network: {
|
|
86
|
+
domains: ["127.0.0.1", "localhost"],
|
|
87
|
+
reason: "Allows realtime_autopilot to query an optional local OpenAI-compatible model endpoint.",
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
commands: [statusCommand, reflexClearCommand, autopilotCommand],
|
|
32
91
|
async onLoad(ctx) {
|
|
33
|
-
winMs =
|
|
34
|
-
|
|
35
|
-
|
|
92
|
+
const winMs = typeof ctx.config?.causalWindowMs === "number" ? ctx.config.causalWindowMs : 4000;
|
|
93
|
+
ctx.cli?.registerCommand?.(statusCommand);
|
|
94
|
+
ctx.cli?.registerCommand?.(reflexClearCommand);
|
|
95
|
+
ctx.cli?.registerCommand?.(autopilotCommand);
|
|
96
|
+
ctx.mcp.registerTool(defineTool({
|
|
36
97
|
name: "realtime_canvas",
|
|
37
|
-
description: "
|
|
98
|
+
description: "Records Canvas2D draw calls across requestAnimationFrame ticks and returns moving/static entities with bounding boxes and px/frame velocity.",
|
|
38
99
|
requiresSession: true,
|
|
39
100
|
schema: {
|
|
40
|
-
action: z.enum(["sample", "attach", "detach"]).
|
|
41
|
-
frames: z.number().
|
|
101
|
+
action: z.enum(["sample", "attach", "detach"]).optional().describe("sample (default), attach, or detach"),
|
|
102
|
+
frames: z.number().optional().describe("Frames to sample (2-15, default 6)"),
|
|
103
|
+
},
|
|
104
|
+
handler: async (args, rawCtx) => {
|
|
105
|
+
const out = await canvasTool(sessOf(rawCtx), {
|
|
106
|
+
action: args.action || "sample",
|
|
107
|
+
frames: args.frames,
|
|
108
|
+
});
|
|
109
|
+
return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
|
|
42
110
|
},
|
|
43
|
-
handler: async (a, ctx) => asText(await canvasTool(sessOf(ctx), a)),
|
|
44
111
|
}));
|
|
45
|
-
|
|
112
|
+
ctx.mcp.registerTool(defineTool({
|
|
46
113
|
name: "realtime_reflex",
|
|
47
|
-
description: "
|
|
114
|
+
description: "Registers, removes, lists, or clears in-page 60Hz requestAnimationFrame rules that click or press keys on DOM or Canvas2D triggers without an LLM round-trip.",
|
|
48
115
|
requiresSession: true,
|
|
49
116
|
schema: {
|
|
50
|
-
action: z.enum(["add", "remove", "list", "clear"]).
|
|
51
|
-
id: z.string().optional(),
|
|
117
|
+
action: z.enum(["add", "remove", "list", "clear"]).describe("Operation to perform"),
|
|
118
|
+
id: z.string().optional().describe("Rule identifier"),
|
|
52
119
|
when: z
|
|
53
120
|
.object({
|
|
54
121
|
selector: z.string().optional(),
|
|
55
|
-
text: z.string().optional()
|
|
56
|
-
only_new: z.boolean().optional()
|
|
57
|
-
enabled: z.boolean().optional()
|
|
58
|
-
canvas: z
|
|
122
|
+
text: z.string().optional(),
|
|
123
|
+
only_new: z.boolean().optional(),
|
|
124
|
+
enabled: z.boolean().optional(),
|
|
125
|
+
canvas: z
|
|
126
|
+
.object({
|
|
127
|
+
names: z.record(z.string(), z.union([z.string(), z.array(z.string())])),
|
|
128
|
+
js: z.string(),
|
|
129
|
+
})
|
|
130
|
+
.optional(),
|
|
59
131
|
})
|
|
60
132
|
.optional(),
|
|
61
133
|
do: z
|
|
@@ -64,72 +136,107 @@ const plugin = definePlugin({
|
|
|
64
136
|
x: z.number().optional(),
|
|
65
137
|
y: z.number().optional(),
|
|
66
138
|
key: z.string().optional(),
|
|
67
|
-
hold_ms: z.number().optional()
|
|
139
|
+
hold_ms: z.number().optional(),
|
|
68
140
|
code: z.string().optional(),
|
|
69
141
|
})
|
|
70
142
|
.optional(),
|
|
71
|
-
cooldown_ms: z.number().optional()
|
|
72
|
-
max_hits: z.number().optional()
|
|
73
|
-
ttl_ms: z.number().optional()
|
|
143
|
+
cooldown_ms: z.number().optional(),
|
|
144
|
+
max_hits: z.number().optional(),
|
|
145
|
+
ttl_ms: z.number().optional(),
|
|
74
146
|
stop_selector: z.string().optional(),
|
|
75
147
|
},
|
|
76
|
-
handler: async (
|
|
148
|
+
handler: async (args, rawCtx) => {
|
|
149
|
+
const out = await reflexTool(sessOf(rawCtx), args);
|
|
150
|
+
return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
|
|
151
|
+
},
|
|
77
152
|
}));
|
|
78
|
-
|
|
153
|
+
ctx.mcp.registerTool(defineTool({
|
|
79
154
|
name: "realtime_autopilot",
|
|
80
|
-
description: "
|
|
155
|
+
description: "Runs a bounded in-page click loop toward a short goal ('accept cookies', 'continue') with causal age/dialog scoring and optional local model pick.",
|
|
81
156
|
requiresSession: true,
|
|
82
157
|
schema: {
|
|
83
|
-
goal: z.string().
|
|
84
|
-
max_steps: z.number().
|
|
158
|
+
goal: z.string().describe("Short natural-language click goal"),
|
|
159
|
+
max_steps: z.number().optional(),
|
|
85
160
|
stop_when_selector: z.string().optional(),
|
|
86
161
|
stop_when_text: z.string().optional(),
|
|
87
|
-
stop_when_url: z.string().optional()
|
|
88
|
-
avoid_labels: z.string().optional()
|
|
89
|
-
model_url: z.string().
|
|
162
|
+
stop_when_url: z.string().optional(),
|
|
163
|
+
avoid_labels: z.string().optional(),
|
|
164
|
+
model_url: z.string().optional(),
|
|
90
165
|
model: z.string().optional(),
|
|
91
|
-
model_timeout_ms: z.number().optional()
|
|
166
|
+
model_timeout_ms: z.number().optional(),
|
|
167
|
+
},
|
|
168
|
+
handler: async (args, rawCtx) => {
|
|
169
|
+
const out = await autopilotTool(sessOf(rawCtx), args);
|
|
170
|
+
return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
|
|
92
171
|
},
|
|
93
|
-
handler: async (a, ctx) => asText(await autopilotTool(sessOf(ctx), a)),
|
|
94
172
|
}));
|
|
95
|
-
|
|
173
|
+
ctx.mcp.registerTool(defineTool({
|
|
96
174
|
name: "realtime_status",
|
|
97
|
-
description: "
|
|
175
|
+
description: "Returns the live state of the in-page probe, Canvas2D spy, and active 60Hz reflex rules. Optionally waits until reflexes complete.",
|
|
98
176
|
requiresSession: true,
|
|
99
|
-
schema: {
|
|
100
|
-
|
|
177
|
+
schema: {
|
|
178
|
+
wait_ms: z.number().optional().describe("Wait up to wait_ms (max 30000) for active reflexes to finish"),
|
|
179
|
+
},
|
|
180
|
+
handler: async (args, rawCtx) => {
|
|
181
|
+
const out = await statusTool(sessOf(rawCtx), args.wait_ms ?? 0);
|
|
182
|
+
return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
|
|
183
|
+
},
|
|
101
184
|
}));
|
|
102
|
-
ctx.mcp.registerPrompt
|
|
185
|
+
ctx.mcp.registerPrompt(definePrompt({
|
|
103
186
|
name: "realtime_watch",
|
|
104
|
-
|
|
105
|
-
description: "Set up an in-page 60Hz reflex rule or canvas game loop and wait until it finishes.",
|
|
187
|
+
description: "Snapshot live DOM mutations, active reflexes, and Canvas2D entities, then guide the agent to wire a 60Hz reflex rule.",
|
|
106
188
|
arguments: {
|
|
107
|
-
goal: z.string().describe("What
|
|
108
|
-
mode: z.
|
|
189
|
+
goal: z.string().describe("What the reflex should watch for and do"),
|
|
190
|
+
mode: z.enum(["auto", "dom", "canvas"]).optional().describe("Trigger mode (default: auto)"),
|
|
109
191
|
},
|
|
110
|
-
handler: async (
|
|
111
|
-
const feed = await readFeed(sessOf(
|
|
112
|
-
const m = (mode || "auto").toLowerCase();
|
|
192
|
+
handler: async (args, rawCtx) => {
|
|
193
|
+
const feed = await readFeed(sessOf(rawCtx), winMs);
|
|
113
194
|
return [
|
|
114
|
-
`Goal: ${goal}`,
|
|
115
|
-
`Mode: ${
|
|
116
|
-
`Live
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
: "1. If targeting a 2D <canvas>, run realtime_canvas (action='sample') first to grab entity signatures; if targeting DOM changes, pick a selector/text filter (set only_new=true to ignore elements already on screen).\n2. Arm the rule with realtime_reflex (action='add').\n3. Call realtime_status with wait_ms (up to 30000) to sleep until the reflex hits max_hits or stop_selector.",
|
|
120
|
-
].join("\n\n");
|
|
195
|
+
`Goal: ${args.goal}`,
|
|
196
|
+
`Mode: ${args.mode || "auto"}`,
|
|
197
|
+
`Live feed: ${JSON.stringify(feed)}`,
|
|
198
|
+
`Use realtime_canvas, realtime_reflex, and realtime_status to arm and verify the rule.`,
|
|
199
|
+
].join("\n");
|
|
121
200
|
},
|
|
122
201
|
}));
|
|
123
|
-
ctx.mcp.registerResource
|
|
202
|
+
ctx.mcp.registerResource(defineResource({
|
|
203
|
+
name: "realtime_canvas_feed",
|
|
204
|
+
uri: "realtime://canvas",
|
|
205
|
+
description: "Live snapshot of Canvas2D draw counts, rAF frame rate, and active moving entity bounding boxes.",
|
|
206
|
+
mimeType: "application/json",
|
|
207
|
+
read: async (_uri, _vars, rawCtx) => {
|
|
208
|
+
const sess = sessOf(rawCtx);
|
|
209
|
+
const canvasData = await canvasTool(sess, { action: "sample", frames: 3 });
|
|
210
|
+
return JSON.stringify(canvasData, null, 2);
|
|
211
|
+
},
|
|
212
|
+
}));
|
|
213
|
+
ctx.mcp.registerResource(defineResource({
|
|
124
214
|
name: "realtime_feed",
|
|
125
|
-
title: "Realtime Page Feed",
|
|
126
215
|
uri: "realtime://feed",
|
|
216
|
+
description: "Live snapshot of newly mutated DOM elements (with click distance), active 60Hz reflex rules, and tracked Canvas2D entities.",
|
|
127
217
|
mimeType: "application/json",
|
|
128
|
-
|
|
129
|
-
|
|
218
|
+
read: async (_uri, _vars, rawCtx) => {
|
|
219
|
+
const feed = await readFeed(sessOf(rawCtx), winMs);
|
|
220
|
+
return JSON.stringify(feed, null, 2);
|
|
221
|
+
},
|
|
130
222
|
}));
|
|
131
|
-
ctx.logger?.info("realtime
|
|
223
|
+
ctx.logger?.info("@weavetab/plugin-realtime loaded");
|
|
224
|
+
},
|
|
225
|
+
async onBeforeAction(_action, _args, rawCtx) {
|
|
226
|
+
const sess = sessOf(rawCtx);
|
|
227
|
+
if (sess?.Runtime) {
|
|
228
|
+
await arm(sess).catch(() => { });
|
|
229
|
+
}
|
|
230
|
+
},
|
|
231
|
+
async onAfterAction(action, result, rawCtx) {
|
|
232
|
+
if (action !== "browser_map")
|
|
233
|
+
return;
|
|
234
|
+
const sess = sessOf(rawCtx);
|
|
235
|
+
if (!sess?.Runtime)
|
|
236
|
+
return;
|
|
237
|
+
const winMs = typeof rawCtx?.config?.causalWindowMs === "number" ? rawCtx.config.causalWindowMs : 4000;
|
|
238
|
+
return await tagMap(sess, result, winMs).catch(() => undefined);
|
|
132
239
|
},
|
|
133
240
|
});
|
|
134
|
-
export default
|
|
135
|
-
export
|
|
241
|
+
export default realtimePlugin;
|
|
242
|
+
export const plugin = realtimePlugin;
|
package/dist/reflex.d.ts
CHANGED
package/dist/reflex.js
CHANGED
|
@@ -16,7 +16,11 @@ const RUNTIME = `(() => {
|
|
|
16
16
|
const o = { key: k, code: c[0], keyCode: c[1], which: c[1], bubbles: true, cancelable: true };
|
|
17
17
|
const tgt = document.activeElement || document.body;
|
|
18
18
|
tgt.dispatchEvent(new KeyboardEvent('keydown', o));
|
|
19
|
-
|
|
19
|
+
if (tgt !== document) document.dispatchEvent(new KeyboardEvent('keydown', o));
|
|
20
|
+
setTimeout(() => {
|
|
21
|
+
tgt.dispatchEvent(new KeyboardEvent('keyup', o));
|
|
22
|
+
if (tgt !== document) document.dispatchEvent(new KeyboardEvent('keyup', o));
|
|
23
|
+
}, hold);
|
|
20
24
|
};
|
|
21
25
|
const click = (el, x, y) => {
|
|
22
26
|
if (x == null) { const b = el.getBoundingClientRect(); x = b.left + b.width / 2; y = b.top + b.height / 2; }
|
|
@@ -48,8 +52,12 @@ const RUNTIME = `(() => {
|
|
|
48
52
|
const dist = (a, b) => (a && b) ? Math.hypot(a.x + a.w / 2 - b.x - b.w / 2, a.y + a.h / 2 - b.y - b.h / 2) : Infinity;
|
|
49
53
|
const gap = (a, b) => (a && b) ? b.x - (a.x + a.w) : Infinity;
|
|
50
54
|
const E = new Proxy({}, { get: (_, k) => {
|
|
51
|
-
const cv = window.__wt_cv
|
|
52
|
-
|
|
55
|
+
const cv = window.__wt_cv;
|
|
56
|
+
if (!cv) return [];
|
|
57
|
+
if (k === 'all') return cv.track('*', 2);
|
|
58
|
+
if (k === 'moving') return cv.track('*', 2).filter((e) => Math.abs(e.vx) > 0.1 || Math.abs(e.vy) > 0.1);
|
|
59
|
+
const sig = rf.names && rf.names[k];
|
|
60
|
+
return sig ? cv.track(sig, 2) : [];
|
|
53
61
|
} });
|
|
54
62
|
|
|
55
63
|
const tick = () => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@weavetab/plugin-realtime",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop for Weavetab.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
"dist",
|
|
16
16
|
"weavetab.json",
|
|
17
17
|
"README.md",
|
|
18
|
+
"AGENTS.md",
|
|
18
19
|
"LICENSE"
|
|
19
20
|
],
|
|
20
21
|
"scripts": {
|
|
@@ -25,7 +26,7 @@
|
|
|
25
26
|
"prepublishOnly": "npm run build"
|
|
26
27
|
},
|
|
27
28
|
"dependencies": {
|
|
28
|
-
"@weavetab/sdk": "^2.6.
|
|
29
|
+
"@weavetab/sdk": "^2.6.2"
|
|
29
30
|
},
|
|
30
31
|
"devDependencies": {
|
|
31
32
|
"@types/node": "^22.10.0",
|
package/weavetab.json
CHANGED
|
@@ -2,157 +2,269 @@
|
|
|
2
2
|
"$schema": "https://weavetab.dev/schema/plugin.json",
|
|
3
3
|
"name": "@weavetab/plugin-realtime",
|
|
4
4
|
"displayName": "Realtime",
|
|
5
|
-
"version": "0.
|
|
6
|
-
"description": "Causal tags for browser_map, in-page reflex rules, canvas entity sampling and a bounded click loop
|
|
5
|
+
"version": "0.3.0",
|
|
6
|
+
"description": "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop.",
|
|
7
7
|
"entry": "dist/index.js",
|
|
8
|
+
"agentsMd": "AGENTS.md",
|
|
9
|
+
"author": {
|
|
10
|
+
"name": "fy2ne",
|
|
11
|
+
"url": "https://fy2ne.me"
|
|
12
|
+
},
|
|
13
|
+
"license": "Apache-2.0",
|
|
14
|
+
"homepage": "https://github.com/Weavetab/Realtime#readme",
|
|
15
|
+
"repository": "https://github.com/Weavetab/Realtime",
|
|
8
16
|
"engines": {
|
|
9
|
-
"weavetab": "
|
|
17
|
+
"weavetab": "^2.6.2",
|
|
10
18
|
"node": ">=18.0.0"
|
|
11
19
|
},
|
|
12
|
-
"environments": [
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
"
|
|
20
|
+
"environments": [
|
|
21
|
+
"mcp",
|
|
22
|
+
"cli",
|
|
23
|
+
"browser"
|
|
24
|
+
],
|
|
25
|
+
"keywords": [
|
|
26
|
+
"weavetab",
|
|
27
|
+
"weavetab-plugin",
|
|
28
|
+
"cdp",
|
|
29
|
+
"canvas",
|
|
30
|
+
"browser-automation"
|
|
31
|
+
],
|
|
32
|
+
"defaultConfig": {
|
|
33
|
+
"causalWindowMs": 4000
|
|
34
|
+
},
|
|
18
35
|
"permissions": {
|
|
19
36
|
"cdp": {
|
|
20
37
|
"evaluate": true,
|
|
21
38
|
"input": true,
|
|
22
|
-
"reason": "
|
|
39
|
+
"reason": "Installs the in-page MutationObserver, hooks Canvas2D/rAF, and dispatches click/key events."
|
|
23
40
|
},
|
|
24
41
|
"tools": {
|
|
25
42
|
"add": true,
|
|
26
|
-
"reason": "Registers realtime_canvas, realtime_reflex, realtime_autopilot and realtime_status."
|
|
43
|
+
"reason": "Registers realtime_canvas, realtime_reflex, realtime_autopilot, and realtime_status."
|
|
44
|
+
},
|
|
45
|
+
"commands": {
|
|
46
|
+
"add": true,
|
|
47
|
+
"reason": "Registers status and reflex-clear CLI commands."
|
|
27
48
|
},
|
|
28
49
|
"prompts": {
|
|
29
50
|
"add": true,
|
|
30
|
-
"reason": "Registers /realtime_watch
|
|
51
|
+
"reason": "Registers the /realtime_watch prompt utility."
|
|
31
52
|
},
|
|
32
53
|
"resources": {
|
|
33
54
|
"add": true,
|
|
34
|
-
"reason": "Registers @realtime_feed
|
|
55
|
+
"reason": "Registers the @realtime_feed resource utility."
|
|
35
56
|
},
|
|
36
57
|
"hooks": {
|
|
37
58
|
"onBeforeAction": true,
|
|
38
59
|
"onAfterAction": true,
|
|
39
|
-
"reason": "
|
|
60
|
+
"reason": "Arms the in-page mutation probe before actions and tags newly added nodes in browser_map output."
|
|
40
61
|
},
|
|
41
62
|
"network": {
|
|
42
|
-
"
|
|
43
|
-
|
|
63
|
+
"domains": [
|
|
64
|
+
"127.0.0.1",
|
|
65
|
+
"localhost"
|
|
66
|
+
],
|
|
67
|
+
"reason": "Allows realtime_autopilot to query an optional local OpenAI-compatible model endpoint."
|
|
44
68
|
}
|
|
45
69
|
},
|
|
70
|
+
"commands": [
|
|
71
|
+
{
|
|
72
|
+
"name": "status",
|
|
73
|
+
"description": "Read the in-page realtime probe state, canvas frame counter, and active 60Hz reflex rules.",
|
|
74
|
+
"usage": "weavetab plugin use realtime status [--wait_ms <ms>]",
|
|
75
|
+
"requiresSession": true,
|
|
76
|
+
"options": [
|
|
77
|
+
{
|
|
78
|
+
"name": "wait_ms",
|
|
79
|
+
"type": "number",
|
|
80
|
+
"description": "Block up to wait_ms until active reflexes finish",
|
|
81
|
+
"required": false
|
|
82
|
+
}
|
|
83
|
+
]
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"name": "reflex-clear",
|
|
87
|
+
"description": "Clear all active 60Hz reflex rules on the active browser tab via CLI.",
|
|
88
|
+
"usage": "weavetab plugin use realtime reflex-clear",
|
|
89
|
+
"requiresSession": true,
|
|
90
|
+
"options": []
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"name": "autopilot",
|
|
94
|
+
"description": "Run bounded in-page autopilot toward a goal directly via CLI.",
|
|
95
|
+
"usage": "weavetab plugin use realtime autopilot --goal <text> [--max_steps <n>]",
|
|
96
|
+
"requiresSession": true,
|
|
97
|
+
"options": [
|
|
98
|
+
{
|
|
99
|
+
"name": "goal",
|
|
100
|
+
"type": "string",
|
|
101
|
+
"description": "Natural language click goal",
|
|
102
|
+
"required": true
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"name": "max_steps",
|
|
106
|
+
"type": "number",
|
|
107
|
+
"description": "Maximum click steps (default 5)",
|
|
108
|
+
"required": false
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"name": "stop_when_text",
|
|
112
|
+
"type": "string",
|
|
113
|
+
"description": "Stop when this text appears",
|
|
114
|
+
"required": false
|
|
115
|
+
}
|
|
116
|
+
]
|
|
117
|
+
}
|
|
118
|
+
],
|
|
119
|
+
"prompts": [
|
|
120
|
+
{
|
|
121
|
+
"name": "realtime_watch",
|
|
122
|
+
"description": "Snapshot live DOM mutations, active reflexes, and Canvas2D entities, then guide the agent to wire a 60Hz reflex rule.",
|
|
123
|
+
"arguments": [
|
|
124
|
+
{
|
|
125
|
+
"name": "goal",
|
|
126
|
+
"description": "What the reflex should watch for and do",
|
|
127
|
+
"required": true
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"name": "mode",
|
|
131
|
+
"description": "Trigger mode (auto, dom, canvas)",
|
|
132
|
+
"required": false
|
|
133
|
+
}
|
|
134
|
+
]
|
|
135
|
+
}
|
|
136
|
+
],
|
|
137
|
+
"resources": [
|
|
138
|
+
{
|
|
139
|
+
"name": "realtime_canvas_feed",
|
|
140
|
+
"uri": "realtime://canvas",
|
|
141
|
+
"description": "Live snapshot of Canvas2D draw counts, rAF frame rate, and active moving entity bounding boxes.",
|
|
142
|
+
"mimeType": "application/json"
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
"name": "realtime_feed",
|
|
146
|
+
"uri": "realtime://feed",
|
|
147
|
+
"description": "Live snapshot of newly mutated DOM elements (with click distance), active 60Hz reflex rules, and tracked Canvas2D entities.",
|
|
148
|
+
"mimeType": "application/json"
|
|
149
|
+
}
|
|
150
|
+
],
|
|
46
151
|
"tools": [
|
|
47
152
|
{
|
|
48
153
|
"name": "realtime_canvas",
|
|
49
|
-
"description": "
|
|
154
|
+
"description": "Records Canvas2D draw calls across requestAnimationFrame ticks and returns moving/static entities with bounding boxes and px/frame velocity.",
|
|
50
155
|
"inputSchema": {
|
|
51
156
|
"type": "object",
|
|
52
157
|
"properties": {
|
|
53
|
-
"action": {
|
|
54
|
-
|
|
158
|
+
"action": {
|
|
159
|
+
"type": "string",
|
|
160
|
+
"enum": [
|
|
161
|
+
"sample",
|
|
162
|
+
"attach",
|
|
163
|
+
"detach"
|
|
164
|
+
],
|
|
165
|
+
"description": "sample (default), attach, or detach"
|
|
166
|
+
},
|
|
167
|
+
"frames": {
|
|
168
|
+
"type": "number",
|
|
169
|
+
"description": "Frames to sample (2-15, default 6)"
|
|
170
|
+
}
|
|
55
171
|
}
|
|
56
172
|
}
|
|
57
173
|
},
|
|
58
174
|
{
|
|
59
175
|
"name": "realtime_reflex",
|
|
60
|
-
"description": "
|
|
176
|
+
"description": "Registers, removes, lists, or clears in-page 60Hz requestAnimationFrame rules that click or press keys on DOM or Canvas2D triggers without an LLM round-trip.",
|
|
61
177
|
"inputSchema": {
|
|
62
178
|
"type": "object",
|
|
63
179
|
"properties": {
|
|
64
|
-
"action": {
|
|
65
|
-
|
|
180
|
+
"action": {
|
|
181
|
+
"type": "string",
|
|
182
|
+
"enum": [
|
|
183
|
+
"add",
|
|
184
|
+
"remove",
|
|
185
|
+
"list",
|
|
186
|
+
"clear"
|
|
187
|
+
],
|
|
188
|
+
"description": "Operation to perform"
|
|
189
|
+
},
|
|
190
|
+
"id": {
|
|
191
|
+
"type": "string",
|
|
192
|
+
"description": "Rule identifier"
|
|
193
|
+
},
|
|
66
194
|
"when": {
|
|
67
195
|
"type": "object",
|
|
68
|
-
"
|
|
69
|
-
"selector": { "type": "string" },
|
|
70
|
-
"text": { "type": "string" },
|
|
71
|
-
"only_new": { "type": "boolean" },
|
|
72
|
-
"enabled": { "type": "boolean" },
|
|
73
|
-
"canvas": {
|
|
74
|
-
"type": "object",
|
|
75
|
-
"properties": {
|
|
76
|
-
"names": { "type": "object", "additionalProperties": { "type": "string" } },
|
|
77
|
-
"js": { "type": "string" }
|
|
78
|
-
},
|
|
79
|
-
"required": ["names", "js"]
|
|
80
|
-
}
|
|
81
|
-
}
|
|
196
|
+
"description": "DOM selector or Canvas2D expression trigger"
|
|
82
197
|
},
|
|
83
198
|
"do": {
|
|
84
199
|
"type": "object",
|
|
85
|
-
"
|
|
86
|
-
"type": { "type": "string", "enum": ["click", "click_xy", "key", "js"] },
|
|
87
|
-
"x": { "type": "number" },
|
|
88
|
-
"y": { "type": "number" },
|
|
89
|
-
"key": { "type": "string" },
|
|
90
|
-
"hold_ms": { "type": "number" },
|
|
91
|
-
"code": { "type": "string" }
|
|
92
|
-
},
|
|
93
|
-
"required": ["type"]
|
|
200
|
+
"description": "Response action (click, click_xy, key, js)"
|
|
94
201
|
},
|
|
95
|
-
"cooldown_ms": {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
202
|
+
"cooldown_ms": {
|
|
203
|
+
"type": "number",
|
|
204
|
+
"description": "Minimum ms between firings (default 100)"
|
|
205
|
+
},
|
|
206
|
+
"max_hits": {
|
|
207
|
+
"type": "number",
|
|
208
|
+
"description": "Auto-stop after N hits (default 50)"
|
|
209
|
+
},
|
|
210
|
+
"ttl_ms": {
|
|
211
|
+
"type": "number",
|
|
212
|
+
"description": "Rule lifetime in ms (default 30000)"
|
|
213
|
+
},
|
|
214
|
+
"stop_selector": {
|
|
215
|
+
"type": "string",
|
|
216
|
+
"description": "Selector that halts the rule when present"
|
|
217
|
+
}
|
|
218
|
+
},
|
|
219
|
+
"required": [
|
|
220
|
+
"action"
|
|
221
|
+
]
|
|
100
222
|
}
|
|
101
223
|
},
|
|
102
224
|
{
|
|
103
225
|
"name": "realtime_autopilot",
|
|
104
|
-
"description": "
|
|
226
|
+
"description": "Runs a bounded in-page click loop toward a short goal ('accept cookies', 'continue') with causal age/dialog scoring and optional local model pick.",
|
|
105
227
|
"inputSchema": {
|
|
106
228
|
"type": "object",
|
|
107
229
|
"properties": {
|
|
108
|
-
"goal": {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
"
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
"
|
|
230
|
+
"goal": {
|
|
231
|
+
"type": "string",
|
|
232
|
+
"description": "Short natural-language click goal"
|
|
233
|
+
},
|
|
234
|
+
"max_steps": {
|
|
235
|
+
"type": "number",
|
|
236
|
+
"description": "Maximum clicks (default 8, max 25)"
|
|
237
|
+
},
|
|
238
|
+
"stop_when_selector": {
|
|
239
|
+
"type": "string",
|
|
240
|
+
"description": "Stop when selector becomes visible"
|
|
241
|
+
},
|
|
242
|
+
"stop_when_text": {
|
|
243
|
+
"type": "string",
|
|
244
|
+
"description": "Stop when body text contains string"
|
|
245
|
+
},
|
|
246
|
+
"stop_when_url": {
|
|
247
|
+
"type": "string",
|
|
248
|
+
"description": "Stop when URL matches regex"
|
|
249
|
+
}
|
|
117
250
|
},
|
|
118
|
-
"required": [
|
|
251
|
+
"required": [
|
|
252
|
+
"goal"
|
|
253
|
+
]
|
|
119
254
|
}
|
|
120
255
|
},
|
|
121
256
|
{
|
|
122
257
|
"name": "realtime_status",
|
|
123
|
-
"description": "
|
|
258
|
+
"description": "Returns the live state of the in-page probe, Canvas2D spy, and active 60Hz reflex rules. Optionally waits until reflexes complete.",
|
|
124
259
|
"inputSchema": {
|
|
125
260
|
"type": "object",
|
|
126
|
-
"properties": {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
{
|
|
132
|
-
"name": "realtime_watch",
|
|
133
|
-
"title": "Watch & React (60Hz Reflex)",
|
|
134
|
-
"description": "Set up an in-page 60Hz reflex rule or canvas game loop and wait until it finishes.",
|
|
135
|
-
"arguments": [
|
|
136
|
-
{
|
|
137
|
-
"name": "goal",
|
|
138
|
-
"description": "What to watch for and trigger (e.g. 'click Claim as soon as it appears' or 'jump over green obstacles in the canvas game')",
|
|
139
|
-
"required": true
|
|
140
|
-
},
|
|
141
|
-
{
|
|
142
|
-
"name": "mode",
|
|
143
|
-
"description": "'dom', 'canvas', or 'auto' (default 'auto')",
|
|
144
|
-
"required": false
|
|
261
|
+
"properties": {
|
|
262
|
+
"wait_ms": {
|
|
263
|
+
"type": "number",
|
|
264
|
+
"description": "Wait up to wait_ms (max 30000) for active reflexes to finish"
|
|
265
|
+
}
|
|
145
266
|
}
|
|
146
|
-
|
|
147
|
-
}
|
|
148
|
-
],
|
|
149
|
-
"resources": [
|
|
150
|
-
{
|
|
151
|
-
"name": "realtime_feed",
|
|
152
|
-
"title": "Realtime Page Feed",
|
|
153
|
-
"uri": "realtime://feed",
|
|
154
|
-
"mimeType": "application/json",
|
|
155
|
-
"description": "Live snapshot of active 60Hz reflex rules, recently mutated DOM elements with click distance, and tracked Canvas2D entities."
|
|
267
|
+
}
|
|
156
268
|
}
|
|
157
269
|
]
|
|
158
270
|
}
|