@weavetab/plugin-realtime 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +19 -0
- package/LICENSE +202 -0
- package/README.md +113 -2
- package/dist/autopilot.d.ts +18 -0
- package/dist/autopilot.js +171 -0
- package/dist/canvas.d.ts +5 -0
- package/dist/canvas.js +179 -0
- package/dist/causal.d.ts +1 -0
- package/dist/causal.js +72 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +242 -0
- package/dist/page.d.ts +3 -0
- package/dist/page.js +33 -0
- package/dist/reflex.d.ts +29 -0
- package/dist/reflex.js +171 -0
- package/dist/surface.d.ts +19 -0
- package/dist/surface.js +59 -0
- package/package.json +52 -4
- package/weavetab.json +270 -0
package/dist/canvas.js
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { run } from "./page.js";
|
|
2
|
+
// recorder hooks CanvasRenderingContext2D.prototype and window.requestAnimationFrame.
|
|
3
|
+
// draws are only kept while a rAF callback is running, that is what defines a frame.
|
|
4
|
+
const SPY = `(() => {
|
|
5
|
+
if (window.__wt_cv) return 1;
|
|
6
|
+
const cv = window.__wt_cv = { frames: [], n: 0, cur: null, orig: {}, canvases: [], bg: 0, raf: window.requestAnimationFrame };
|
|
7
|
+
const P = CanvasRenderingContext2D.prototype;
|
|
8
|
+
|
|
9
|
+
const rec = (ctx, kind, key, x, y, w, h) => {
|
|
10
|
+
const m = ctx.getTransform();
|
|
11
|
+
const x1 = m.a * x + m.c * y + m.e, y1 = m.b * x + m.d * y + m.f;
|
|
12
|
+
const x2 = m.a * (x + w) + m.c * (y + h) + m.e, y2 = m.b * (x + w) + m.d * (y + h) + m.f;
|
|
13
|
+
let ci = cv.canvases.indexOf(ctx.canvas);
|
|
14
|
+
if (ci < 0) ci = cv.canvases.push(ctx.canvas) - 1;
|
|
15
|
+
const ww = Math.abs(x2 - x1), hh = Math.abs(y2 - y1);
|
|
16
|
+
cv.cur.push({ sig: kind + '|' + Math.round(ww) + 'x' + Math.round(hh) + '|' + key,
|
|
17
|
+
x: Math.min(x1, x2), y: Math.min(y1, y2), w: ww, h: hh, c: ci });
|
|
18
|
+
};
|
|
19
|
+
const styleKey = (s) => typeof s === 'string' ? s : 'grad';
|
|
20
|
+
const imgKey = (im) => {
|
|
21
|
+
const s = im.currentSrc || im.src;
|
|
22
|
+
return s ? s.split('?')[0].split('/').pop() : (im.id || im.tagName || 'bmp');
|
|
23
|
+
};
|
|
24
|
+
const wrap = (name, fn) => {
|
|
25
|
+
const o = P[name];
|
|
26
|
+
cv.orig[name] = o;
|
|
27
|
+
P[name] = function () {
|
|
28
|
+
const r = o.apply(this, arguments);
|
|
29
|
+
// the spy must never break the game's own draw call
|
|
30
|
+
try { if (cv.cur && this.canvas.isConnected && cv.cur.length < 2000) fn(this, arguments); } catch (e) {}
|
|
31
|
+
return r;
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
wrap('fillRect', (c, a) => rec(c, 'rect', styleKey(c.fillStyle), a[0], a[1], a[2], a[3]));
|
|
36
|
+
wrap('strokeRect', (c, a) => rec(c, 'frame', styleKey(c.strokeStyle), a[0], a[1], a[2], a[3]));
|
|
37
|
+
wrap('drawImage', (c, a) => {
|
|
38
|
+
const im = a[0], n = a.length;
|
|
39
|
+
if (n >= 9) rec(c, 'img', imgKey(im) + '@' + a[1] + ',' + a[2], a[5], a[6], a[7], a[8]);
|
|
40
|
+
else if (n >= 5) rec(c, 'img', imgKey(im), a[1], a[2], a[3], a[4]);
|
|
41
|
+
else rec(c, 'img', imgKey(im), a[1], a[2], im.naturalWidth || im.videoWidth || im.width || 0, im.naturalHeight || im.videoHeight || im.height || 0);
|
|
42
|
+
});
|
|
43
|
+
const text = (c, a) => {
|
|
44
|
+
const m = c.measureText(a[0]);
|
|
45
|
+
rec(c, 'text', String(a[0]).slice(0, 24), a[1], a[2] - m.actualBoundingBoxAscent, m.width, m.actualBoundingBoxAscent + m.actualBoundingBoxDescent);
|
|
46
|
+
};
|
|
47
|
+
wrap('fillText', text);
|
|
48
|
+
wrap('strokeText', text);
|
|
49
|
+
wrap('beginPath', (c) => { c.__wtp = []; });
|
|
50
|
+
wrap('arc', (c, a) => {
|
|
51
|
+
if (Math.abs(a[4] - a[3]) >= 6.2) (c.__wtp || (c.__wtp = [])).push([a[0], a[1], a[2]]);
|
|
52
|
+
});
|
|
53
|
+
const commit = (style) => (c) => {
|
|
54
|
+
for (const p of (c.__wtp || [])) rec(c, 'arc', styleKey(c[style]), p[0] - p[2], p[1] - p[2], p[2] * 2, p[2] * 2);
|
|
55
|
+
c.__wtp = [];
|
|
56
|
+
};
|
|
57
|
+
wrap('fill', commit('fillStyle'));
|
|
58
|
+
wrap('stroke', commit('strokeStyle'));
|
|
59
|
+
|
|
60
|
+
window.requestAnimationFrame = function (cb) {
|
|
61
|
+
return cv.raf.call(window, function (ts) {
|
|
62
|
+
cv.cur = [];
|
|
63
|
+
try { return cb(ts); } finally {
|
|
64
|
+
cv.frames.push({ t: ts, d: cv.cur });
|
|
65
|
+
if (cv.frames.length > 16) cv.frames.shift();
|
|
66
|
+
cv.cur = null;
|
|
67
|
+
cv.n++;
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const matchSig = (pattern, s) => {
|
|
73
|
+
if (!pattern) return false;
|
|
74
|
+
if (pattern === '*' || pattern === true) return true;
|
|
75
|
+
if (Array.isArray(pattern)) return pattern.some((p) => matchSig(p, s));
|
|
76
|
+
if (typeof pattern !== 'string') return false;
|
|
77
|
+
if (pattern.indexOf('*') < 0) return s === pattern;
|
|
78
|
+
const parts = pattern.split('*');
|
|
79
|
+
if (!s.startsWith(parts[0])) return false;
|
|
80
|
+
if (!s.endsWith(parts[parts.length - 1])) return false;
|
|
81
|
+
let idx = 0;
|
|
82
|
+
for (let i = 0; i < parts.length; i++) {
|
|
83
|
+
const part = parts[i];
|
|
84
|
+
if (!part) continue;
|
|
85
|
+
const found = s.indexOf(part, idx);
|
|
86
|
+
if (found < 0) return false;
|
|
87
|
+
idx = found + part.length;
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
cv.track = (sig, depth) => {
|
|
93
|
+
const fr = cv.frames;
|
|
94
|
+
if (!fr.length) return [];
|
|
95
|
+
const last = fr[fr.length - 1].d.filter((d) => matchSig(sig, d.sig));
|
|
96
|
+
if (last.length > 200) return last.map((d) => Object.assign({ vx: 0, vy: 0 }, d));
|
|
97
|
+
const prevs = fr.slice(Math.max(0, fr.length - 1 - (depth || 2)), -1).reverse().map((f) => f.d.filter((d) => matchSig(sig, d.sig)));
|
|
98
|
+
return last.map((d) => {
|
|
99
|
+
let cur = d, steps = 0;
|
|
100
|
+
for (const p of prevs) {
|
|
101
|
+
let best = null, bd = 1e9;
|
|
102
|
+
for (const q of p) {
|
|
103
|
+
const dd = (q.x - cur.x) * (q.x - cur.x) + (q.y - cur.y) * (q.y - cur.y);
|
|
104
|
+
if (dd < bd) { bd = dd; best = q; }
|
|
105
|
+
}
|
|
106
|
+
if (!best || bd > 40000) break;
|
|
107
|
+
cur = best; steps++;
|
|
108
|
+
}
|
|
109
|
+
return Object.assign({}, d, { vx: steps ? (d.x - cur.x) / steps : 0, vy: steps ? (d.y - cur.y) / steps : 0 });
|
|
110
|
+
});
|
|
111
|
+
};
|
|
112
|
+
return 0;
|
|
113
|
+
})()`;
|
|
114
|
+
const WAIT = (frames) => `new Promise((res) => {
|
|
115
|
+
const cv = window.__wt_cv, start = cv.n, t0 = performance.now();
|
|
116
|
+
const tick = () => (cv.n - start >= ${frames} || performance.now() - t0 > 3000) ? res(cv.n - start) : setTimeout(tick, 8);
|
|
117
|
+
tick();
|
|
118
|
+
})`;
|
|
119
|
+
const REPORT = `(() => {
|
|
120
|
+
const cv = window.__wt_cv;
|
|
121
|
+
const fr = cv.frames, last = fr[fr.length - 1];
|
|
122
|
+
const r1 = (v) => Math.round(v * 10) / 10;
|
|
123
|
+
const dt = fr.length > 1 ? fr[fr.length - 1].t - fr[0].t : 0;
|
|
124
|
+
const cans = [];
|
|
125
|
+
const info = (i) => {
|
|
126
|
+
let o = cans.find((c) => c.i === i);
|
|
127
|
+
if (!o) {
|
|
128
|
+
const el = cv.canvases[i], r = el.getBoundingClientRect();
|
|
129
|
+
o = { i, id: el.id || null, w: el.width, h: el.height, left: r.left, top: r.top, sx: r.width / el.width, sy: r.height / el.height };
|
|
130
|
+
cans.push(o);
|
|
131
|
+
}
|
|
132
|
+
return o;
|
|
133
|
+
};
|
|
134
|
+
const sigs = {};
|
|
135
|
+
for (const d of last.d) (sigs[d.sig] || (sigs[d.sig] = [])).push(d);
|
|
136
|
+
const ents = [], groups = [];
|
|
137
|
+
let bg = 0;
|
|
138
|
+
for (const sig of Object.keys(sigs).slice(0, 200)) {
|
|
139
|
+
const inst = sigs[sig];
|
|
140
|
+
const kind = sig.split('|')[0];
|
|
141
|
+
const tr = cv.track(sig, 5);
|
|
142
|
+
if (inst.length > 6) {
|
|
143
|
+
groups.push({ sig, count: inst.length, moving: tr.filter((t) => t.vx || t.vy).length });
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
for (const t of tr) {
|
|
147
|
+
const cn = info(t.c);
|
|
148
|
+
if ((kind === 'rect') && t.w >= cn.w * 0.9 && t.h >= cn.h * 0.9) { bg++; continue; }
|
|
149
|
+
ents.push({ sig, canvas: t.c, x: r1(t.x), y: r1(t.y), w: r1(t.w), h: r1(t.h), vx: r1(t.vx), vy: r1(t.vy) });
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
ents.sort((a, b) => (Math.abs(b.vx) + Math.abs(b.vy)) - (Math.abs(a.vx) + Math.abs(a.vy)));
|
|
153
|
+
return { frames: fr.length, fps: dt ? Math.round((fr.length - 1) * 1000 / dt) : null, canvases: cans,
|
|
154
|
+
entities: ents.slice(0, 40), groups: groups.slice(0, 20), background_fills: bg, truncated: ents.length > 40 };
|
|
155
|
+
})()`;
|
|
156
|
+
const OFF = `(() => {
|
|
157
|
+
const cv = window.__wt_cv;
|
|
158
|
+
if (!cv) return 0;
|
|
159
|
+
const P = CanvasRenderingContext2D.prototype;
|
|
160
|
+
for (const k of Object.keys(cv.orig)) P[k] = cv.orig[k];
|
|
161
|
+
window.requestAnimationFrame = cv.raf;
|
|
162
|
+
delete window.__wt_cv;
|
|
163
|
+
return 1;
|
|
164
|
+
})()`;
|
|
165
|
+
export async function canvasTool(sess, a) {
|
|
166
|
+
if (a.action === "detach")
|
|
167
|
+
return { detached: Boolean(await run(sess, OFF)) };
|
|
168
|
+
const fresh = (await run(sess, SPY)) === 0;
|
|
169
|
+
if (a.action === "attach")
|
|
170
|
+
return { attached: true, already: !fresh };
|
|
171
|
+
const want = Math.min(Math.max(a.frames ?? 6, 2), 15);
|
|
172
|
+
const got = await run(sess, WAIT(want), true);
|
|
173
|
+
if (!got)
|
|
174
|
+
throw new Error("no frames seen: tab hidden, or the game does not use requestAnimationFrame");
|
|
175
|
+
return run(sess, REPORT);
|
|
176
|
+
}
|
|
177
|
+
export function canvasSnap(sess) {
|
|
178
|
+
return run(sess, `window.__wt_cv && window.__wt_cv.frames.length ? ${REPORT} : null`);
|
|
179
|
+
}
|
package/dist/causal.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function tagMap(sess: any, res: any, winMs: number): Promise<any>;
|
package/dist/causal.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { run } from "./page.js";
|
|
2
|
+
const LINE = /^(\s*)\[w:(\d+)\]/;
|
|
3
|
+
function probeIds(ids, winMs) {
|
|
4
|
+
return `((ids, win) => {
|
|
5
|
+
const rt = window.__wt_rt;
|
|
6
|
+
if (!rt) return [];
|
|
7
|
+
const now = performance.now();
|
|
8
|
+
const clk = rt.click && now - rt.click.t < 15000 ? rt.click : null;
|
|
9
|
+
const out = [];
|
|
10
|
+
for (const id of ids) {
|
|
11
|
+
const el = document.querySelector('[data-wt-id="' + id + '"]');
|
|
12
|
+
if (!el) continue;
|
|
13
|
+
let b;
|
|
14
|
+
for (let n = el; n && b === undefined; n = n.parentElement) b = rt.born.get(n);
|
|
15
|
+
if (b === undefined || now - b > win) continue;
|
|
16
|
+
let dist = null;
|
|
17
|
+
if (clk) {
|
|
18
|
+
const r = el.getBoundingClientRect();
|
|
19
|
+
dist = Math.round(Math.hypot(r.left + r.width / 2 - clk.x, r.top + r.height / 2 - clk.y));
|
|
20
|
+
}
|
|
21
|
+
out.push([id, Math.round(now - b), dist]);
|
|
22
|
+
}
|
|
23
|
+
return out;
|
|
24
|
+
})(${JSON.stringify(ids)}, ${winMs})`;
|
|
25
|
+
}
|
|
26
|
+
async function tagText(sess, text, winMs) {
|
|
27
|
+
const lines = text.split("\n");
|
|
28
|
+
const ids = new Set();
|
|
29
|
+
for (const ln of lines) {
|
|
30
|
+
const m = LINE.exec(ln);
|
|
31
|
+
if (m)
|
|
32
|
+
ids.add(m[2]);
|
|
33
|
+
}
|
|
34
|
+
if (!ids.size)
|
|
35
|
+
return text;
|
|
36
|
+
const hits = await run(sess, probeIds([...ids], winMs));
|
|
37
|
+
if (!hits.length)
|
|
38
|
+
return text;
|
|
39
|
+
const byId = new Map(hits.map((h) => [h[0], h]));
|
|
40
|
+
return lines
|
|
41
|
+
.map((ln) => {
|
|
42
|
+
const m = LINE.exec(ln);
|
|
43
|
+
const h = m && byId.get(m[2]);
|
|
44
|
+
if (!h)
|
|
45
|
+
return ln;
|
|
46
|
+
return `${ln} {new ${h[1]}ms${h[2] == null ? "" : `, ${h[2]}px from click`}}`;
|
|
47
|
+
})
|
|
48
|
+
.join("\n");
|
|
49
|
+
}
|
|
50
|
+
// returns undefined when nothing changed so the pipeline keeps the original result
|
|
51
|
+
export async function tagMap(sess, res, winMs) {
|
|
52
|
+
if (typeof res === "string") {
|
|
53
|
+
const t = await tagText(sess, res, winMs);
|
|
54
|
+
return t === res ? undefined : t;
|
|
55
|
+
}
|
|
56
|
+
if (!Array.isArray(res?.content))
|
|
57
|
+
return;
|
|
58
|
+
let touched = false;
|
|
59
|
+
const content = [];
|
|
60
|
+
for (const blk of res.content) {
|
|
61
|
+
if (blk?.type === "text" && typeof blk.text === "string") {
|
|
62
|
+
const t = await tagText(sess, blk.text, winMs);
|
|
63
|
+
if (t !== blk.text)
|
|
64
|
+
touched = true;
|
|
65
|
+
content.push({ ...blk, text: t });
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
content.push(blk);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return touched ? { ...res, content } : undefined;
|
|
72
|
+
}
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { definePlugin, defineTool, defineCommand, definePrompt, defineResource, z, } from "@weavetab/sdk";
|
|
2
|
+
import { sessOf, arm } from "./page.js";
|
|
3
|
+
import { tagMap } from "./causal.js";
|
|
4
|
+
import { canvasTool } from "./canvas.js";
|
|
5
|
+
import { reflexTool, statusTool } from "./reflex.js";
|
|
6
|
+
import { autopilotTool } from "./autopilot.js";
|
|
7
|
+
import { readFeed } from "./surface.js";
|
|
8
|
+
const statusCommand = defineCommand({
|
|
9
|
+
name: "status",
|
|
10
|
+
description: "Read the in-page realtime probe state, canvas frame counter, and active 60Hz reflex rules.",
|
|
11
|
+
usage: "weavetab plugin use realtime status [--wait_ms <ms>]",
|
|
12
|
+
requiresSession: true,
|
|
13
|
+
schema: {
|
|
14
|
+
wait_ms: z.number().optional().describe("Block up to wait_ms until active reflexes finish"),
|
|
15
|
+
},
|
|
16
|
+
handler: async (args, ctx) => {
|
|
17
|
+
const sess = sessOf(ctx);
|
|
18
|
+
return await statusTool(sess, Number(args.wait_ms || 0));
|
|
19
|
+
},
|
|
20
|
+
});
|
|
21
|
+
const reflexClearCommand = defineCommand({
|
|
22
|
+
name: "reflex-clear",
|
|
23
|
+
description: "Clear all active 60Hz reflex rules on the active browser tab via CLI.",
|
|
24
|
+
usage: "weavetab plugin use realtime reflex-clear",
|
|
25
|
+
requiresSession: true,
|
|
26
|
+
handler: async (_args, ctx) => {
|
|
27
|
+
const sess = sessOf(ctx);
|
|
28
|
+
return await reflexTool(sess, { action: "clear" });
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
const autopilotCommand = defineCommand({
|
|
32
|
+
name: "autopilot",
|
|
33
|
+
description: "Run bounded in-page autopilot toward a goal directly via CLI.",
|
|
34
|
+
usage: "weavetab plugin use realtime autopilot --goal <text> [--max_steps <n>]",
|
|
35
|
+
requiresSession: true,
|
|
36
|
+
schema: {
|
|
37
|
+
goal: z.string().describe("Natural language click goal"),
|
|
38
|
+
max_steps: z.number().optional().describe("Maximum click steps (default 5)"),
|
|
39
|
+
stop_when_text: z.string().optional().describe("Stop when this text appears"),
|
|
40
|
+
},
|
|
41
|
+
handler: async (args, ctx) => {
|
|
42
|
+
const sess = sessOf(ctx);
|
|
43
|
+
return await autopilotTool(sess, {
|
|
44
|
+
goal: args.goal,
|
|
45
|
+
max_steps: args.max_steps ?? 5,
|
|
46
|
+
stop_when_text: args.stop_when_text,
|
|
47
|
+
});
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
export const realtimePlugin = definePlugin({
|
|
51
|
+
name: "@weavetab/plugin-realtime",
|
|
52
|
+
version: "0.3.0",
|
|
53
|
+
description: "Causal age/proximity tags for browser_map, 60Hz in-page reflex rules, canvas entity sampling and a bounded click loop.",
|
|
54
|
+
agentsMd: "AGENTS.md",
|
|
55
|
+
defaultConfig: {
|
|
56
|
+
causalWindowMs: 4000,
|
|
57
|
+
},
|
|
58
|
+
permissions: {
|
|
59
|
+
cdp: {
|
|
60
|
+
evaluate: true,
|
|
61
|
+
input: true,
|
|
62
|
+
reason: "Installs the in-page MutationObserver, hooks Canvas2D/rAF, and dispatches click/key events.",
|
|
63
|
+
},
|
|
64
|
+
tools: {
|
|
65
|
+
add: true,
|
|
66
|
+
reason: "Registers realtime_canvas, realtime_reflex, realtime_autopilot, and realtime_status.",
|
|
67
|
+
},
|
|
68
|
+
commands: {
|
|
69
|
+
add: true,
|
|
70
|
+
reason: "Registers status and reflex-clear CLI commands.",
|
|
71
|
+
},
|
|
72
|
+
prompts: {
|
|
73
|
+
add: true,
|
|
74
|
+
reason: "Registers the /realtime_watch prompt utility.",
|
|
75
|
+
},
|
|
76
|
+
resources: {
|
|
77
|
+
add: true,
|
|
78
|
+
reason: "Registers the @realtime_feed resource utility.",
|
|
79
|
+
},
|
|
80
|
+
hooks: {
|
|
81
|
+
onBeforeAction: true,
|
|
82
|
+
onAfterAction: true,
|
|
83
|
+
reason: "Arms the in-page mutation probe before actions and tags newly added nodes in browser_map output.",
|
|
84
|
+
},
|
|
85
|
+
network: {
|
|
86
|
+
domains: ["127.0.0.1", "localhost"],
|
|
87
|
+
reason: "Allows realtime_autopilot to query an optional local OpenAI-compatible model endpoint.",
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
commands: [statusCommand, reflexClearCommand, autopilotCommand],
|
|
91
|
+
async onLoad(ctx) {
|
|
92
|
+
const winMs = typeof ctx.config?.causalWindowMs === "number" ? ctx.config.causalWindowMs : 4000;
|
|
93
|
+
ctx.cli?.registerCommand?.(statusCommand);
|
|
94
|
+
ctx.cli?.registerCommand?.(reflexClearCommand);
|
|
95
|
+
ctx.cli?.registerCommand?.(autopilotCommand);
|
|
96
|
+
ctx.mcp.registerTool(defineTool({
|
|
97
|
+
name: "realtime_canvas",
|
|
98
|
+
description: "Records Canvas2D draw calls across requestAnimationFrame ticks and returns moving/static entities with bounding boxes and px/frame velocity.",
|
|
99
|
+
requiresSession: true,
|
|
100
|
+
schema: {
|
|
101
|
+
action: z.enum(["sample", "attach", "detach"]).optional().describe("sample (default), attach, or detach"),
|
|
102
|
+
frames: z.number().optional().describe("Frames to sample (2-15, default 6)"),
|
|
103
|
+
},
|
|
104
|
+
handler: async (args, rawCtx) => {
|
|
105
|
+
const out = await canvasTool(sessOf(rawCtx), {
|
|
106
|
+
action: args.action || "sample",
|
|
107
|
+
frames: args.frames,
|
|
108
|
+
});
|
|
109
|
+
return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
|
|
110
|
+
},
|
|
111
|
+
}));
|
|
112
|
+
ctx.mcp.registerTool(defineTool({
|
|
113
|
+
name: "realtime_reflex",
|
|
114
|
+
description: "Registers, removes, lists, or clears in-page 60Hz requestAnimationFrame rules that click or press keys on DOM or Canvas2D triggers without an LLM round-trip.",
|
|
115
|
+
requiresSession: true,
|
|
116
|
+
schema: {
|
|
117
|
+
action: z.enum(["add", "remove", "list", "clear"]).describe("Operation to perform"),
|
|
118
|
+
id: z.string().optional().describe("Rule identifier"),
|
|
119
|
+
when: z
|
|
120
|
+
.object({
|
|
121
|
+
selector: z.string().optional(),
|
|
122
|
+
text: z.string().optional(),
|
|
123
|
+
only_new: z.boolean().optional(),
|
|
124
|
+
enabled: z.boolean().optional(),
|
|
125
|
+
canvas: z
|
|
126
|
+
.object({
|
|
127
|
+
names: z.record(z.string(), z.union([z.string(), z.array(z.string())])),
|
|
128
|
+
js: z.string(),
|
|
129
|
+
})
|
|
130
|
+
.optional(),
|
|
131
|
+
})
|
|
132
|
+
.optional(),
|
|
133
|
+
do: z
|
|
134
|
+
.object({
|
|
135
|
+
type: z.enum(["click", "click_xy", "key", "js"]),
|
|
136
|
+
x: z.number().optional(),
|
|
137
|
+
y: z.number().optional(),
|
|
138
|
+
key: z.string().optional(),
|
|
139
|
+
hold_ms: z.number().optional(),
|
|
140
|
+
code: z.string().optional(),
|
|
141
|
+
})
|
|
142
|
+
.optional(),
|
|
143
|
+
cooldown_ms: z.number().optional(),
|
|
144
|
+
max_hits: z.number().optional(),
|
|
145
|
+
ttl_ms: z.number().optional(),
|
|
146
|
+
stop_selector: z.string().optional(),
|
|
147
|
+
},
|
|
148
|
+
handler: async (args, rawCtx) => {
|
|
149
|
+
const out = await reflexTool(sessOf(rawCtx), args);
|
|
150
|
+
return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
|
|
151
|
+
},
|
|
152
|
+
}));
|
|
153
|
+
ctx.mcp.registerTool(defineTool({
|
|
154
|
+
name: "realtime_autopilot",
|
|
155
|
+
description: "Runs a bounded in-page click loop toward a short goal ('accept cookies', 'continue') with causal age/dialog scoring and optional local model pick.",
|
|
156
|
+
requiresSession: true,
|
|
157
|
+
schema: {
|
|
158
|
+
goal: z.string().describe("Short natural-language click goal"),
|
|
159
|
+
max_steps: z.number().optional(),
|
|
160
|
+
stop_when_selector: z.string().optional(),
|
|
161
|
+
stop_when_text: z.string().optional(),
|
|
162
|
+
stop_when_url: z.string().optional(),
|
|
163
|
+
avoid_labels: z.string().optional(),
|
|
164
|
+
model_url: z.string().optional(),
|
|
165
|
+
model: z.string().optional(),
|
|
166
|
+
model_timeout_ms: z.number().optional(),
|
|
167
|
+
},
|
|
168
|
+
handler: async (args, rawCtx) => {
|
|
169
|
+
const out = await autopilotTool(sessOf(rawCtx), args);
|
|
170
|
+
return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
|
|
171
|
+
},
|
|
172
|
+
}));
|
|
173
|
+
ctx.mcp.registerTool(defineTool({
|
|
174
|
+
name: "realtime_status",
|
|
175
|
+
description: "Returns the live state of the in-page probe, Canvas2D spy, and active 60Hz reflex rules. Optionally waits until reflexes complete.",
|
|
176
|
+
requiresSession: true,
|
|
177
|
+
schema: {
|
|
178
|
+
wait_ms: z.number().optional().describe("Wait up to wait_ms (max 30000) for active reflexes to finish"),
|
|
179
|
+
},
|
|
180
|
+
handler: async (args, rawCtx) => {
|
|
181
|
+
const out = await statusTool(sessOf(rawCtx), args.wait_ms ?? 0);
|
|
182
|
+
return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
|
|
183
|
+
},
|
|
184
|
+
}));
|
|
185
|
+
ctx.mcp.registerPrompt(definePrompt({
|
|
186
|
+
name: "realtime_watch",
|
|
187
|
+
description: "Snapshot live DOM mutations, active reflexes, and Canvas2D entities, then guide the agent to wire a 60Hz reflex rule.",
|
|
188
|
+
arguments: {
|
|
189
|
+
goal: z.string().describe("What the reflex should watch for and do"),
|
|
190
|
+
mode: z.enum(["auto", "dom", "canvas"]).optional().describe("Trigger mode (default: auto)"),
|
|
191
|
+
},
|
|
192
|
+
handler: async (args, rawCtx) => {
|
|
193
|
+
const feed = await readFeed(sessOf(rawCtx), winMs);
|
|
194
|
+
return [
|
|
195
|
+
`Goal: ${args.goal}`,
|
|
196
|
+
`Mode: ${args.mode || "auto"}`,
|
|
197
|
+
`Live feed: ${JSON.stringify(feed)}`,
|
|
198
|
+
`Use realtime_canvas, realtime_reflex, and realtime_status to arm and verify the rule.`,
|
|
199
|
+
].join("\n");
|
|
200
|
+
},
|
|
201
|
+
}));
|
|
202
|
+
ctx.mcp.registerResource(defineResource({
|
|
203
|
+
name: "realtime_canvas_feed",
|
|
204
|
+
uri: "realtime://canvas",
|
|
205
|
+
description: "Live snapshot of Canvas2D draw counts, rAF frame rate, and active moving entity bounding boxes.",
|
|
206
|
+
mimeType: "application/json",
|
|
207
|
+
read: async (_uri, _vars, rawCtx) => {
|
|
208
|
+
const sess = sessOf(rawCtx);
|
|
209
|
+
const canvasData = await canvasTool(sess, { action: "sample", frames: 3 });
|
|
210
|
+
return JSON.stringify(canvasData, null, 2);
|
|
211
|
+
},
|
|
212
|
+
}));
|
|
213
|
+
ctx.mcp.registerResource(defineResource({
|
|
214
|
+
name: "realtime_feed",
|
|
215
|
+
uri: "realtime://feed",
|
|
216
|
+
description: "Live snapshot of newly mutated DOM elements (with click distance), active 60Hz reflex rules, and tracked Canvas2D entities.",
|
|
217
|
+
mimeType: "application/json",
|
|
218
|
+
read: async (_uri, _vars, rawCtx) => {
|
|
219
|
+
const feed = await readFeed(sessOf(rawCtx), winMs);
|
|
220
|
+
return JSON.stringify(feed, null, 2);
|
|
221
|
+
},
|
|
222
|
+
}));
|
|
223
|
+
ctx.logger?.info("@weavetab/plugin-realtime loaded");
|
|
224
|
+
},
|
|
225
|
+
async onBeforeAction(_action, _args, rawCtx) {
|
|
226
|
+
const sess = sessOf(rawCtx);
|
|
227
|
+
if (sess?.Runtime) {
|
|
228
|
+
await arm(sess).catch(() => { });
|
|
229
|
+
}
|
|
230
|
+
},
|
|
231
|
+
async onAfterAction(action, result, rawCtx) {
|
|
232
|
+
if (action !== "browser_map")
|
|
233
|
+
return;
|
|
234
|
+
const sess = sessOf(rawCtx);
|
|
235
|
+
if (!sess?.Runtime)
|
|
236
|
+
return;
|
|
237
|
+
const winMs = typeof rawCtx?.config?.causalWindowMs === "number" ? rawCtx.config.causalWindowMs : 4000;
|
|
238
|
+
return await tagMap(sess, result, winMs).catch(() => undefined);
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
export default realtimePlugin;
|
|
242
|
+
export const plugin = realtimePlugin;
|
package/dist/page.d.ts
ADDED
package/dist/page.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// page-side probe shared by every tool in here. one per document, survives until navigation.
|
|
2
|
+
const PROBE = `(() => {
|
|
3
|
+
if (window.__wt_rt) return 1;
|
|
4
|
+
const rt = window.__wt_rt = { born: new WeakMap(), click: null, lastMut: 0 };
|
|
5
|
+
new MutationObserver(recs => {
|
|
6
|
+
const now = performance.now();
|
|
7
|
+
for (const r of recs) {
|
|
8
|
+
if (r.type === 'childList') {
|
|
9
|
+
for (const n of r.addedNodes) if (n.nodeType === 1 && !rt.born.has(n)) rt.born.set(n, now);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
rt.lastMut = now;
|
|
13
|
+
}).observe(document.documentElement, {
|
|
14
|
+
childList: true, subtree: true, characterData: true, attributes: true,
|
|
15
|
+
attributeFilter: ['class', 'hidden', 'aria-expanded', 'aria-hidden', 'open', 'disabled']
|
|
16
|
+
});
|
|
17
|
+
addEventListener('pointerdown', e => { rt.click = { x: e.clientX, y: e.clientY, t: performance.now() }; }, true);
|
|
18
|
+
return 0;
|
|
19
|
+
})()`;
|
|
20
|
+
export function sessOf(ctx) {
|
|
21
|
+
return ctx?.tab?.session || ctx?.session || ctx;
|
|
22
|
+
}
|
|
23
|
+
export async function run(sess, expression, awaitPromise = false) {
|
|
24
|
+
const out = await sess.Runtime.evaluate({ expression, returnByValue: true, awaitPromise });
|
|
25
|
+
if (out.exceptionDetails) {
|
|
26
|
+
const ex = out.exceptionDetails;
|
|
27
|
+
throw new Error(ex.exception?.description?.split("\n")[0] || ex.text || "page eval failed");
|
|
28
|
+
}
|
|
29
|
+
return out.result?.value;
|
|
30
|
+
}
|
|
31
|
+
export function arm(sess) {
|
|
32
|
+
return run(sess, PROBE);
|
|
33
|
+
}
|
package/dist/reflex.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
type Rule = {
|
|
2
|
+
action: "add" | "remove" | "list" | "clear";
|
|
3
|
+
id?: string;
|
|
4
|
+
when?: {
|
|
5
|
+
selector?: string;
|
|
6
|
+
text?: string;
|
|
7
|
+
only_new?: boolean;
|
|
8
|
+
enabled?: boolean;
|
|
9
|
+
canvas?: {
|
|
10
|
+
names: Record<string, string | string[]>;
|
|
11
|
+
js: string;
|
|
12
|
+
};
|
|
13
|
+
};
|
|
14
|
+
do?: {
|
|
15
|
+
type: "click" | "click_xy" | "key" | "js";
|
|
16
|
+
x?: number;
|
|
17
|
+
y?: number;
|
|
18
|
+
key?: string;
|
|
19
|
+
hold_ms?: number;
|
|
20
|
+
code?: string;
|
|
21
|
+
};
|
|
22
|
+
cooldown_ms?: number;
|
|
23
|
+
max_hits?: number;
|
|
24
|
+
ttl_ms?: number;
|
|
25
|
+
stop_selector?: string;
|
|
26
|
+
};
|
|
27
|
+
export declare function reflexTool(sess: any, a: Rule): Promise<any>;
|
|
28
|
+
export declare function statusTool(sess: any, waitMs?: number): Promise<any>;
|
|
29
|
+
export {};
|