effing-use 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -26
- package/package.json +4 -3
- package/skills/effing-use/SKILL.md +38 -11
- package/src/browser/delta.ts +173 -0
- package/src/browser/engine.ts +581 -58
- package/src/browser/evidence.ts +208 -0
- package/src/browser/identity.ts +237 -0
- package/src/browser/macro.ts +118 -0
- package/src/browser/record.ts +149 -0
- package/src/browser/refs.ts +192 -7
- package/src/browser/session.ts +16 -0
- package/src/browser/state.ts +100 -0
- package/src/cli.ts +173 -0
- package/src/config.ts +10 -0
- package/src/http.ts +1 -0
- package/src/tools/act.ts +10 -7
- package/src/tools/extract.ts +1 -0
- package/src/tools/observe.ts +5 -1
package/src/browser/engine.ts
CHANGED
|
@@ -1,8 +1,45 @@
|
|
|
1
1
|
import type { Page, BrowserContext } from "playwright";
|
|
2
2
|
import type { Config } from "../config.js";
|
|
3
|
-
import { EngineError, resolveLocator } from "./refs.js";
|
|
3
|
+
import { EngineError, resolveLocator, stableSelectorFor } from "./refs.js";
|
|
4
4
|
import { cap, saveText, stamp } from "./output.js";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
getConsoleLogs,
|
|
7
|
+
getNetworkLogs,
|
|
8
|
+
closeSession,
|
|
9
|
+
getMustObserve,
|
|
10
|
+
setMustObserve,
|
|
11
|
+
clearMustObserve,
|
|
12
|
+
} from "./session.js";
|
|
13
|
+
import { collectEffect, evaluateExpect, parseExpect } from "./evidence.js";
|
|
14
|
+
import {
|
|
15
|
+
getBaseline,
|
|
16
|
+
getBaselineUrl,
|
|
17
|
+
setBaseline,
|
|
18
|
+
computeDelta,
|
|
19
|
+
injectDirtyObserver,
|
|
20
|
+
checkDirty,
|
|
21
|
+
clearDirtyFlag,
|
|
22
|
+
ensureState as ensureDeltaState,
|
|
23
|
+
} from "./delta.js";
|
|
24
|
+
import { appendAction, appendNote, readState } from "./state.js";
|
|
25
|
+
import {
|
|
26
|
+
isRecording,
|
|
27
|
+
captureStep,
|
|
28
|
+
saveRecording,
|
|
29
|
+
loadRecording,
|
|
30
|
+
startRecording,
|
|
31
|
+
stopRecording,
|
|
32
|
+
getReplayCursor,
|
|
33
|
+
setReplayCursor,
|
|
34
|
+
clearReplayCursor,
|
|
35
|
+
} from "./record.js";
|
|
36
|
+
import { compileMacro, isIrreversible } from "./macro.js";
|
|
37
|
+
import {
|
|
38
|
+
registerFingerprints,
|
|
39
|
+
clearRegistry,
|
|
40
|
+
getFingerprint,
|
|
41
|
+
extractFingerprints as collectFingerprints,
|
|
42
|
+
} from "./identity.js";
|
|
6
43
|
|
|
7
44
|
export type ActAction =
|
|
8
45
|
| "open"
|
|
@@ -31,26 +68,45 @@ export type ActAction =
|
|
|
31
68
|
| "tab_new"
|
|
32
69
|
| "tab_select"
|
|
33
70
|
| "tab_close"
|
|
34
|
-
| "resize"
|
|
71
|
+
| "resize"
|
|
72
|
+
| "note"
|
|
73
|
+
| "record_start"
|
|
74
|
+
| "record_stop"
|
|
75
|
+
| "compile"
|
|
76
|
+
| "replay";
|
|
35
77
|
|
|
36
78
|
export interface ActOpts {
|
|
37
79
|
context?: BrowserContext;
|
|
38
80
|
sessionId?: string;
|
|
81
|
+
expect?: string;
|
|
82
|
+
approve?: boolean;
|
|
39
83
|
}
|
|
40
|
-
|
|
41
84
|
export interface ObserveOpts {
|
|
42
85
|
context?: BrowserContext;
|
|
43
86
|
sessionId?: string;
|
|
87
|
+
mode?: "full" | "delta";
|
|
88
|
+
scope?: string;
|
|
44
89
|
}
|
|
45
90
|
|
|
46
91
|
function timeoutOf(config: Config): number {
|
|
47
92
|
return config.timeoutMs;
|
|
48
93
|
}
|
|
49
|
-
|
|
50
94
|
async function liteState(page: Page): Promise<{ url: string; title: string }> {
|
|
51
95
|
return { url: page.url(), title: await page.title().catch(() => "") };
|
|
52
96
|
}
|
|
53
97
|
|
|
98
|
+
/**
|
|
99
|
+
* SPA navigations commit after the action returns (client router awaits data),
|
|
100
|
+
* so evidence collected immediately can report urlChanged:false for a click
|
|
101
|
+
* that DOES navigate (observed: kikitai "Try paste & read" → /read).
|
|
102
|
+
* Brief settle: one tick, then a second if the URL moved.
|
|
103
|
+
*/
|
|
104
|
+
async function settleSpa(page: Page, action: string): Promise<void> {
|
|
105
|
+
if (!["click", "dblclick", "press", "goal"].includes(action)) return;
|
|
106
|
+
const u0 = page.url();
|
|
107
|
+
await page.waitForTimeout(60);
|
|
108
|
+
if (page.url() !== u0) await page.waitForTimeout(60);
|
|
109
|
+
}
|
|
54
110
|
function toEngineError(e: unknown, fallbackHint: string): EngineError {
|
|
55
111
|
if (e instanceof EngineError) return e;
|
|
56
112
|
const msg = e instanceof Error ? e.message : String(e);
|
|
@@ -63,9 +119,35 @@ function toEngineError(e: unknown, fallbackHint: string): EngineError {
|
|
|
63
119
|
fallbackHint,
|
|
64
120
|
);
|
|
65
121
|
}
|
|
122
|
+
const MUTATING = new Set<string>([
|
|
123
|
+
"click",
|
|
124
|
+
"dblclick",
|
|
125
|
+
"fill",
|
|
126
|
+
"type",
|
|
127
|
+
"press",
|
|
128
|
+
"select",
|
|
129
|
+
"check",
|
|
130
|
+
"uncheck",
|
|
131
|
+
"hover",
|
|
132
|
+
"drag",
|
|
133
|
+
"upload",
|
|
134
|
+
"scroll",
|
|
135
|
+
"back",
|
|
136
|
+
"forward",
|
|
137
|
+
"reload",
|
|
138
|
+
"open",
|
|
139
|
+
"goto",
|
|
140
|
+
"resize",
|
|
141
|
+
"goal",
|
|
142
|
+
"tab_new",
|
|
143
|
+
"tab_select",
|
|
144
|
+
"tab_close",
|
|
145
|
+
]);
|
|
146
|
+
function isMutating(a: string): boolean {
|
|
147
|
+
return MUTATING.has(a);
|
|
148
|
+
}
|
|
66
149
|
|
|
67
150
|
// ---------------------------------------------------------------- act
|
|
68
|
-
|
|
69
151
|
export async function doAct(
|
|
70
152
|
page: Page,
|
|
71
153
|
config: Config,
|
|
@@ -75,6 +157,176 @@ export async function doAct(
|
|
|
75
157
|
opts?: ActOpts,
|
|
76
158
|
): Promise<Record<string, unknown>> {
|
|
77
159
|
const timeout = timeoutOf(config);
|
|
160
|
+
const sid = opts?.sessionId ?? "default";
|
|
161
|
+
|
|
162
|
+
// Per-action stash must never leak from a previous (possibly failed) act
|
|
163
|
+
const pageBag = page as unknown as Record<string, unknown>;
|
|
164
|
+
delete pageBag.__rebound;
|
|
165
|
+
|
|
166
|
+
// Non-page meta actions (no mustObserve guard, no evidence)
|
|
167
|
+
if (action === "note") {
|
|
168
|
+
if (!value && !target)
|
|
169
|
+
throw new EngineError(
|
|
170
|
+
"E_BAD_INPUT",
|
|
171
|
+
"Missing note text.",
|
|
172
|
+
"Pass value with the note.",
|
|
173
|
+
);
|
|
174
|
+
const note = value ?? target ?? "";
|
|
175
|
+
await appendNote(config, sid, note);
|
|
176
|
+
return { noted: true, note: note.slice(0, 300) };
|
|
177
|
+
}
|
|
178
|
+
if (action === "record_start") {
|
|
179
|
+
const name = (value ?? target ?? "recording").trim() || "recording";
|
|
180
|
+
startRecording(sid, name);
|
|
181
|
+
return { recording: true, name };
|
|
182
|
+
}
|
|
183
|
+
if (action === "record_stop") {
|
|
184
|
+
const rec = stopRecording(sid);
|
|
185
|
+
if (!rec)
|
|
186
|
+
throw new EngineError(
|
|
187
|
+
"E_BAD_INPUT",
|
|
188
|
+
"No active recording.",
|
|
189
|
+
"Call record_start first.",
|
|
190
|
+
);
|
|
191
|
+
const path = await saveRecording(config, rec.name, rec.steps as any);
|
|
192
|
+
return { recording: false, name: rec.name, steps: rec.steps.length, path };
|
|
193
|
+
}
|
|
194
|
+
if (action === "compile") {
|
|
195
|
+
const name = (value ?? target ?? "").trim();
|
|
196
|
+
if (!name)
|
|
197
|
+
throw new EngineError(
|
|
198
|
+
"E_BAD_INPUT",
|
|
199
|
+
"Missing macro name.",
|
|
200
|
+
"Pass value with the recording name.",
|
|
201
|
+
);
|
|
202
|
+
const rec = await loadRecording(config, name).catch(() => {
|
|
203
|
+
throw new EngineError(
|
|
204
|
+
"E_NOT_FOUND",
|
|
205
|
+
`No recording "${name}".`,
|
|
206
|
+
"Call record_start/stop first.",
|
|
207
|
+
);
|
|
208
|
+
});
|
|
209
|
+
const { tsPath, mdPath, warnings } = await compileMacro(
|
|
210
|
+
config,
|
|
211
|
+
name,
|
|
212
|
+
rec.steps as any,
|
|
213
|
+
);
|
|
214
|
+
return {
|
|
215
|
+
compiled: true,
|
|
216
|
+
name,
|
|
217
|
+
steps: rec.steps.length,
|
|
218
|
+
tsPath,
|
|
219
|
+
mdPath,
|
|
220
|
+
warnings,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
if (action === "replay") {
|
|
224
|
+
const name = (value ?? target ?? "").trim();
|
|
225
|
+
if (!name)
|
|
226
|
+
throw new EngineError(
|
|
227
|
+
"E_BAD_INPUT",
|
|
228
|
+
"Missing macro name.",
|
|
229
|
+
"Pass value with the macro name.",
|
|
230
|
+
);
|
|
231
|
+
const rec = await loadRecording(config, name).catch(() => {
|
|
232
|
+
throw new EngineError(
|
|
233
|
+
"E_NOT_FOUND",
|
|
234
|
+
`No recording "${name}".`,
|
|
235
|
+
"Compile first or check name.",
|
|
236
|
+
);
|
|
237
|
+
});
|
|
238
|
+
const approve = opts?.approve === true;
|
|
239
|
+
// Plan §6.3: run benign steps, pause AT each approval-gated step, and keep
|
|
240
|
+
// a cursor so approve:true resumes here instead of re-running the prefix.
|
|
241
|
+
const stepsArr = rec.steps as any[];
|
|
242
|
+
const start = getReplayCursor(sid, name) ?? 0;
|
|
243
|
+
const results: any[] = [];
|
|
244
|
+
for (let i = start; i < stepsArr.length; i++) {
|
|
245
|
+
const s = stepsArr[i];
|
|
246
|
+
if (isIrreversible(s) && !approve) {
|
|
247
|
+
setReplayCursor(sid, name, i);
|
|
248
|
+
const ran = i - start;
|
|
249
|
+
return {
|
|
250
|
+
ok: false,
|
|
251
|
+
code: "E_APPROVAL_REQUIRED",
|
|
252
|
+
message:
|
|
253
|
+
`Step ${i + 1} requires approval (${s.op}).` +
|
|
254
|
+
(ran > 0 ? ` Ran ${ran} prior step(s).` : ""),
|
|
255
|
+
hint: "Re-run with approve:true to resume from this step.",
|
|
256
|
+
step: i + 1,
|
|
257
|
+
completed: results.length,
|
|
258
|
+
description: `${s.op} ${s.target ?? ""}`,
|
|
259
|
+
} as any;
|
|
260
|
+
}
|
|
261
|
+
try {
|
|
262
|
+
const r = await doAct(
|
|
263
|
+
page,
|
|
264
|
+
config,
|
|
265
|
+
s.op as ActAction,
|
|
266
|
+
s.target,
|
|
267
|
+
s.value === "«redacted»" ? undefined : s.value,
|
|
268
|
+
{ ...opts, expect: s.expect },
|
|
269
|
+
);
|
|
270
|
+
results.push({ ok: true, seq: s.seq, ...r });
|
|
271
|
+
} catch (e) {
|
|
272
|
+
clearReplayCursor(sid, name);
|
|
273
|
+
const err = toEngineError(e, "Replay failed.");
|
|
274
|
+
results.push({
|
|
275
|
+
ok: false,
|
|
276
|
+
seq: s.seq,
|
|
277
|
+
code: err.code,
|
|
278
|
+
message: err.message,
|
|
279
|
+
});
|
|
280
|
+
return { replayed: false, name, failSeq: s.seq, results } as any;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
clearReplayCursor(sid, name);
|
|
284
|
+
return {
|
|
285
|
+
replayed: true,
|
|
286
|
+
name,
|
|
287
|
+
steps: results.length,
|
|
288
|
+
resumedFrom: start,
|
|
289
|
+
results,
|
|
290
|
+
} as any;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// Failure contract guard
|
|
294
|
+
if (isMutating(action) && getMustObserve(sid)) {
|
|
295
|
+
throw new EngineError(
|
|
296
|
+
"E_MUST_OBSERVE",
|
|
297
|
+
"Must observe before next mutation.",
|
|
298
|
+
"Call browser_observe kind=snapshot first.",
|
|
299
|
+
{ mustObserve: true } as any,
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// Validate expect syntax early
|
|
304
|
+
if (opts?.expect) {
|
|
305
|
+
const parsed = parseExpect(opts.expect);
|
|
306
|
+
if (!parsed)
|
|
307
|
+
throw new EngineError(
|
|
308
|
+
"E_BAD_EXPECT",
|
|
309
|
+
`Bad expect "${opts.expect}".`,
|
|
310
|
+
"Use url~<regex> | text~<regex> | visible=<css> | gone=<css>",
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const urlBefore = page.url();
|
|
315
|
+
let domBefore: string | null = null;
|
|
316
|
+
try {
|
|
317
|
+
// Full innerText (50k cap) — evidence diffs need the whole page, not just
|
|
318
|
+
// the first 200 chars (v0.2: below-fold changes looked like no-ops).
|
|
319
|
+
domBefore = await page.evaluate(
|
|
320
|
+
() => document.body?.innerText?.slice(0, 50000) ?? "",
|
|
321
|
+
);
|
|
322
|
+
} catch {
|
|
323
|
+
domBefore = null;
|
|
324
|
+
}
|
|
325
|
+
const consoleBefore = getConsoleLogs(sid).length;
|
|
326
|
+
const networkBefore = getNetworkLogs(sid).length;
|
|
327
|
+
|
|
328
|
+
let result: Record<string, unknown>;
|
|
329
|
+
let isTimeout = false;
|
|
78
330
|
try {
|
|
79
331
|
switch (action) {
|
|
80
332
|
case "open":
|
|
@@ -87,7 +339,8 @@ export async function doAct(
|
|
|
87
339
|
"Pass the URL in value (or target).",
|
|
88
340
|
);
|
|
89
341
|
await page.goto(url, { waitUntil: "domcontentloaded", timeout });
|
|
90
|
-
|
|
342
|
+
result = { ...(await liteState(page)) };
|
|
343
|
+
break;
|
|
91
344
|
}
|
|
92
345
|
case "click": {
|
|
93
346
|
if (!target)
|
|
@@ -96,11 +349,16 @@ export async function doAct(
|
|
|
96
349
|
"Missing target.",
|
|
97
350
|
"Pass an e-ref, role= selector, or CSS.",
|
|
98
351
|
);
|
|
99
|
-
const loc = await resolveLocator(page, target);
|
|
352
|
+
const loc = await resolveLocator(page, target, sid);
|
|
100
353
|
const button =
|
|
101
354
|
(value as "left" | "middle" | "right" | undefined) ?? "left";
|
|
102
355
|
await loc.click({ button, timeout });
|
|
103
|
-
|
|
356
|
+
result = { ...(await liteState(page)) };
|
|
357
|
+
if ((page as any).__rebound) {
|
|
358
|
+
(result as any).rebound = true;
|
|
359
|
+
delete (page as any).__rebound;
|
|
360
|
+
}
|
|
361
|
+
break;
|
|
104
362
|
}
|
|
105
363
|
case "dblclick": {
|
|
106
364
|
if (!target)
|
|
@@ -109,9 +367,10 @@ export async function doAct(
|
|
|
109
367
|
"Missing target.",
|
|
110
368
|
"Pass an e-ref, role= selector, or CSS.",
|
|
111
369
|
);
|
|
112
|
-
const loc = await resolveLocator(page, target);
|
|
370
|
+
const loc = await resolveLocator(page, target, sid);
|
|
113
371
|
await loc.dblclick({ timeout });
|
|
114
|
-
|
|
372
|
+
result = { ...(await liteState(page)) };
|
|
373
|
+
break;
|
|
115
374
|
}
|
|
116
375
|
case "fill": {
|
|
117
376
|
if (!target)
|
|
@@ -120,13 +379,15 @@ export async function doAct(
|
|
|
120
379
|
"Missing target.",
|
|
121
380
|
"Pass an e-ref, role= selector, or CSS.",
|
|
122
381
|
);
|
|
123
|
-
const loc = await resolveLocator(page, target);
|
|
382
|
+
const loc = await resolveLocator(page, target, sid);
|
|
124
383
|
await loc.fill(value ?? "", { timeout });
|
|
125
|
-
|
|
384
|
+
result = { ...(await liteState(page)) };
|
|
385
|
+
break;
|
|
126
386
|
}
|
|
127
387
|
case "type": {
|
|
128
388
|
await page.keyboard.type(value ?? "", { delay: 0 });
|
|
129
|
-
|
|
389
|
+
result = { ...(await liteState(page)) };
|
|
390
|
+
break;
|
|
130
391
|
}
|
|
131
392
|
case "press": {
|
|
132
393
|
if (!value)
|
|
@@ -136,7 +397,8 @@ export async function doAct(
|
|
|
136
397
|
"Pass a key like Enter, Tab, Escape in value.",
|
|
137
398
|
);
|
|
138
399
|
await page.keyboard.press(value);
|
|
139
|
-
|
|
400
|
+
result = { ...(await liteState(page)) };
|
|
401
|
+
break;
|
|
140
402
|
}
|
|
141
403
|
case "select": {
|
|
142
404
|
if (!target)
|
|
@@ -145,9 +407,10 @@ export async function doAct(
|
|
|
145
407
|
"Missing target.",
|
|
146
408
|
"Pass a select element ref in target.",
|
|
147
409
|
);
|
|
148
|
-
const loc = await resolveLocator(page, target);
|
|
410
|
+
const loc = await resolveLocator(page, target, sid);
|
|
149
411
|
await loc.selectOption(value ?? "", { timeout });
|
|
150
|
-
|
|
412
|
+
result = { ...(await liteState(page)) };
|
|
413
|
+
break;
|
|
151
414
|
}
|
|
152
415
|
case "check": {
|
|
153
416
|
if (!target)
|
|
@@ -156,9 +419,10 @@ export async function doAct(
|
|
|
156
419
|
"Missing target.",
|
|
157
420
|
"Pass a checkbox ref in target.",
|
|
158
421
|
);
|
|
159
|
-
const loc = await resolveLocator(page, target);
|
|
422
|
+
const loc = await resolveLocator(page, target, sid);
|
|
160
423
|
await loc.check({ timeout });
|
|
161
|
-
|
|
424
|
+
result = { ...(await liteState(page)) };
|
|
425
|
+
break;
|
|
162
426
|
}
|
|
163
427
|
case "uncheck": {
|
|
164
428
|
if (!target)
|
|
@@ -167,9 +431,10 @@ export async function doAct(
|
|
|
167
431
|
"Missing target.",
|
|
168
432
|
"Pass a checkbox ref in target.",
|
|
169
433
|
);
|
|
170
|
-
const loc = await resolveLocator(page, target);
|
|
434
|
+
const loc = await resolveLocator(page, target, sid);
|
|
171
435
|
await loc.uncheck({ timeout });
|
|
172
|
-
|
|
436
|
+
result = { ...(await liteState(page)) };
|
|
437
|
+
break;
|
|
173
438
|
}
|
|
174
439
|
case "hover": {
|
|
175
440
|
if (!target)
|
|
@@ -178,9 +443,10 @@ export async function doAct(
|
|
|
178
443
|
"Missing target.",
|
|
179
444
|
"Pass an e-ref, role= selector, or CSS.",
|
|
180
445
|
);
|
|
181
|
-
const loc = await resolveLocator(page, target);
|
|
446
|
+
const loc = await resolveLocator(page, target, sid);
|
|
182
447
|
await loc.hover({ timeout });
|
|
183
|
-
|
|
448
|
+
result = { ...(await liteState(page)) };
|
|
449
|
+
break;
|
|
184
450
|
}
|
|
185
451
|
case "drag": {
|
|
186
452
|
if (!target || !value)
|
|
@@ -189,10 +455,11 @@ export async function doAct(
|
|
|
189
455
|
"Missing drag endpoints.",
|
|
190
456
|
"Pass start ref in target and end ref in value.",
|
|
191
457
|
);
|
|
192
|
-
const start = await resolveLocator(page, target);
|
|
193
|
-
const end = await resolveLocator(page, value);
|
|
458
|
+
const start = await resolveLocator(page, target, sid);
|
|
459
|
+
const end = await resolveLocator(page, value, sid);
|
|
194
460
|
await start.dragTo(end, { timeout });
|
|
195
|
-
|
|
461
|
+
result = { ...(await liteState(page)) };
|
|
462
|
+
break;
|
|
196
463
|
}
|
|
197
464
|
case "upload": {
|
|
198
465
|
const files = (value ?? "")
|
|
@@ -206,7 +473,8 @@ export async function doAct(
|
|
|
206
473
|
"Pass comma-separated file paths in value.",
|
|
207
474
|
);
|
|
208
475
|
await page.setInputFiles("input[type=file]", files, { timeout });
|
|
209
|
-
|
|
476
|
+
result = { ...(await liteState(page)), files };
|
|
477
|
+
break;
|
|
210
478
|
}
|
|
211
479
|
case "scroll": {
|
|
212
480
|
const dir = (target ?? "").toLowerCase();
|
|
@@ -219,52 +487,60 @@ export async function doAct(
|
|
|
219
487
|
window.scrollTo(0, document.body.scrollHeight),
|
|
220
488
|
);
|
|
221
489
|
else if (target) {
|
|
222
|
-
const loc = await resolveLocator(page, target);
|
|
490
|
+
const loc = await resolveLocator(page, target, sid);
|
|
223
491
|
await loc.scrollIntoViewIfNeeded({ timeout });
|
|
224
|
-
} else
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
return { ...(await liteState(page)) };
|
|
492
|
+
} else await page.mouse.wheel(0, 500);
|
|
493
|
+
result = { ...(await liteState(page)) };
|
|
494
|
+
break;
|
|
228
495
|
}
|
|
229
496
|
case "back":
|
|
230
497
|
await page
|
|
231
498
|
.goBack({ waitUntil: "domcontentloaded", timeout })
|
|
232
499
|
.catch(() => null);
|
|
233
|
-
|
|
500
|
+
result = { ...(await liteState(page)) };
|
|
501
|
+
break;
|
|
234
502
|
case "forward":
|
|
235
503
|
await page
|
|
236
504
|
.goForward({ waitUntil: "domcontentloaded", timeout })
|
|
237
505
|
.catch(() => null);
|
|
238
|
-
|
|
506
|
+
result = { ...(await liteState(page)) };
|
|
507
|
+
break;
|
|
239
508
|
case "reload":
|
|
240
509
|
await page.reload({ waitUntil: "domcontentloaded", timeout });
|
|
241
|
-
|
|
510
|
+
result = { ...(await liteState(page)) };
|
|
511
|
+
break;
|
|
242
512
|
case "wait": {
|
|
243
513
|
const t = target ?? "";
|
|
244
514
|
const msMatch = /^ms:(\d+)$/.exec(t);
|
|
245
515
|
if (msMatch) {
|
|
246
516
|
await page.waitForTimeout(Number(msMatch[1]));
|
|
247
|
-
|
|
517
|
+
result = { ...(await liteState(page)), waited: t };
|
|
518
|
+
break;
|
|
248
519
|
}
|
|
249
520
|
const textMatch = /^text:(.+)$/.exec(t);
|
|
250
521
|
if (textMatch) {
|
|
251
522
|
await page.getByText(textMatch[1]).first().waitFor({ timeout });
|
|
252
|
-
|
|
523
|
+
result = { ...(await liteState(page)), waited: t };
|
|
524
|
+
break;
|
|
253
525
|
}
|
|
254
526
|
if (t) {
|
|
255
|
-
const loc = await resolveLocator(page, t);
|
|
527
|
+
const loc = await resolveLocator(page, t, sid);
|
|
256
528
|
await loc.waitFor({ timeout });
|
|
257
|
-
|
|
529
|
+
result = { ...(await liteState(page)), waited: t };
|
|
530
|
+
break;
|
|
258
531
|
}
|
|
259
532
|
await page.waitForTimeout(500);
|
|
260
|
-
|
|
533
|
+
result = { ...(await liteState(page)) };
|
|
534
|
+
break;
|
|
261
535
|
}
|
|
262
536
|
case "dialog_accept":
|
|
263
537
|
page.once("dialog", (d) => void d.accept(value));
|
|
264
|
-
|
|
538
|
+
result = { armed: true };
|
|
539
|
+
break;
|
|
265
540
|
case "dialog_dismiss":
|
|
266
541
|
page.once("dialog", (d) => void d.dismiss());
|
|
267
|
-
|
|
542
|
+
result = { armed: true };
|
|
543
|
+
break;
|
|
268
544
|
case "resize": {
|
|
269
545
|
const m = /^(\d+)x(\d+)$/.exec(value ?? "");
|
|
270
546
|
if (!m)
|
|
@@ -277,7 +553,8 @@ export async function doAct(
|
|
|
277
553
|
width: Number(m[1]),
|
|
278
554
|
height: Number(m[2]),
|
|
279
555
|
});
|
|
280
|
-
|
|
556
|
+
result = { ...(await liteState(page)), viewport: value };
|
|
557
|
+
break;
|
|
281
558
|
}
|
|
282
559
|
case "tab_new": {
|
|
283
560
|
if (!opts?.context)
|
|
@@ -292,7 +569,11 @@ export async function doAct(
|
|
|
292
569
|
waitUntil: "domcontentloaded",
|
|
293
570
|
timeout,
|
|
294
571
|
});
|
|
295
|
-
|
|
572
|
+
result = {
|
|
573
|
+
url: p.url(),
|
|
574
|
+
tabs: opts.context.pages().map((x) => x.url()),
|
|
575
|
+
};
|
|
576
|
+
break;
|
|
296
577
|
}
|
|
297
578
|
case "tab_select": {
|
|
298
579
|
if (!opts?.context)
|
|
@@ -310,7 +591,8 @@ export async function doAct(
|
|
|
310
591
|
"Call browser_observe kind=tabs for the tab list.",
|
|
311
592
|
);
|
|
312
593
|
await pages[i].bringToFront();
|
|
313
|
-
|
|
594
|
+
result = { url: pages[i].url() };
|
|
595
|
+
break;
|
|
314
596
|
}
|
|
315
597
|
case "tab_close": {
|
|
316
598
|
if (!opts?.context)
|
|
@@ -329,13 +611,21 @@ export async function doAct(
|
|
|
329
611
|
"Call browser_observe kind=tabs for the tab list.",
|
|
330
612
|
);
|
|
331
613
|
await pages[i].close();
|
|
332
|
-
|
|
614
|
+
result = { closed: i, tabs: opts.context.pages().map((x) => x.url()) };
|
|
615
|
+
break;
|
|
333
616
|
}
|
|
334
617
|
case "close":
|
|
335
|
-
await closeSession(
|
|
336
|
-
|
|
618
|
+
await closeSession(sid);
|
|
619
|
+
clearRegistry(sid);
|
|
620
|
+
result = { closed: true };
|
|
621
|
+
break;
|
|
337
622
|
case "goal":
|
|
338
|
-
|
|
623
|
+
// Route through the normal post-action pipeline (guard, evidence,
|
|
624
|
+
// dirty flag, recording) — v0.2 returned here and skipped all of it.
|
|
625
|
+
result = {
|
|
626
|
+
...(await doGoal(page, config, value ?? target ?? "", opts)),
|
|
627
|
+
};
|
|
628
|
+
break;
|
|
339
629
|
case "batch":
|
|
340
630
|
throw new EngineError(
|
|
341
631
|
"E_BAD_INPUT",
|
|
@@ -350,11 +640,94 @@ export async function doAct(
|
|
|
350
640
|
);
|
|
351
641
|
}
|
|
352
642
|
} catch (e) {
|
|
353
|
-
|
|
643
|
+
const err = toEngineError(
|
|
354
644
|
e,
|
|
355
645
|
"Retry with a fresh snapshot ref, or re-observe state.",
|
|
356
646
|
);
|
|
647
|
+
if (err.code === "E_TIMEOUT") {
|
|
648
|
+
isTimeout = true;
|
|
649
|
+
setMustObserve(sid, true);
|
|
650
|
+
}
|
|
651
|
+
throw err;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
// Post-action: evidence, expect, state, dirty, recording
|
|
655
|
+
let effect: any = undefined;
|
|
656
|
+
if (isMutating(action)) {
|
|
657
|
+
await settleSpa(page, action);
|
|
658
|
+
effect = await collectEffect(
|
|
659
|
+
page,
|
|
660
|
+
sid,
|
|
661
|
+
urlBefore,
|
|
662
|
+
domBefore,
|
|
663
|
+
consoleBefore,
|
|
664
|
+
networkBefore,
|
|
665
|
+
config.effectMaxChars,
|
|
666
|
+
);
|
|
667
|
+
// Failure contract: timeout with no DOM/URL change => mustObserve
|
|
668
|
+
if (isTimeout && !effect.urlChanged && effect.domChanged.length === 0)
|
|
669
|
+
setMustObserve(sid, true);
|
|
670
|
+
// Expect evaluation
|
|
671
|
+
if (opts?.expect) {
|
|
672
|
+
const ev = await evaluateExpect(page, opts.expect);
|
|
673
|
+
if (!ev.ok) {
|
|
674
|
+
effect.mustObserve = false;
|
|
675
|
+
throw new EngineError(
|
|
676
|
+
"E_EXPECT",
|
|
677
|
+
ev.message ?? "Expect failed.",
|
|
678
|
+
"Check effect and re-plan.",
|
|
679
|
+
{ effect, expect: opts.expect } as any,
|
|
680
|
+
);
|
|
681
|
+
}
|
|
682
|
+
(result as any).expect = "pass";
|
|
683
|
+
}
|
|
684
|
+
if (effect) {
|
|
685
|
+
(result as any).effect = effect;
|
|
686
|
+
if (effect.mustObserve) (result as any).mustObserve = true;
|
|
687
|
+
}
|
|
688
|
+
// Mark dirty for delta
|
|
689
|
+
ensureDeltaState(sid).dirty = true;
|
|
690
|
+
try {
|
|
691
|
+
await page.evaluate(() => {
|
|
692
|
+
(window as any).__effDirty = true;
|
|
693
|
+
});
|
|
694
|
+
} catch {}
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
// Append to state ring
|
|
698
|
+
const actionLine = `${action} ${target ?? ""} ${value ?? ""}`
|
|
699
|
+
.trim()
|
|
700
|
+
.slice(0, 120);
|
|
701
|
+
await appendAction(
|
|
702
|
+
config,
|
|
703
|
+
sid,
|
|
704
|
+
`${actionLine} -> ${effect ? JSON.stringify(effect).slice(0, 80) : "ok"}`,
|
|
705
|
+
);
|
|
706
|
+
|
|
707
|
+
// Recording capture — resolve a portable selector (id → name → ARIA →
|
|
708
|
+
// data-* → placeholder → text) so compiled macros don't depend on e-refs.
|
|
709
|
+
if (isRecording(sid) && isMutating(action)) {
|
|
710
|
+
let resolved = target;
|
|
711
|
+
if (target) {
|
|
712
|
+
resolved =
|
|
713
|
+
(await stableSelectorFor(page, target, sid).catch(() => null)) ??
|
|
714
|
+
target;
|
|
715
|
+
}
|
|
716
|
+
captureStep(
|
|
717
|
+
sid,
|
|
718
|
+
{
|
|
719
|
+
op: action,
|
|
720
|
+
target,
|
|
721
|
+
value,
|
|
722
|
+
expect: opts?.expect,
|
|
723
|
+
resolvedSelector: resolved,
|
|
724
|
+
targetFingerprint: target ? getFingerprint(sid, target) : undefined,
|
|
725
|
+
},
|
|
726
|
+
config,
|
|
727
|
+
);
|
|
357
728
|
}
|
|
729
|
+
|
|
730
|
+
return result;
|
|
358
731
|
}
|
|
359
732
|
|
|
360
733
|
// ---------------------------------------------------------------- goal
|
|
@@ -473,15 +846,124 @@ export async function doObserve(
|
|
|
473
846
|
case "title":
|
|
474
847
|
return { title: await page.title().catch(() => ""), url: page.url() };
|
|
475
848
|
case "snapshot": {
|
|
476
|
-
const
|
|
849
|
+
const mode = (opts?.mode ?? (config.deltaDefault ? "delta" : "full")) as
|
|
850
|
+
| "full"
|
|
851
|
+
| "delta";
|
|
852
|
+
const scope = opts?.scope;
|
|
853
|
+
// Navigation forces a fresh full baseline (plan §5.1) and needs a settle
|
|
854
|
+
// beat: SPA routes hydrate after the URL changes — snapshots taken too
|
|
855
|
+
// early miss the header/nav entirely (observed on kikitai /read).
|
|
856
|
+
const prevUrl = getBaselineUrl(sessionId);
|
|
857
|
+
const urlChanged = prevUrl !== null && prevUrl !== page.url();
|
|
858
|
+
const settleHydration = async () => {
|
|
859
|
+
if (!urlChanged) return;
|
|
860
|
+
await page
|
|
861
|
+
.waitForLoadState("networkidle", { timeout: 1500 })
|
|
862
|
+
.catch(() => {});
|
|
863
|
+
await page.waitForTimeout(80);
|
|
864
|
+
};
|
|
865
|
+
// Scoped snapshot: only within selector
|
|
866
|
+
let yaml: string;
|
|
867
|
+
if (scope) {
|
|
868
|
+
await settleHydration();
|
|
869
|
+
yaml = await buildSnapshot(page, scope);
|
|
870
|
+
const path = await saveText(config, stamp("snapshot", "yaml"), yaml);
|
|
871
|
+
const { text, truncated } = cap(yaml, config.outputMaxChars);
|
|
872
|
+
clearMustObserve(sessionId);
|
|
873
|
+
return {
|
|
874
|
+
path,
|
|
875
|
+
summary: text,
|
|
876
|
+
truncated,
|
|
877
|
+
url: page.url(),
|
|
878
|
+
title: await page.title().catch(() => ""),
|
|
879
|
+
scope,
|
|
880
|
+
mode: "scoped",
|
|
881
|
+
};
|
|
882
|
+
}
|
|
883
|
+
// Delta path
|
|
884
|
+
if (mode === "delta") {
|
|
885
|
+
const baseline = getBaseline(sessionId);
|
|
886
|
+
// No baseline, or the page navigated since the baseline → forced full
|
|
887
|
+
if (!baseline || urlChanged) {
|
|
888
|
+
await settleHydration();
|
|
889
|
+
yaml = await buildSnapshot(page);
|
|
890
|
+
setBaseline(sessionId, yaml, page.url());
|
|
891
|
+
await injectDirtyObserver(page, sessionId);
|
|
892
|
+
// register fingerprints
|
|
893
|
+
const fps = await collectFingerprints(page);
|
|
894
|
+
registerFingerprints(sessionId, fps);
|
|
895
|
+
const path = await saveText(config, stamp("snapshot", "yaml"), yaml);
|
|
896
|
+
const { text, truncated } = cap(yaml, config.outputMaxChars);
|
|
897
|
+
clearMustObserve(sessionId);
|
|
898
|
+
return {
|
|
899
|
+
path,
|
|
900
|
+
summary: text,
|
|
901
|
+
truncated,
|
|
902
|
+
url: page.url(),
|
|
903
|
+
title: await page.title().catch(() => ""),
|
|
904
|
+
mode: "full",
|
|
905
|
+
};
|
|
906
|
+
}
|
|
907
|
+
// Check dirty flag
|
|
908
|
+
const dirty = await checkDirty(page).catch(() => true);
|
|
909
|
+
if (!dirty) {
|
|
910
|
+
clearMustObserve(sessionId);
|
|
911
|
+
return {
|
|
912
|
+
unchanged: true,
|
|
913
|
+
url: page.url(),
|
|
914
|
+
title: await page.title().catch(() => ""),
|
|
915
|
+
hint: "last snapshot still valid",
|
|
916
|
+
mode: "delta",
|
|
917
|
+
};
|
|
918
|
+
}
|
|
919
|
+
yaml = await buildSnapshot(page);
|
|
920
|
+
const { delta, unchanged } = computeDelta(baseline, yaml);
|
|
921
|
+
if (unchanged) {
|
|
922
|
+
await clearDirtyFlag(page);
|
|
923
|
+
clearMustObserve(sessionId);
|
|
924
|
+
return {
|
|
925
|
+
unchanged: true,
|
|
926
|
+
url: page.url(),
|
|
927
|
+
title: await page.title().catch(() => ""),
|
|
928
|
+
hint: "no changes",
|
|
929
|
+
mode: "delta",
|
|
930
|
+
};
|
|
931
|
+
}
|
|
932
|
+
setBaseline(sessionId, yaml, page.url());
|
|
933
|
+
await clearDirtyFlag(page);
|
|
934
|
+
await injectDirtyObserver(page, sessionId);
|
|
935
|
+
const fps = await collectFingerprints(page);
|
|
936
|
+
registerFingerprints(sessionId, fps);
|
|
937
|
+
const path = await saveText(config, stamp("snapshot", "yaml"), yaml);
|
|
938
|
+
const { text: dtext, truncated } = cap(delta, config.outputMaxChars);
|
|
939
|
+
clearMustObserve(sessionId);
|
|
940
|
+
return {
|
|
941
|
+
path,
|
|
942
|
+
delta: dtext,
|
|
943
|
+
truncated,
|
|
944
|
+
url: page.url(),
|
|
945
|
+
title: await page.title().catch(() => ""),
|
|
946
|
+
mode: "delta",
|
|
947
|
+
fullPath: path,
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
// Full mode
|
|
951
|
+
await settleHydration();
|
|
952
|
+
yaml = await buildSnapshot(page);
|
|
953
|
+
setBaseline(sessionId, yaml, page.url());
|
|
954
|
+
await injectDirtyObserver(page, sessionId);
|
|
955
|
+
const fps2 = await collectFingerprints(page);
|
|
956
|
+
registerFingerprints(sessionId, fps2);
|
|
477
957
|
const path = await saveText(config, stamp("snapshot", "yaml"), yaml);
|
|
478
958
|
const { text, truncated } = cap(yaml, config.outputMaxChars);
|
|
959
|
+
clearMustObserve(sessionId);
|
|
479
960
|
return {
|
|
480
961
|
path,
|
|
481
962
|
summary: text,
|
|
482
963
|
truncated,
|
|
483
964
|
url: page.url(),
|
|
484
965
|
title: await page.title().catch(() => ""),
|
|
966
|
+
mode: "full",
|
|
485
967
|
};
|
|
486
968
|
}
|
|
487
969
|
case "screenshot": {
|
|
@@ -524,23 +1006,52 @@ export async function doObserve(
|
|
|
524
1006
|
}
|
|
525
1007
|
}
|
|
526
1008
|
|
|
527
|
-
async function buildSnapshot(page: Page): Promise<string> {
|
|
1009
|
+
async function buildSnapshot(page: Page, scope?: string): Promise<string> {
|
|
528
1010
|
const header = [
|
|
529
1011
|
"# Snapshot — refs are nth-match: eN = Nth match of (button, a, input, select, textarea, [role=button], [tabindex]) in DOM order.",
|
|
530
1012
|
`# url: ${page.url()}`,
|
|
531
1013
|
].join("\n");
|
|
532
|
-
const items = await page.evaluate(() => {
|
|
1014
|
+
const items = await page.evaluate((scopeSel) => {
|
|
1015
|
+
const root = scopeSel ? document.querySelector(scopeSel) : document;
|
|
1016
|
+
if (!root) return [] as string[];
|
|
533
1017
|
const els = [
|
|
534
|
-
...
|
|
1018
|
+
...(root as Element).querySelectorAll(
|
|
535
1019
|
"button, a, input, select, textarea, [role=button], [tabindex]",
|
|
536
1020
|
),
|
|
537
1021
|
];
|
|
1022
|
+
// also include root itself if it matches
|
|
1023
|
+
if (
|
|
1024
|
+
scopeSel &&
|
|
1025
|
+
(root as Element).matches?.(
|
|
1026
|
+
"button, a, input, select, textarea, [role=button], [tabindex]",
|
|
1027
|
+
)
|
|
1028
|
+
) {
|
|
1029
|
+
els.unshift(root as Element);
|
|
1030
|
+
}
|
|
538
1031
|
return els.slice(0, 200).map((el) => {
|
|
539
1032
|
const tag = el.tagName.toLowerCase();
|
|
540
|
-
|
|
1033
|
+
let text = (el.textContent ?? "")
|
|
541
1034
|
.trim()
|
|
542
1035
|
.replace(/\s+/g, " ")
|
|
543
1036
|
.slice(0, 80);
|
|
1037
|
+
// Form controls carry no text — surface placeholder/label so refs stop
|
|
1038
|
+
// showing as `input ""`, and mark checked state (real state changes that
|
|
1039
|
+
// delta should surface).
|
|
1040
|
+
const input = el as HTMLInputElement;
|
|
1041
|
+
if (!text) {
|
|
1042
|
+
text = (
|
|
1043
|
+
input.placeholder ||
|
|
1044
|
+
(input.labels && input.labels[0]?.textContent) ||
|
|
1045
|
+
""
|
|
1046
|
+
)
|
|
1047
|
+
.trim()
|
|
1048
|
+
.replace(/\s+/g, " ")
|
|
1049
|
+
.slice(0, 80);
|
|
1050
|
+
}
|
|
1051
|
+
const checked =
|
|
1052
|
+
(tag === "input" || tag === "select") && input.checked
|
|
1053
|
+
? " [checked]"
|
|
1054
|
+
: "";
|
|
544
1055
|
const aria = el.getAttribute("aria-label") ?? "";
|
|
545
1056
|
const name = el.getAttribute("name") ?? "";
|
|
546
1057
|
const id = el.getAttribute("id") ?? "";
|
|
@@ -551,9 +1062,9 @@ async function buildSnapshot(page: Page): Promise<string> {
|
|
|
551
1062
|
]
|
|
552
1063
|
.filter(Boolean)
|
|
553
1064
|
.join(" ");
|
|
554
|
-
return `${tag} "${text}"${extra ? " " + extra : ""}`;
|
|
1065
|
+
return `${tag} "${text}"${extra ? " " + extra : ""}${checked}`;
|
|
555
1066
|
});
|
|
556
|
-
});
|
|
1067
|
+
}, scope ?? null);
|
|
557
1068
|
const lines = items.map((line, i) => `[e${i}] ${line}`);
|
|
558
1069
|
return `${header}\n${lines.join("\n") || "(no interactive elements)"}`;
|
|
559
1070
|
}
|
|
@@ -657,11 +1168,23 @@ export async function doExtract(
|
|
|
657
1168
|
await ctx.tracing.stop({ path: saved });
|
|
658
1169
|
return { path: saved };
|
|
659
1170
|
}
|
|
1171
|
+
case "state": {
|
|
1172
|
+
const sid = opts?.sessionId ?? "default";
|
|
1173
|
+
const st = await readState(config, sid);
|
|
1174
|
+
const dirty = await checkDirty(page).catch(() => false);
|
|
1175
|
+
return {
|
|
1176
|
+
notes: st.notes,
|
|
1177
|
+
lastActions: st.lastActions,
|
|
1178
|
+
path: st.path,
|
|
1179
|
+
url: page.url(),
|
|
1180
|
+
dirty,
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
660
1183
|
default:
|
|
661
1184
|
throw new EngineError(
|
|
662
1185
|
"E_BAD_INPUT",
|
|
663
1186
|
`Unknown extract kind "${kind}".`,
|
|
664
|
-
"Valid: text, html, table, query, pdf, trace_start, trace_stop.",
|
|
1187
|
+
"Valid: text, html, table, query, pdf, trace_start, trace_stop, state.",
|
|
665
1188
|
);
|
|
666
1189
|
}
|
|
667
1190
|
}
|