dsh-annotator 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/session.js CHANGED
@@ -11,6 +11,7 @@ import path from 'node:path';
11
11
  import { CdpWs } from './cdp.js';
12
12
  import { BINDING_NAME, OVERLAY_SCRIPT } from './overlay.js';
13
13
  import { decideAutoPick, nextUserTurnedOff } from './shared/auto-pick.js';
14
+ import { annotationMarkdown } from './shared/annotation-markdown.js';
14
15
  import { classifyBindingAction } from './shared/binding-action.js';
15
16
  import {
16
17
  ADOPT_RUNNING_ENV,
@@ -29,6 +30,25 @@ import { externalCandidates, parseVersionReply } from './shared/cdp-endpoint.js'
29
30
  import { EGO_AGENT_SPACE_NAME, isEgoBrowserCommand, isEgoOnboardingTarget, listeningPortsByPid } from './shared/ego.js';
30
31
  import { advancePendingConsume, popLastPick } from './shared/pick-actions.js';
31
32
  import { isDuplicatePick, pickSignature } from './shared/pick-dedupe.js';
33
+ import {
34
+ clearPairTo,
35
+ delegationSummary,
36
+ firstUnpairedTarget,
37
+ pairAllUnpaired,
38
+ pairIdsOf,
39
+ pairTo,
40
+ setPairIds,
41
+ unpair,
42
+ unpairAll,
43
+ normalizeSpec,
44
+ pageHudInfo,
45
+ pageKeyOf,
46
+ pagePickCounts,
47
+ pickRoleOf,
48
+ refLabelOf,
49
+ refLabelsOf,
50
+ SPEC_DIMENSIONS,
51
+ } from './shared/page-roles.js';
32
52
  import { cropPng, readPngSize } from './shared/png.js';
33
53
  import { isGuiPage } from './shared/gui-page.js';
34
54
  import { scratchDir } from './shared/scratch-dir.js';
@@ -177,9 +197,18 @@ export class VisualPicker {
177
197
  activeTitle: '',
178
198
  mode: 'off',
179
199
  picks: [],
200
+ // 页面角色表(目标 0..1 / 参考 0..5 / 中性)。pick 通过 pageKey 继承它,
201
+ // 因此改一次页面角色就能一次改写那一页所有标注,不必逐条点。
202
+ pages: [],
180
203
  // Monotonic token bumped when the in-page HUD asks the panel to insert
181
204
  // everything; the panel compares tokens so one request inserts exactly once.
182
205
  pendingConsume: 0,
206
+ // 武装落位(面板「+ 新增改动 / + 添加参照」经 /arm 下发):记录下一处标注
207
+ // 应落的栏与目标。host 收到拾取时据此判断「点到的是不是已标注的元素」,
208
+ // 是就复用那一条(改角色、按需配对),而不是再造一张同元素的重复标注卡。
209
+ arming: null,
210
+ // 最近一次「复用已标注」通知:列表不会增长,面板靠它清武装色条并提示。
211
+ reuseNotice: null,
183
212
  tabs: [],
184
213
  // 宿主自带沙箱时 Chrome 的浏览器进程会立刻 SIGTRAP;这时我们会带
185
214
  // --no-sandbox 重试一次,面板据此提示「本机浏览器在无沙箱模式下运行」。
@@ -205,6 +234,13 @@ export class VisualPicker {
205
234
  this.windowKept = false;
206
235
  this.windowProfileDir = null;
207
236
  this.attached = new Map();
237
+ // 每个标签页的标注层是否真的在里面。新标签页只靠 Target.targetCreated 挂一次,
238
+ // attach 或注入失败过一次就永久没有 HUD(用户看到的是「第二个标签页没有 HUD,
239
+ // 快捷键也没反应」),所以这里记状态,由 refreshTabs 定期补挂与补注入。
240
+ this.overlayOk = new Map();
241
+ this.preloadDone = new Set();
242
+ this.overlayTriedAt = new Map();
243
+ this.attachTriedAt = new Map();
208
244
  this.activeTargetId = null;
209
245
  // 调试空间 = Chromium BrowserContext(与 ego lite 的 task space 同一原语):
210
246
  // 同一进程、同一 profile,但 cookie / storage / 标签页各自独立。'' = 默认空间,
@@ -274,9 +310,22 @@ export class VisualPicker {
274
310
  tabs: filterTabsBySpace(this.state.tabs, this.activeSpace, this.defaultContextId).map((tab) => ({
275
311
  ...tab,
276
312
  active: tab.targetId === this.activeTargetId,
313
+ // false = 这一页里没有标注层(host 会继续重试),面板据此标一个警示点,
314
+ // 不然用户只能看到「HUD 没出来」而不明白为什么。
315
+ hud: this.overlayOk.get(tab.targetId) !== false,
277
316
  })),
317
+ // 页面条:只是标注出处(哪一页、几处、R 几)。角色不在页面上,面板
318
+ // 也不用再自己推导一遍——picks[].role 是唯一权威。
319
+ pages: this.pageList(),
320
+ // 当前活动页的 key:面板用它高亮页面条里的那一枚 chip。
321
+ activePageKey: pageKeyOf(this.state.activeUrl),
322
+ // 参考集合的确定性委托摘要(不调模型);用户可编辑后再插入。
323
+ summary: delegationSummary({ picks: this.state.picks, activeTitle: this.state.activeTitle }),
278
324
  picks: this.state.picks.map((pick) => this._publicPick(pick)),
279
325
  pendingConsume: this.state.pendingConsume,
326
+ // 武装落位与复用通知:面板据此同步色条状态、提示「点的是旧标注不是新标了一条」。
327
+ arming: this.state.arming,
328
+ reuseNotice: this.state.reuseNotice,
280
329
  // Inserted picks leave the list but stay resolvable: their chips expand at
281
330
  // submit time, which can be minutes after the insert.
282
331
  archived: this.retired.length,
@@ -308,6 +357,9 @@ export class VisualPicker {
308
357
 
309
358
  /** The one public shape of a pick, shared by /state, /pick and the agent tool. */
310
359
  _publicPick(pick) {
360
+ const pageKey = pick.pageKey || pageKeyOf(pick.url);
361
+ const role = pickRoleOf(pick);
362
+ const spec = normalizeSpec(pick.spec);
311
363
  return {
312
364
  id: pick.id,
313
365
  seq: pick.seq,
@@ -323,9 +375,81 @@ export class VisualPicker {
323
375
  ts: pick.ts,
324
376
  shotPath: pick.shotPath ?? null,
325
377
  shotUrl: pick.shotPath ? `/api/visual-annotator/shot?id=${encodeURIComponent(pick.id)}` : null,
378
+ // 参考模式:角色、参考维度、「本次包含」。角色是标注自己的字段
379
+ // (高于页面),R 编号按参照标注首次出现的页序派生,见 shared/page-roles.js。
380
+ role,
381
+ roleSource: pick.roleSource === 'user' || pick.roleSource === 'landing' ? pick.roleSource : 'auto',
382
+ pageKey,
383
+ pageTitle: pick.pageTitle ?? pick.title ?? '',
384
+ refLabel: role === 'reference' ? refLabelOf(this.state.picks, pageKey) || 'R' : '',
385
+ spec,
386
+ included: pick.included !== false,
387
+ referencedAt: pick.referencedAt ?? null,
388
+ resend: role === 'reference' && pick.included === true && Boolean(pick.referencedAt),
389
+ pageClosed: this._pageClosed(pageKey),
390
+ // 目标标注参照的全部参考:面板靠它画「这条改动下面挂着哪几条参考」。
391
+ pairIds: pairIdsOf(pick),
392
+ pairId: pick.pairId == null ? null : String(pick.pairId),
326
393
  };
327
394
  }
328
395
 
396
+ /**
397
+ * Panel-facing page list: every recorded page plus the active one. Pages carry
398
+ * provenance only (which URL, how many picks, which R-number); the role lives on
399
+ * each pick, so one page can legitimately be both 改动 and 参照 at once.
400
+ */
401
+ pageList() {
402
+ const activeKey = pageKeyOf(this.state.activeUrl);
403
+ const labels = refLabelsOf(this.state.picks);
404
+ // 计数走 shared 里那个带单测的纯函数,别再手写一遍 bucket 累加:两份口径一旦
405
+ // 分叉,症状就是标签条写 3 条、HUD 写 2 处,用户不知道该信哪个。
406
+ const shape = (page) => ({
407
+ key: page.key,
408
+ url: page.url,
409
+ title: page.title,
410
+ label: labels.get(page.key) || '',
411
+ active: page.key === activeKey,
412
+ closed: this._pageClosed(page.key),
413
+ fullShot: page.fullShotPath ? `/api/visual-annotator/pageshot?key=${encodeURIComponent(page.key)}` : null,
414
+ picks: pagePickCounts(this.state.picks, page.key),
415
+ });
416
+ const pages = this.state.pages.map(shape);
417
+ // 当前页还没标注过时也在清单里:面板据此显示「本页 0 处」,标签条与出处
418
+ // 用的是同一份地址,不需要用户再单独指定一次页面。
419
+ if (activeKey && !pages.some((page) => page.key === activeKey)) {
420
+ pages.push(shape({ key: activeKey, url: this.state.activeUrl, title: this.state.activeTitle }));
421
+ }
422
+ return pages;
423
+ }
424
+
425
+ /**
426
+ * Remember a page we have seen, so the panel can offer 设为目标/设为参考 before
427
+ * anything is annotated there. Only the active page and pages with picks are
428
+ * recorded: an untouched background tab is not part of the task yet.
429
+ */
430
+ _touchPage(url, title) {
431
+ const key = pageKeyOf(url);
432
+ if (!key) return null;
433
+ let page = this.state.pages.find((item) => item.key === key);
434
+ if (!page) {
435
+ page = { key, url: String(url ?? ''), title: String(title ?? '') };
436
+ this.state.pages.push(page);
437
+ return page;
438
+ }
439
+ if (url) page.url = String(url);
440
+ if (title) page.title = String(title);
441
+ return page;
442
+ }
443
+
444
+ /** A page counts as closed when no open tab carries its key any more. */
445
+ _pageClosed(key) {
446
+ if (!key) return false;
447
+ const open = (this.state.tabs ?? []).some((tab) => pageKeyOf(tab.url) === key);
448
+ if (open) return false;
449
+ // 浏览器没连上时 tabs 为空:那不是「页面关了」,是没在跑。
450
+ return this.state.tabs.length > 0;
451
+ }
452
+
329
453
  /**
330
454
  * Resolve one pick by id, live or already inserted. A chip inserted in round 1 is
331
455
  * still submitted later, so "not in the list" must not mean "gone".
@@ -1012,8 +1136,7 @@ export class VisualPicker {
1012
1136
  await this.cdp.send('Page.enable', {}, sessionId).catch(() => {});
1013
1137
  await this.cdp.send('Runtime.enable', {}, sessionId).catch(() => {});
1014
1138
  await this.cdp.send('Runtime.addBinding', { name: BINDING_NAME }, sessionId).catch(() => {});
1015
- await this.cdp.send('Page.addScriptToEvaluateOnNewDocument', { source: OVERLAY_SCRIPT }, sessionId).catch(() => {});
1016
- await this._evaluate(OVERLAY_SCRIPT, sessionId).catch(() => {});
1139
+ await this._ensureOverlay(targetId, sessionId, { preload: true });
1017
1140
  await this._applyMode(sessionId).catch(() => {});
1018
1141
  await this._syncActiveTarget(targetId, sessionId);
1019
1142
  // 挂上时页面可能仍在导航(标题还是空的)。等它落定后各对一次真实信息;
@@ -1051,6 +1174,7 @@ export class VisualPicker {
1051
1174
  this.activeTargetId = targetId;
1052
1175
  this.state.activeUrl = url;
1053
1176
  this.state.activeTitle = parsed.title ?? '';
1177
+ this._touchPage(url, parsed.title ?? '');
1054
1178
  this._emit();
1055
1179
  } catch {
1056
1180
  /* target may be navigating */
@@ -1074,9 +1198,15 @@ export class VisualPicker {
1074
1198
  else if (action === 'mode') void this.setMode(payload.mode);
1075
1199
  else if (action === 'undo') this.undoLastPick();
1076
1200
  else if (action === 'consume') this._requestConsume();
1201
+ // HUD 上的落点开关:与面板按钮同一条语义,只是入口在页面里。
1202
+ else if (action === 'landing') void this._setLandingFromPage(payload.slot);
1077
1203
  return;
1078
1204
  }
1079
1205
  if (method === 'Page.loadEventFired') {
1206
+ // 导航会换掉整个文档:preload 正常时无需做什么,preload 没挂上(新标签页
1207
+ // 第一次 attach 恰好失败)时这里是唯一能把它救回来的时机。
1208
+ const targetId = this._targetOfSession(message.sessionId);
1209
+ if (targetId) void this._ensureOverlay(targetId, message.sessionId, { preload: true }).catch(() => {});
1080
1210
  void this._applyMode(message.sessionId);
1081
1211
  // 标题是页面加载后才有的:浏览器级 Target.targetInfoChanged 对新开/后台页
1082
1212
  // 并不保证送达(面板侧每秒 /state 轮询时靠 refreshTabs 兜住,但 host 自己没
@@ -1103,6 +1233,8 @@ export class VisualPicker {
1103
1233
  const targetId = message.params?.targetId;
1104
1234
  this.state.tabs = (this.state.tabs ?? []).filter((tab) => tab.targetId !== targetId);
1105
1235
  this.attached.delete(targetId);
1236
+ this.overlayOk.delete(targetId);
1237
+ this.preloadDone.delete(targetId);
1106
1238
  if (this.activeTargetId === targetId) {
1107
1239
  this.activeTargetId = null;
1108
1240
  const rest = this.state.tabs ?? [];
@@ -1156,6 +1288,21 @@ export class VisualPicker {
1156
1288
  const before = fingerprint(this.state.tabs);
1157
1289
  for (const info of targetInfos) this._upsertTab(info, { emit: false });
1158
1290
  if (before !== fingerprint(this.state.tabs)) this._emit();
1291
+ // 新标签页原本只有 Target.targetCreated 一条命:事件漏了、或那一次 attach /
1292
+ // 注入失败,这个页就永远没有 HUD。轮询(面板每秒 /state)时补挂与补注入。
1293
+ for (const info of targetInfos) {
1294
+ if (!this.attached.has(info.targetId)) {
1295
+ if (now - (this.attachTriedAt.get(info.targetId) ?? 0) < 1500) continue;
1296
+ this.attachTriedAt.set(info.targetId, now);
1297
+ void this._attach(info.targetId).catch(() => {});
1298
+ continue;
1299
+ }
1300
+ if (this.overlayOk.get(info.targetId) !== false) continue;
1301
+ if (now - (this.overlayTriedAt.get(info.targetId) ?? 0) < 1500) continue;
1302
+ this.overlayTriedAt.set(info.targetId, now);
1303
+ const sid = this.attached.get(info.targetId);
1304
+ void this._ensureOverlay(info.targetId, sid, { preload: true }).catch(() => {});
1305
+ }
1159
1306
  }
1160
1307
 
1161
1308
  /** Pages that already exist when we connect never fire targetCreated. */
@@ -1178,17 +1325,142 @@ export class VisualPicker {
1178
1325
  return this.cdp.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: false }, sessionId);
1179
1326
  }
1180
1327
 
1181
- async _applyMode(sessionId) {
1328
+ /**
1329
+ * 页面自己知道自己是不是参考页:这里按「这个标签页的 URL」算角色。
1330
+ * 用 activeUrl 算的话,后台那个参考页会被推成目标页(文案「怎么改」、也没有蓝色
1331
+ * 参考徽标),用户在同一套 HUD 上就会去改参考页。
1332
+ */
1333
+ async _applyMode(sessionId = null, url = null) {
1182
1334
  if (!this.cdp) return;
1183
- // sync() 是 host -> page 的唯一同步口:mode 加上当前已标注数一起推。
1184
- const info = JSON.stringify({ mode: this.state.mode, count: this.state.picks.length });
1185
- const expression = `window.__DSH_VP__ && window.__DSH_VP__.sync(${info})`;
1186
- if (sessionId) await this._evaluate(expression, sessionId).catch(() => {});
1187
- else for (const sid of this.attached.values()) await this._evaluate(expression, sid).catch(() => {});
1335
+ // 提示词只算一次:整份标注是同一份,逐个标签页重复生成只是白烧 CPU。
1336
+ const prompt = this.toMarkdown();
1337
+ const entries = sessionId ? [[this._targetOfSession(sessionId), sessionId]] : [...this.attached.entries()];
1338
+ for (const [targetId, sid] of entries) {
1339
+ if (!sid) continue;
1340
+ const info = JSON.stringify(
1341
+ pageHudInfo({
1342
+ url: url ?? this._urlOfTarget(targetId),
1343
+ picks: this.state.picks,
1344
+ // 落点决定「下一处会进哪一栏」,不是页面的多数票:HUD 说的是
1345
+ // 用户现在点下去会发生什么。
1346
+ landing: this.state.arming?.slot ?? null,
1347
+ mode: this.state.mode,
1348
+ specs: SPEC_DIMENSIONS,
1349
+ prompt,
1350
+ }),
1351
+ );
1352
+ await this._evaluate(`window.__DSH_VP__ && window.__DSH_VP__.sync(${info})`, sid).catch(() => {});
1353
+ }
1354
+ }
1355
+
1356
+ /** Which tab a session belongs to (reverse of this.attached). */
1357
+ _targetOfSession(sessionId) {
1358
+ for (const [targetId, sid] of this.attached) if (sid === sessionId) return targetId;
1359
+ return null;
1360
+ }
1361
+
1362
+ /** A tab's own address; falls back to the active page so unknown tabs keep old behaviour. */
1363
+ _urlOfTarget(targetId) {
1364
+ const tab = (this.state.tabs ?? []).find((item) => item.targetId === targetId);
1365
+ return tab?.url || this.state.activeUrl;
1366
+ }
1367
+
1368
+ /**
1369
+ * Make sure this tab really has the overlay: probe, and if it is missing inject and
1370
+ * probe again. A swallowed failure here used to leave the tab permanently dead —
1371
+ * no HUD, no ⌥A — with nothing in the panel saying why.
1372
+ */
1373
+ async _ensureOverlay(targetId, sessionId, { preload = false } = {}) {
1374
+ if (!sessionId) return false;
1375
+ const probe = () =>
1376
+ this._evaluate(
1377
+ "JSON.stringify({vp: typeof window.__DSH_VP__, hud: !!document.getElementById('__dsh-vp-root')})",
1378
+ sessionId,
1379
+ );
1380
+ try {
1381
+ if (preload && !this.preloadDone.has(targetId)) {
1382
+ await this.cdp.send('Page.addScriptToEvaluateOnNewDocument', { source: OVERLAY_SCRIPT }, sessionId);
1383
+ this.preloadDone.add(targetId);
1384
+ }
1385
+ const first = await probe();
1386
+ const seen = JSON.parse(first?.result?.value ?? '{}');
1387
+ if (seen.vp === 'object') {
1388
+ // 脚本在、挂载点没了(页面自己清了 body)时补一次挂载即可,别整段重注入。
1389
+ if (!seen.hud)
1390
+ await this._evaluate(
1391
+ 'window.__DSH_VP__ && window.__DSH_VP__.reattach && window.__DSH_VP__.reattach()',
1392
+ sessionId,
1393
+ );
1394
+ this._markOverlay(targetId, true);
1395
+ return true;
1396
+ }
1397
+ await this._evaluate(OVERLAY_SCRIPT, sessionId);
1398
+ const again = await probe();
1399
+ const ok = JSON.parse(again?.result?.value ?? '{}').vp === 'object';
1400
+ this._markOverlay(targetId, ok);
1401
+ return ok;
1402
+ } catch {
1403
+ this._markOverlay(targetId, false);
1404
+ return false;
1405
+ }
1406
+ }
1407
+
1408
+ _markOverlay(targetId, ok) {
1409
+ if (!targetId) return;
1410
+ const before = this.overlayOk.get(targetId);
1411
+ this.overlayOk.set(targetId, ok);
1412
+ if (ok && before === false) this.log?.('info', `标签页 ${targetId} 的标注层已补上`);
1413
+ if (!ok && before !== false) this.log?.('warn', `标签页 ${targetId} 注入标注层失败,面板会继续重试`);
1414
+ }
1415
+
1416
+ /**
1417
+ * 当前落点:面板点过「+ 新增改动 / + 添加参照」后一直有效,直到再点一次取消
1418
+ * (或 10 分钟没动静自己过期)。落点就是「下一处标注进哪一栏」,与页面无关。
1419
+ */
1420
+ _landing() {
1421
+ const arming = this.state.arming;
1422
+ if (!arming) return null;
1423
+ if (Date.now() - (Number(arming.ts) || 0) > 10 * 60 * 1000) {
1424
+ // 过期也要广播:直接清 state 的话,面板的落点色条与 HUD 的颜色会一直亮到
1425
+ // 下一次快照,用户点下去才发现落点其实已经回落到「改动」。
1426
+ this.setArming(null);
1427
+ return null;
1428
+ }
1429
+ return arming;
1430
+ }
1431
+
1432
+ /**
1433
+ * 刚落的一处参照归到哪条改动:面板上点「+ 添加参照」的那张改动卡优先
1434
+ * (pairId 只生效一次),否则配给第一条还没配对的改动。边标边配,不用再点第二下。
1435
+ */
1436
+ _pairFreshReference(role, pickId, landing) {
1437
+ if (role !== 'reference') return;
1438
+ const wanted = String(landing?.pairId ?? '');
1439
+ if (wanted) {
1440
+ this.setPair(wanted, pickId, { move: true });
1441
+ return;
1442
+ }
1443
+ const partner = firstUnpairedTarget(this.state.picks);
1444
+ if (partner && String(partner.id) !== String(pickId)) pairTo(this.state.picks, partner.id, pickId);
1445
+ }
1446
+
1447
+ /** 落点粘住(连续标多处都进同一栏);只有「配给某条改动」是一次性的。 */
1448
+ _settleLanding() {
1449
+ if (!this.state.arming) return;
1450
+ this.state.arming = { ...this.state.arming, pairId: '', ts: Date.now() };
1188
1451
  }
1189
1452
 
1190
1453
  async _recordPick(payload, sessionId) {
1191
1454
  const id = 'p' + ++this.seq;
1455
+ const url = payload.url ?? this.state.activeUrl;
1456
+ const title = payload.title ?? this.state.activeTitle;
1457
+ const page = this._touchPage(url, title);
1458
+ const pageKey = pageKeyOf(url);
1459
+ // 角色在这里当场定:面板上的落点(「+ 新增改动 / + 添加参照」)说了算,
1460
+ // 没有落点就是改动。页面只是出处,不再决定标注去哪一栏——按页面推、再由
1461
+ // 面板事后纠正,页签一多必然对不上,用户看到的就是「选的页签和标注不符」。
1462
+ const landing = this._landing();
1463
+ const role = landing ? landing.slot : 'target';
1192
1464
  const pick = {
1193
1465
  id,
1194
1466
  seq: payload.seq ?? this.seq,
@@ -1199,11 +1471,22 @@ export class VisualPicker {
1199
1471
  html: payload.html ?? '',
1200
1472
  rect: payload.rect ?? null,
1201
1473
  pageRect: payload.pageRect ?? null,
1202
- url: payload.url ?? this.state.activeUrl,
1203
- title: payload.title ?? this.state.activeTitle,
1474
+ url,
1475
+ title,
1204
1476
  ts: Date.now(),
1205
1477
  shotPath: null,
1206
1478
  sessionId,
1479
+ role,
1480
+ roleSource: landing ? 'landing' : 'auto',
1481
+ pageKey,
1482
+ pageTitle: page?.title || title || '',
1483
+ spec: normalizeSpec(payload.spec),
1484
+ included: true,
1485
+ referencedAt: null,
1486
+ // 目标标注参照哪几条参考标注(参考对照视图里挂在同一条目标下面)。
1487
+ // pairId 是 pairIds 的第一条,agent 工具契约只认它。
1488
+ pairIds: [],
1489
+ pairId: null,
1207
1490
  };
1208
1491
  // 页面脚本一次提交只应发一份,但旧版/多份注入脚本会让同一次提交投递多次,
1209
1492
  // 列表里就成了「三条一模一样的标注」。判重规则见 lib/shared/pick-dedupe.js。
@@ -1212,7 +1495,28 @@ export class VisualPicker {
1212
1495
  if (isDuplicatePick({ signature: this.lastPickSignature, at: this.lastPickAt }, signature, now)) return;
1213
1496
  this.lastPickSignature = signature;
1214
1497
  this.lastPickAt = now;
1498
+ // 落点落在已经标过的元素上:复用那一条(改角色、按需配对),不再造一张
1499
+ // 同元素的重复标注卡。「改动也可以参照已标注的元素」——重复标一遍只会多出
1500
+ // 一张一模一样的卡,不是用户要的。同元素判定=同一页 + 同一选择器。
1501
+ if (landing) {
1502
+ const existing = this.state.picks.find(
1503
+ (candidate) =>
1504
+ (candidate.pageKey || pageKeyOf(candidate.url)) === pageKey &&
1505
+ String(candidate.selector ?? '') === String(payload.selector ?? ''),
1506
+ );
1507
+ if (existing) {
1508
+ if (existing.role !== role) this.setPickRole(existing.id, role);
1509
+ this._pairFreshReference(role, existing.id, landing);
1510
+ if (String(payload.comment ?? '').trim()) this.setComment(existing.id, payload.comment);
1511
+ this._settleLanding();
1512
+ this.state.reuseNotice = { pickId: existing.id, role, ts: Date.now() };
1513
+ this._emit();
1514
+ return this.snapshot();
1515
+ }
1516
+ }
1215
1517
  this.state.picks.push(pick);
1518
+ this._pairFreshReference(role, pick.id, landing);
1519
+ this._settleLanding();
1216
1520
  if (payload.title) this.state.activeTitle = payload.title;
1217
1521
  if (payload.url) this.state.activeUrl = payload.url;
1218
1522
  this._emit();
@@ -1508,6 +1812,8 @@ export class VisualPicker {
1508
1812
  const index = this.state.picks.findIndex((pick) => pick.id === id);
1509
1813
  if (index < 0) return this.snapshot();
1510
1814
  const [removed] = this.state.picks.splice(index, 1);
1815
+ // 删掉的可能是一条参考:指向它的目标会留下悬空 pairId,一并解开。
1816
+ clearPairTo(this.state.picks, removed.id);
1511
1817
  await this._afterRemoval([removed], false);
1512
1818
  return this.snapshot();
1513
1819
  }
@@ -1518,6 +1824,7 @@ export class VisualPicker {
1518
1824
  const popped = popLastPick(this.state.picks);
1519
1825
  if (!popped) return this.snapshot();
1520
1826
  this.state.picks = popped.remaining;
1827
+ clearPairTo(this.state.picks, popped.removed.id);
1521
1828
  await this._afterRemoval([popped.removed], false);
1522
1829
  return this.snapshot();
1523
1830
  }
@@ -1539,16 +1846,30 @@ export class VisualPicker {
1539
1846
  */
1540
1847
  async consumePicks(ids) {
1541
1848
  const wanted = new Set((Array.isArray(ids) ? ids : []).map(String));
1542
- if (!wanted.size) return { ...this.snapshot(), consumed: 0 };
1849
+ if (!wanted.size) return { ...this.snapshot(), consumed: 0, referenced: 0 };
1543
1850
  const removed = [];
1544
- this.state.picks = this.state.picks.filter((pick) => {
1545
- if (!wanted.has(String(pick.id))) return true;
1851
+ // 参考标注插入后不离开列表:它们是可复用的只读规范,下次还可能再发。
1852
+ // 记录插入前的 included/referencedAt 快照,撤销一次插入要连这两个字段一起还原。
1853
+ const referenced = [];
1854
+ const keep = [];
1855
+ for (const pick of this.state.picks) {
1856
+ if (!wanted.has(String(pick.id))) {
1857
+ keep.push(pick);
1858
+ continue;
1859
+ }
1860
+ if (pickRoleOf(pick) === 'reference') {
1861
+ referenced.push({ pick, included: pick.included !== false, referencedAt: pick.referencedAt ?? null });
1862
+ pick.included = false;
1863
+ pick.referencedAt = Date.now();
1864
+ keep.push(pick);
1865
+ continue;
1866
+ }
1546
1867
  removed.push(pick);
1547
- return false;
1548
- });
1549
- if (!removed.length) return { ...this.snapshot(), consumed: 0 };
1550
- await this._afterRemoval(removed, true);
1551
- return { ...this.snapshot(), consumed: removed.length };
1868
+ }
1869
+ this.state.picks = keep;
1870
+ if (!removed.length && !referenced.length) return { ...this.snapshot(), consumed: 0, referenced: 0 };
1871
+ await this._afterRemoval(removed, true, { referenced });
1872
+ return { ...this.snapshot(), consumed: removed.length, referenced: referenced.length };
1552
1873
  }
1553
1874
 
1554
1875
  /**
@@ -1570,13 +1891,14 @@ export class VisualPicker {
1570
1891
  * 截图本身不删;否则(单条移除/撤销)截图留在槽里供 restore 用。被顶掉的旧槽
1571
1892
  * 截图无论如何都删——它已不可恢复。
1572
1893
  */
1573
- async _afterRemoval(removed, archive) {
1894
+ async _afterRemoval(removed, archive, extra = {}) {
1574
1895
  if (this.lastRemoved) for (const pick of this.lastRemoved.picks) this._deleteShot(pick);
1896
+ const referenced = extra.referenced ?? [];
1575
1897
  if (archive) {
1576
1898
  this._archive(removed);
1577
- this.lastRemoved = { picks: removed };
1899
+ this.lastRemoved = { picks: removed, referenced };
1578
1900
  } else {
1579
- this.lastRemoved = removed.length ? { picks: removed } : null;
1901
+ this.lastRemoved = removed.length ? { picks: removed, referenced } : null;
1580
1902
  }
1581
1903
  this._emit();
1582
1904
  await this._applyMode(null);
@@ -1587,6 +1909,14 @@ export class VisualPicker {
1587
1909
  const entry = this.lastRemoved;
1588
1910
  if (!entry) return { ...this.snapshot(), restored: 0 };
1589
1911
  this.lastRemoved = null;
1912
+ // 参考插入只是改了 included/referencedAt,标注本身还在列表里:先还原这两个
1913
+ // 字段,撤销一次「3 参考 + 2 目标」的插入才是完整复原。
1914
+ for (const item of entry.referenced ?? []) {
1915
+ const pick = this.state.picks.find((candidate) => candidate.id === item.pick.id);
1916
+ if (!pick) continue;
1917
+ pick.included = item.included;
1918
+ pick.referencedAt = item.referencedAt;
1919
+ }
1590
1920
  for (const pick of entry.picks) {
1591
1921
  let index = this.state.picks.length;
1592
1922
  for (let i = 0; i < this.state.picks.length; i++) {
@@ -1605,7 +1935,7 @@ export class VisualPicker {
1605
1935
  }
1606
1936
  this._emit();
1607
1937
  void this._applyMode(null);
1608
- return { ...this.snapshot(), restored: entry.picks.length };
1938
+ return { ...this.snapshot(), restored: entry.picks.length + (entry.referenced?.length ?? 0) };
1609
1939
  }
1610
1940
 
1611
1941
  _deleteShot(pick) {
@@ -1624,47 +1954,217 @@ export class VisualPicker {
1624
1954
  return this.snapshot();
1625
1955
  }
1626
1956
 
1957
+ /**
1958
+ * 换一条标注的角色:拖到另一栏、或落点落在一条已标过的元素上时走这里。
1959
+ * roleSource='user' 记录「这是用户亲手定的」,之后换落点不会动它。
1960
+ */
1961
+ setPickRole(id, role) {
1962
+ const pick = this.state.picks.find((item) => item.id === String(id ?? ''));
1963
+ if (!pick || (role !== 'target' && role !== 'reference'))
1964
+ return { ok: false, error: 'bad-pick-role', ...this.snapshot() };
1965
+ // 换角色等于换一侧:它自己挂在谁下面的关系先断,别人挂在它下面的关系也要断,
1966
+ // 否则会留下一条「目标参照一条目标」或指向不存在参考的悬空配对。
1967
+ if (pick.role !== role) setPairIds(pick, []);
1968
+ pick.role = role;
1969
+ pick.roleSource = 'user';
1970
+ if (role === 'target') pick.referencedAt = null;
1971
+ if (role === 'target') clearPairTo(this.state.picks, pick.id);
1972
+ this._emit();
1973
+ return { ok: true, ...this.snapshot() };
1974
+ }
1975
+
1976
+ /**
1977
+ * 配对:把一条参考标注挂到一条目标标注上(refId 传空=这条目标的参考全解开)。
1978
+ * 改一处可以参照多处(版式照 A、配色照 B),所以这里是追加而不是覆盖;
1979
+ * 已经挂着的重复挂不报错,拖拽来回放不会产生重复行。
1980
+ */
1981
+ setPair(targetId, refId, options = {}) {
1982
+ const target = this.state.picks.find((item) => item.id === String(targetId ?? ''));
1983
+ if (!target) return { ok: false, error: 'pick-not-found', ...this.snapshot() };
1984
+ if (refId === null || refId === undefined || refId === '') {
1985
+ unpairAll(this.state.picks, target.id);
1986
+ this._emit();
1987
+ return { ok: true, ...this.snapshot() };
1988
+ }
1989
+ const ref = this.state.picks.find((item) => item.id === String(refId));
1990
+ if (!ref || pickRoleOf(ref) !== 'reference')
1991
+ return { ok: false, error: 'bad-pair', message: '只有归到参照栏的标注才能挂到改动上', ...this.snapshot() };
1992
+ if (String(ref.id) === String(target.id))
1993
+ return { ok: false, error: 'bad-pair', message: '一条标注不能参照它自己', ...this.snapshot() };
1994
+ // move:这条参考只归这一条目标。落点标注时用——host 已经把它自动配给了
1995
+ // 「第一条没配的目标」,用户点的那一行才是真正的落点。
1996
+ if (options?.move) {
1997
+ for (const other of this.state.picks)
1998
+ if (String(other.id) !== String(target.id)) unpair(this.state.picks, other.id, ref.id);
1999
+ }
2000
+ pairTo(this.state.picks, target.id, ref.id);
2001
+ this._emit();
2002
+ return { ok: true, ...this.snapshot() };
2003
+ }
2004
+
2005
+ /** 只解开一条:右栏参考卡上的小 ×(其余参考还挂在这条目标上)。 */
2006
+ unpair(targetId, refId) {
2007
+ const target = this.state.picks.find((item) => item.id === String(targetId ?? ''));
2008
+ if (!target) return { ok: false, error: 'pick-not-found', ...this.snapshot() };
2009
+ unpair(this.state.picks, target.id, refId);
2010
+ this._emit();
2011
+ return { ok: true, ...this.snapshot() };
2012
+ }
2013
+
2014
+ /** 一条目标现在参照了哪几条参考(面板拖拽后刷新用)。 */
2015
+ pairStats() {
2016
+ const counts = new Map();
2017
+ for (const pick of this.state.picks) for (const id of pairIdsOf(pick)) counts.set(id, (counts.get(id) ?? 0) + 1);
2018
+ return counts;
2019
+ }
2020
+
2021
+ /**
2022
+ * 一条参考套用到所有还没配对的目标:同一条规范(配色、圆角)常常要落到好几处,
2023
+ * 逐条点下拉太慢。已经配过的目标不覆盖——那可能是用户特意挑的另一条参考。
2024
+ */
2025
+ pairAll(refId) {
2026
+ const ref = this.state.picks.find((item) => item.id === String(refId ?? ''));
2027
+ if (!ref || pickRoleOf(ref) !== 'reference')
2028
+ return { ok: false, error: 'bad-pair', message: '只有归到参照栏的标注才能挂到改动上', ...this.snapshot() };
2029
+ const touched = pairAllUnpaired(this.state.picks, ref.id);
2030
+ this._emit();
2031
+ return { ok: true, paired: touched.length, ...this.snapshot() };
2032
+ }
2033
+
2034
+ /**
2035
+ * 落点由面板经 /arm 下发(slot=target/reference,传 null 取消)。host 有两处用它:
2036
+ * 新标注当场归哪一栏,以及拾取落在已标过的元素上时该复用哪一栏。落点是粘性的,
2037
+ * 所以「连续标五处参照」只需要点一次按钮;pairId 只对下一处生效。
2038
+ */
2039
+ setArming(arming) {
2040
+ if (!arming || (arming.slot !== 'target' && arming.slot !== 'reference')) {
2041
+ this.state.arming = null;
2042
+ void this._applyMode(null);
2043
+ this._emit();
2044
+ return;
2045
+ }
2046
+ this.state.arming = { slot: arming.slot, pairId: arming.pairId ? String(arming.pairId) : '', ts: Date.now() };
2047
+ // 落点一变就得推给页面:HUD 的颜色与措辞说的就是「下一处进哪一栏」,不推的话
2048
+ // 用户在参照落点上仍看到橙色的「改动」,标下去就以为落错了栏。
2049
+ void this._applyMode(null);
2050
+ this._emit();
2051
+ }
2052
+
2053
+ /**
2054
+ * 页面 HUD 上的一键切换(⌥R / 点「改动·参照」开关)。换的只是下一处落点,
2055
+ * 顺手进标注模式:按这个键的人下一步就是点元素,不该再让他回面板开一次。
2056
+ */
2057
+ async _setLandingFromPage(slot) {
2058
+ this.setArming({ slot });
2059
+ if (this.state.mode !== 'pick') await this.setMode('pick');
2060
+ }
2061
+
2062
+ /** 「本次包含」:只有勾着的标注会被插入;参考插入后自动取消,重勾=重发。 */
2063
+ setPickInclude(id, included) {
2064
+ const pick = this.state.picks.find((item) => item.id === String(id ?? ''));
2065
+ if (!pick) return { ok: false, error: 'pick-not-found', ...this.snapshot() };
2066
+ pick.included = Boolean(included);
2067
+ this._emit();
2068
+ return { ok: true, ...this.snapshot() };
2069
+ }
2070
+
2071
+ /** 参考维度:布局/间距/配色/字体/字号/圆角/交互/文案,可多选。 */
2072
+ setPickSpec(id, spec) {
2073
+ const pick = this.state.picks.find((item) => item.id === String(id ?? ''));
2074
+ if (!pick) return { ok: false, error: 'pick-not-found', ...this.snapshot() };
2075
+ pick.spec = normalizeSpec(spec);
2076
+ this._emit();
2077
+ return { ok: true, ...this.snapshot() };
2078
+ }
2079
+
1627
2080
  /**
1628
2081
  * One-click clear. Picks move to the archive (chips still resolve) and stay
1629
2082
  * restorable as one batch, screenshots included, until the undo slot is
1630
2083
  * replaced or the window closes. Nothing to clear leaves the archive alone.
1631
2084
  */
1632
- async clear() {
1633
- const cleared = this.state.picks.slice();
2085
+ async clear(scope = 'all') {
2086
+ // 「清空」分三个口:目标(插入即消费,清空是补救)/ 参考(复用集清理)/ 全部。
2087
+ const wanted = scope === 'target' || scope === 'reference' ? scope : 'all';
2088
+ const match = (pick) => wanted === 'all' || pickRoleOf(pick) === wanted;
2089
+ const cleared = this.state.picks.filter(match);
1634
2090
  if (!cleared.length) return { ...this.snapshot(), cleared: 0 };
1635
- this.state.picks = [];
2091
+ this.state.picks = this.state.picks.filter((pick) => !match(pick));
2092
+ for (const gone of cleared) clearPairTo(this.state.picks, gone.id);
1636
2093
  await this._afterRemoval(cleared, true);
1637
2094
  return { ...this.snapshot(), cleared: cleared.length };
1638
2095
  }
1639
2096
 
1640
2097
  shotFile(id) {
1641
- const pick = this.state.picks.find((item) => item.id === id);
2098
+ const pick = this.state.picks.find((item) => item.id === id) ?? this.retired.find((item) => item.id === id);
1642
2099
  return pick?.shotPath && fs.existsSync(pick.shotPath) ? pick.shotPath : null;
1643
2100
  }
1644
2101
 
1645
- /** Markdown the panel injects into the composer / the agent tool returns. */
1646
- toMarkdown() {
1647
- const picks = this.state.picks;
1648
- if (!picks.length) return '(当前还没有页面元素标注)';
1649
- const lines = [];
1650
- lines.push(
1651
- `页面元素标注(${picks.length} 处,来源 ${this.state.browser ?? 'browser'}:${this.state.activeUrl || '未知页面'})`,
2102
+ /**
2103
+ * 整页截图,按需触发(对比层的「整页」视图)。元素裁剪图只截视口,元素滚动出
2104
+ * 视口就没有图;整页图要撑视口,用户会看到一次重排,所以绝不自动做,只由面板
2105
+ * 按下「整页」时调一次,失败也让面板显示「重试」而不是默默空着。
2106
+ */
2107
+ async capturePageShot(key) {
2108
+ if (!this.running) throw new Error('浏览器未运行');
2109
+ const wanted = String(key ?? '') || pageKeyOf(this.state.activeUrl);
2110
+ const page = this.state.pages.find((item) => item.key === wanted);
2111
+ if (!page) throw new Error('页面不在任务里');
2112
+ let targetId = null;
2113
+ for (const tid of this.attached.keys()) {
2114
+ const tab = (this.state.tabs ?? []).find((item) => item.targetId === tid);
2115
+ if (tab && pageKeyOf(tab.url) === wanted) {
2116
+ targetId = tid;
2117
+ break;
2118
+ }
2119
+ }
2120
+ if (!targetId) throw new Error('页面已关闭');
2121
+ let sessionId = this.attached.get(targetId);
2122
+ if (!sessionId) sessionId = await this._attach(targetId).catch(() => null);
2123
+ if (!sessionId) throw new Error('无法连接该页面');
2124
+ const metrics = await this.cdp.send('Page.getLayoutMetrics', {}, sessionId).catch(() => null);
2125
+ const size = metrics?.cssContentSize ?? metrics?.contentSize;
2126
+ if (!size || !(size.width > 0) || !(size.height > 0)) throw new Error('拿不到页面尺寸');
2127
+ const result = await this.cdp.send(
2128
+ 'Page.captureScreenshot',
2129
+ {
2130
+ format: 'png',
2131
+ captureBeyondViewport: true,
2132
+ fromSurface: true,
2133
+ clip: { x: 0, y: 0, width: Math.ceil(size.width), height: Math.ceil(size.height), scale: 1 },
2134
+ },
2135
+ sessionId,
1652
2136
  );
1653
- picks.forEach((pick, index) => {
1654
- lines.push('');
1655
- lines.push(
1656
- `${index + 1}. \`${pick.selector || pick.tag}\` — ${pick.tag}${pick.text ? `:${pick.text.replace(/\s+/g, ' ').slice(0, 80)}` : ''}`,
1657
- );
1658
- if (pick.url) lines.push(` 页面:${pick.url}`);
1659
- if (pick.pageRect)
1660
- lines.push(
1661
- ` 位置:x=${Math.round(pick.pageRect.x)} y=${Math.round(pick.pageRect.y)} ${Math.round(pick.pageRect.width)}×${Math.round(pick.pageRect.height)}`,
1662
- );
1663
- if (pick.comment) lines.push(` 评论:${pick.comment}`);
1664
- if (pick.html) lines.push(' HTML:' + ' `' + pick.html.replace(/\s+/g, ' ').slice(0, 300) + '`');
1665
- if (pick.shotPath) lines.push(` 截图:${pick.shotPath}`);
2137
+ if (!result?.data) throw new Error('整页截图失败');
2138
+ const dir = path.join(this.activeBaseDir ?? this.baseDir, 'page-shots');
2139
+ fs.mkdirSync(dir, { recursive: true });
2140
+ // 文件名由 pageKey 派生:同一页重复截只覆盖这一张,不在临时目录里堆垃圾。
2141
+ const name = Buffer.from(wanted).toString('base64url').slice(0, 48);
2142
+ const file = path.join(dir, name + '.png');
2143
+ fs.writeFileSync(file, Buffer.from(result.data, 'base64'));
2144
+ page.fullShotPath = file;
2145
+ this._emit();
2146
+ return {
2147
+ ok: true,
2148
+ key: wanted,
2149
+ fullShot: `/api/visual-annotator/pageshot?key=${encodeURIComponent(wanted)}`,
2150
+ };
2151
+ }
2152
+
2153
+ pageShotFile(key) {
2154
+ const page = this.state.pages.find((item) => item.key === String(key ?? ''));
2155
+ return page?.fullShotPath && fs.existsSync(page.fullShotPath) ? page.fullShotPath : null;
2156
+ }
2157
+
2158
+ /**
2159
+ * Markdown the panel injects into the composer / the agent tool returns.
2160
+ * 与工具输出共用 lib/shared/annotation-markdown.js:两处各写一份时,面板那份曾经
2161
+ * 漏掉「参照:」行,粘到别的 agent 里的提示词就丢了改动↔参照的对照关系。
2162
+ */
2163
+ toMarkdown() {
2164
+ return annotationMarkdown(this.state.picks, {
2165
+ empty: '(当前还没有页面元素标注)',
2166
+ source: `${this.state.browser ?? 'browser'}:${this.state.activeUrl || '未知页面'}`,
1666
2167
  });
1667
- return lines.join('\n');
1668
2168
  }
1669
2169
 
1670
2170
  /** Delete the browser profile when policy allows; shared/reused profiles survive. */
@@ -1699,6 +2199,8 @@ export class VisualPicker {
1699
2199
  }
1700
2200
  for (const pick of this.retired) this._deleteShot(pick);
1701
2201
  this.retired = [];
2202
+ // 页面角色表跟着 picks 走:关窗不清空标注(重开同一个窗口还看得见),
2203
+ // 所以角色与 pageKey 也一起留着,下次连上照旧解析。
1702
2204
  // 与运行期一致走 sync() 唯一同步口:先置 off,再由 _applyMode 广播
1703
2205
  // {mode,count};旧版 overlay 没有 sync,由 REV 接管机制兜底。
1704
2206
  this.state.mode = 'off';