@nexum-hub/browser 0.0.3-next.15 → 0.0.3-next.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,7 +6,13 @@ export const visualStyles = `[data-nexum-visual-layer="v1"]{position:fixed!impor
6
6
  export async function visualPage(input) {
7
7
  const slot = Symbol.for("nexum.browser.visual.v1");
8
8
  const now = Date.now();
9
- const state = (globalThis[slot] ??= { current: null, closed: new Map() });
9
+ // A managed tab remains visually identifiable after its control lease ends.
10
+ // Only the live cursor animation is owned by the lease.
11
+ const state = (globalThis[slot] ??= {
12
+ current: null,
13
+ pinned: null,
14
+ closed: new Map(),
15
+ });
10
16
  // An updated worker can encounter tombstones left by the previous build.
11
17
  // Preserve them for the maximum native dispatch window, then release them.
12
18
  if (state.closed instanceof Set)
@@ -31,27 +37,49 @@ export async function visualPage(input) {
31
37
  if (state.current?.token === token) {
32
38
  const current = state.current;
33
39
  current.stopped = true;
34
- current.layer?.remove();
35
- current.icon?.remove();
40
+ if (input.command === "release" && input.retain) {
41
+ current.layer?.setAttribute("data-nexum-visual-state", "idle");
42
+ state.pinned = {
43
+ layer: current.layer,
44
+ icon: current.icon,
45
+ x: current.x,
46
+ y: current.y,
47
+ };
48
+ } else {
49
+ current.layer?.remove();
50
+ current.icon?.remove();
51
+ }
36
52
  state.current = null;
37
53
  }
54
+ if (input.command === "clear") {
55
+ state.pinned?.layer?.remove();
56
+ state.pinned?.icon?.remove();
57
+ state.pinned = null;
58
+ }
38
59
  return { ok: true };
39
60
  }
40
- if (state.closed.has(token) || Date.now() >= input.deadline)
61
+ const restoring = input.command === "restore";
62
+ if (!restoring && (state.closed.has(token) || Date.now() >= input.deadline))
41
63
  return { ok: false, reason: "VISUAL_TARGET_RELEASED" };
64
+ // A navigation restore may not supersede a live AI action on the same tab.
65
+ if (restoring && state.current) return { ok: true, active: true };
42
66
  if (state.current?.token !== token) {
43
67
  if (state.current) {
44
68
  state.current.stopped = true;
45
69
  state.current.layer?.remove();
46
70
  state.current.icon?.remove();
47
71
  }
72
+ const pinned = state.pinned;
73
+ state.pinned = null;
48
74
  state.current = {
49
75
  token,
50
76
  sequence: -1,
51
- x: 0,
52
- y: 0,
77
+ x: pinned?.x ?? 32,
78
+ y: pinned?.y ?? 72,
53
79
  stopped: false,
54
80
  maxDeadline: inputDeadline,
81
+ layer: pinned?.layer,
82
+ icon: pinned?.icon,
55
83
  };
56
84
  }
57
85
  const current = state.current;
@@ -59,7 +87,7 @@ export async function visualPage(input) {
59
87
  const valid = () =>
60
88
  state.current === current &&
61
89
  !current.stopped &&
62
- !state.closed.has(token) &&
90
+ (restoring || !state.closed.has(token)) &&
63
91
  Date.now() < input.deadline;
64
92
  const icon = async (mode) => {
65
93
  if (!current.original)
@@ -172,19 +200,8 @@ export async function visualPage(input) {
172
200
  current.icon.href = canvas.toDataURL("image/png");
173
201
  return true;
174
202
  };
175
- if (input.command === "active" || input.command === "result") {
176
- if (input.command === "result") {
177
- current.layer?.remove();
178
- current.layer = null;
179
- }
180
- return { ok: await icon(input.command) };
181
- }
182
- if (input.command !== "move" || !valid())
183
- return { ok: false, reason: "VISUAL_UNAVAILABLE" };
184
- if (input.moveSequence <= current.sequence) return { ok: true, stale: true };
185
- current.sequence = input.moveSequence;
186
- const sequence = current.sequence;
187
- if (!current.layer) {
203
+ const ensureLayer = () => {
204
+ if (current.layer) return current.layer;
188
205
  const layer = document.createElement("div");
189
206
  layer.setAttribute("data-nexum-visual-layer", "v1");
190
207
  layer.setAttribute("aria-hidden", "true");
@@ -196,9 +213,30 @@ export async function visualPage(input) {
196
213
  document.documentElement.append(layer);
197
214
  layer.getBoundingClientRect?.();
198
215
  layer.style.opacity = "1";
216
+ layer.style.transform = `translate3d(${current.x}px,${current.y}px,0)`;
199
217
  current.layer = layer;
218
+ return layer;
219
+ };
220
+ if (input.command === "active" || input.command === "result" || restoring) {
221
+ const result = await icon(input.command === "active" ? "active" : "result");
222
+ const layer = ensureLayer();
223
+ layer.setAttribute(
224
+ "data-nexum-visual-state",
225
+ input.command === "active" ? "thinking" : "idle",
226
+ );
227
+ if (restoring) {
228
+ current.stopped = true;
229
+ state.pinned = { layer, icon: current.icon, x: current.x, y: current.y };
230
+ state.current = null;
231
+ }
232
+ return { ok: result };
200
233
  }
201
- const layer = current.layer;
234
+ if (input.command !== "move" || !valid())
235
+ return { ok: false, reason: "VISUAL_UNAVAILABLE" };
236
+ if (input.moveSequence <= current.sequence) return { ok: true, stale: true };
237
+ current.sequence = input.moveSequence;
238
+ const sequence = current.sequence;
239
+ const layer = ensureLayer();
202
240
  layer.setAttribute("data-nexum-visual-state", "moving");
203
241
  const startX = current.x,
204
242
  startY = current.y,
@@ -206,6 +244,7 @@ export async function visualPage(input) {
206
244
  dy = input.point.y - startY,
207
245
  distance = Math.hypot(dx, dy);
208
246
  const duration =
247
+ input.nativeStep === true ||
209
248
  globalThis.matchMedia?.("(prefers-reduced-motion: reduce)").matches ||
210
249
  document.visibilityState === "hidden" ||
211
250
  distance < 80
@@ -307,7 +346,6 @@ export function createVisualController(
307
346
  groups = new Map(),
308
347
  names = new Map(),
309
348
  markers = new Map(),
310
- ownActivations = new Map(),
311
349
  closedKeys = new Map(),
312
350
  styles = new Map(),
313
351
  listeners = [],
@@ -401,13 +439,14 @@ export function createVisualController(
401
439
  };
402
440
  const page = async (record, command, ctx, extra = {}) => {
403
441
  const cleanup = command === "release" || command === "clear";
442
+ const restoring = command === "restore";
404
443
  const marker = markers.get(record.tabId),
405
444
  passive =
406
445
  command === "result" &&
407
446
  marker?.documentId === record.documentId &&
408
447
  key(markerCtx(marker)) === record.token;
409
448
  const admit = () => {
410
- if (passive) {
449
+ if (passive || restoring) {
411
450
  requireValue(!stopped, "EXTENSION_OFFLINE");
412
451
  current(ctx);
413
452
  } else assertCtx(ctx);
@@ -418,7 +457,7 @@ export function createVisualController(
418
457
  const deadline = Math.min(ctx.deadline, Date.now() + 250);
419
458
  const target = { tabId: record.tabId, documentIds: [record.documentId] };
420
459
  const flow = (async () => {
421
- if (command === "move") {
460
+ if (command === "move" || restoring) {
422
461
  let style = styles.get(record.tabId);
423
462
  if (style?.documentId !== record.documentId) {
424
463
  const created = {
@@ -566,7 +605,17 @@ export function createVisualController(
566
605
  const tab = await chrome.tabs.get(record.tabId);
567
606
  assertCtx(ctx);
568
607
  record.result = false;
569
- if (markers.delete(record.tabId)) await save();
608
+ // The AI-touched identity belongs to the tab, not to this run.
609
+ markers.set(record.tabId, {
610
+ tabId: record.tabId,
611
+ browserSessionId: ctx.target.browserSessionId,
612
+ documentId: record.documentId,
613
+ target: identity(ctx),
614
+ leaseId: ctx.leaseId,
615
+ runId: ctx.runId,
616
+ operationId: ctx.operationId,
617
+ });
618
+ await save();
570
619
  // Grouping and the favicon use the same checked document but do not
571
620
  // depend on one another. Both must settle before a business action.
572
621
  const [result] = await Promise.all([
@@ -575,7 +624,6 @@ export function createVisualController(
575
624
  await group(record, ctx, tab);
576
625
  assertCtx(ctx);
577
626
  if (ctx.followSessionTab !== false && !tab.active) {
578
- ownActivations.set(record.tabId, Date.now() + 1000);
579
627
  await chrome.tabs.update(record.tabId, { active: true });
580
628
  assertCtx(ctx);
581
629
  }
@@ -638,6 +686,39 @@ export function createVisualController(
638
686
  }
639
687
  };
640
688
  const beforeAction = (ctx) => bounded(ctx, beforeActionRaw);
689
+ // tabs.new and tabs.select are visible AI actions even if no pointer moves.
690
+ // They should not require a CDP input attachment to leave a durable marker.
691
+ const trackTab = (ctx) =>
692
+ bounded(ctx, async (limited) => {
693
+ const tabId = nativeId(ctx.target.tabId);
694
+ const tab = await chrome.tabs.get(tabId);
695
+ assertCtx(limited);
696
+ const frame = await chrome.webNavigation
697
+ .getFrame({ tabId, frameId: 0 })
698
+ .catch(() => null);
699
+ assertCtx(limited);
700
+ const record = {
701
+ tabId,
702
+ token: key(ctx),
703
+ documentId: frame?.documentId ?? "pending",
704
+ };
705
+ markers.set(tabId, {
706
+ tabId,
707
+ browserSessionId: ctx.target.browserSessionId,
708
+ documentId: record.documentId,
709
+ target: identity(ctx),
710
+ leaseId: ctx.leaseId,
711
+ runId: ctx.runId,
712
+ operationId: ctx.operationId,
713
+ });
714
+ await save();
715
+ await group(record, limited, tab);
716
+ if (id(frame?.documentId)) {
717
+ const restored = await page(record, "restore", limited);
718
+ if (!restored.ok) notice(ctx, restored.reason || "VISUAL_UNAVAILABLE");
719
+ }
720
+ return { degraded: false };
721
+ });
641
722
  const beforePointer = async (ctx) => {
642
723
  assertCtx(ctx);
643
724
  requireValue(
@@ -661,6 +742,9 @@ export function createVisualController(
661
742
  const result = await page(record, "move", limited, {
662
743
  point: { x: ctx.point.x, y: ctx.point.y },
663
744
  moveSequence: ctx.moveSequence,
745
+ // The native input already traverses the interpolated path; do not
746
+ // run a second, out-of-sync decorative animation for each step.
747
+ nativeStep: true,
664
748
  });
665
749
  if (!result.ok) notice(ctx, result.reason || "VISUAL_UNAVAILABLE");
666
750
  return { degraded: status.degraded || !result.ok };
@@ -721,27 +805,20 @@ export function createVisualController(
721
805
  closedKeys.set(token, Date.now() + 1000);
722
806
  return { degraded: false };
723
807
  }
724
- let status;
725
- const marker = markers.get(record.tabId);
726
- if (record.result && marker && key(markerCtx(marker)) === token)
727
- status = await page(
728
- { ...record, documentId: marker.documentId },
729
- "result",
730
- markerCtx(marker),
731
- );
732
- else {
733
- record.result = false;
734
- status = await page(record, "release", {
735
- ...ctx,
736
- deadline: Date.now() + 250,
737
- signal: undefined,
738
- });
739
- }
808
+ // Close the input identity, but leave its passive pointer and favicon
809
+ // on any tab the AI has actually touched.
810
+ const retained = markers.has(record.tabId);
811
+ const status = await page(
812
+ record,
813
+ "release",
814
+ { ...ctx, deadline: Date.now() + 250, signal: undefined },
815
+ { retain: retained },
816
+ );
740
817
  await settled(token);
741
818
  records.delete(token);
742
819
  closedKeys.set(token, Date.now() + 1000);
743
820
  const styled = styles.get(record.tabId);
744
- if (styled?.documentId === record.documentId) {
821
+ if (!retained && styled?.documentId === record.documentId) {
745
822
  styles.delete(record.tabId);
746
823
  try {
747
824
  await chrome.scripting.removeCSS({
@@ -752,7 +829,7 @@ export function createVisualController(
752
829
  notice(ctx, "VISUAL_UNAVAILABLE");
753
830
  }
754
831
  }
755
- if (!record.result && record.groupId !== undefined)
832
+ if (!retained && record.groupId !== undefined)
756
833
  try {
757
834
  const tab = await chrome.tabs.get(record.tabId);
758
835
  if (tab.groupId === record.groupId)
@@ -820,34 +897,9 @@ export function createVisualController(
820
897
  listeners.push(() => event.removeListener(fn));
821
898
  }
822
899
  };
823
- listen(chrome.tabs.onActivated, (info) => {
824
- const until = ownActivations.get(info.tabId);
825
- ownActivations.delete(info.tabId);
826
- if (until && Date.now() <= until) return;
827
- const marker = markers.get(info.tabId);
828
- if (!marker) return;
829
- markers.delete(info.tabId);
830
- const record = records.get(key(markerCtx(marker)));
831
- if (record) record.result = false;
832
- void observe(
833
- (async () => {
834
- await page(
835
- {
836
- tabId: marker.tabId,
837
- documentId: marker.documentId,
838
- token: key(markerCtx(marker)),
839
- },
840
- "clear",
841
- markerCtx(marker),
842
- );
843
- await save();
844
- })(),
845
- key(markerCtx(marker)),
846
- ).catch(() => {});
847
- });
900
+ // User switching tabs must not erase the AI-touched indicator.
848
901
  listen(chrome.tabs.onRemoved, (tabId) => {
849
902
  markers.delete(tabId);
850
- ownActivations.delete(tabId);
851
903
  for (const [token, record] of records)
852
904
  if (record.tabId === tabId) records.delete(token);
853
905
  void save().catch(() => {});
@@ -873,7 +925,7 @@ export function createVisualController(
873
925
  documentId: marker.documentId,
874
926
  token: key(markerCtx(marker)),
875
927
  },
876
- "result",
928
+ "restore",
877
929
  markerCtx(marker),
878
930
  );
879
931
  await save();
@@ -894,30 +946,29 @@ export function createVisualController(
894
946
  id(value.operationId)
895
947
  ) {
896
948
  try {
897
- const tabId = nativeId(value.target.tabId),
898
- frame = await observe(
899
- chrome.webNavigation.getFrame({
900
- tabId,
901
- frameId: 0,
902
- }),
903
- key(markerCtx(value)),
904
- );
949
+ const tabId = nativeId(value.target.tabId);
950
+ await chrome.tabs.get(tabId); // A closed tab must never be restored.
905
951
  if (stopped) return;
906
- if (!id(frame?.documentId)) continue;
907
952
  const marker = {
908
953
  ...value,
909
954
  tabId,
910
- documentId: frame.documentId,
911
955
  target: { ...value.target },
912
956
  };
913
957
  markers.set(tabId, marker);
958
+ const frame = await observe(
959
+ chrome.webNavigation.getFrame({ tabId, frameId: 0 }),
960
+ key(markerCtx(marker)),
961
+ );
962
+ if (stopped) return;
963
+ if (!id(frame?.documentId)) continue;
964
+ marker.documentId = frame.documentId;
914
965
  await page(
915
966
  {
916
967
  tabId,
917
968
  documentId: marker.documentId,
918
969
  token: key(markerCtx(marker)),
919
970
  },
920
- "result",
971
+ "restore",
921
972
  markerCtx(marker),
922
973
  );
923
974
  } catch {}
@@ -931,18 +982,24 @@ export function createVisualController(
931
982
  closedKeys.set(token, Number.POSITIVE_INFINITY);
932
983
  disposal = (async () => {
933
984
  await settled();
934
- for (const record of records.values())
935
- if (!record.result) {
936
- await page(record, "release", { deadline: Date.now() + 250 });
937
- if (record.groupId !== undefined)
985
+ for (const record of records.values()) {
986
+ const retained = markers.has(record.tabId);
987
+ await page(
988
+ record,
989
+ "release",
990
+ { deadline: Date.now() + 250 },
991
+ { retain: retained },
992
+ );
993
+ if (!retained && record.groupId !== undefined)
938
994
  try {
939
995
  const tab = await chrome.tabs.get(record.tabId);
940
996
  if (tab.groupId === record.groupId)
941
997
  await chrome.tabs.ungroup([record.tabId]);
942
998
  } catch {}
943
- }
999
+ }
944
1000
  await settled();
945
1001
  for (const [tabId, style] of styles) {
1002
+ if (markers.has(tabId)) continue;
946
1003
  try {
947
1004
  await chrome.scripting.removeCSS({
948
1005
  target: { tabId, documentIds: [style.documentId] },
@@ -951,7 +1008,6 @@ export function createVisualController(
951
1008
  } catch {}
952
1009
  }
953
1010
  records.clear();
954
- ownActivations.clear();
955
1011
  closedKeys.clear();
956
1012
  styles.clear();
957
1013
  groups.clear();
@@ -967,6 +1023,7 @@ export function createVisualController(
967
1023
  for (const wake of [...wakeups]) wake();
968
1024
  },
969
1025
  beforeAction,
1026
+ trackTab,
970
1027
  beforePointer,
971
1028
  markDeliverable,
972
1029
  releaseTab,
package/docs/usage.md CHANGED
@@ -64,7 +64,11 @@ return output.file(file, { name: "报表.pdf" });
64
64
 
65
65
  如果动作已派发后超时、取消或断连,`effect: unknown` 表示不能证明网站副作用。先重新观察,不能自动重发点击、提交、文件选择或整个脚本。用户原生调试取消会使旧控制失效;不通过重新 get 恢复同一已交还的 run 目标。
66
66
 
67
- `selected.visibility` 控制当前浏览器内的任务标签跟随。`false` 时真实控制动作仍执行,但不切换活动标签;恢复 `true` 后,在后续真实控制动作之前切换到目标标签。只读标题、标签列表等元数据不会为了读取而抢走焦点,也不会把浏览器窗口强行带到系统前台。任务标签使用本运行自己的蓝色分组;用户已有的组不被接管。
67
+ `selected.visibility` 控制当前浏览器内的任务标签跟随。`false` 时真实控制动作仍执行,但不切换活动标签;恢复 `true` 后,在后续真实控制动作之前切换到目标标签。只读标题、标签列表等元数据不会为了读取而抢走焦点,也不会把浏览器窗口强行带到系统前台。
68
+
69
+ AI 新建、选中或实际操作过的网页标签会保留蓝色 Nexum 任务分组、修改后的标识图标以及不阻挡鼠标交互的蓝色指针。正在操作时指针随动作移动,运行结束后切换为静态标识;结束 Run、切换标签、切换页面或扩展工作线程恢复均不自动撤销标识。用户已有的分组不被接管,其标签仍可使用页面内标识和 favicon。原生标签关闭后标识与登记一同清理。控制租约照常结束,保留静态标识不代表 AI 仍然拥有控制权。受浏览器保护的内部页面不保证能够注入页面内标识。
70
+
71
+ 高层鼠标移动、点击、悬停、拖拽及滚轮定位会按距离生成带缓入缓出和细微弧度的连续路径,逐点派发原生 `Input.dispatchMouseEvent`,页面收到的是真正的中间 `mousemove`,而非只有屏幕指针动画。拖拽在中间路径保持按键按下,抵达终点后才释放。蓝色指针跟随真实事件坐标,不额外播放一条不同步的动画;显式底层 `cdp.send` 保持调用方原始参数。
68
72
 
69
73
  需要用户接管时,`await tab.requestUserTakeover("authentication")` 会交还该标签,并通过平台真实输入等待请求确认。同一运行里的其他标签继续保留。只有用户明确确认、当前运行及权限仍有效、浏览器重新观察原生标签和文档成功后,才签发新的控制身份。旧 Tab、Locator、ref、Dialog 等不会复活;继续时通过 `selected.tabs.get(tab.id)` 取得新 Tab 并重新观察。否决、取消、超时和撤权都不会恢复控制,也不证明站点登录已经成功。
70
74
 
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "package": "@nexum-hub/browser",
3
- "packageVersion": "0.0.3-next.15",
3
+ "packageVersion": "0.0.3-next.17",
4
4
  "status": "in_progress",
5
5
  "published": false,
6
6
  "browserInstallation": "test_profiles_only",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nexum-hub/browser",
3
- "version": "0.0.3-next.15",
3
+ "version": "0.0.3-next.17",
4
4
  "private": false,
5
5
  "description": "Nexum 可选浏览器客户端与受管扩展",
6
6
  "type": "module",
@@ -55,7 +55,7 @@
55
55
  "scripts": {
56
56
  "build": "tsc -p tsconfig.json && node browser-extension/build.mjs",
57
57
  "typecheck": "tsc -p tsconfig.json --noEmit",
58
- "test": "node --experimental-vm-modules --test test/*.test.mjs",
58
+ "test": "node --experimental-vm-modules --test --test-concurrency=1 test/*.test.mjs",
59
59
  "check": "pnpm typecheck && pnpm build && pnpm test"
60
60
  },
61
61
  "dependencies": {
@@ -64,7 +64,7 @@
64
64
  "ws": "8.23.0"
65
65
  },
66
66
  "peerDependencies": {
67
- "@nexum-hub/sdk": "0.0.3-next.15"
67
+ "@nexum-hub/sdk": "0.0.3-next.17"
68
68
  },
69
69
  "peerDependenciesMeta": {
70
70
  "@nexum-hub/sdk": {