@pointylab/dsh 0.0.0-stage → 0.4.4

package/pet/assets/cursor/make_cursor.py

@@ -0,0 +1,85 @@
+"""把 source.png（画进图里的灰白格子"假透明"）抠成真透明，裁到内容，
+再缩成光标用的 PNG 和 .cur（热点在箭头尖）。用法：python make_cursor.py"""
+import io
+import struct
+from collections import deque
+from pathlib import Path
+
+import numpy as np
+from PIL import Image
+
+HERE = Path(__file__).parent
+SIZES = (32, 48, 64)
+
+img = np.array(Image.open(HERE / 'source.png').convert('RGB')).astype(int)
+h, w, _ = img.shape
+r, g, b = img[..., 0], img[..., 1], img[..., 2]
+# 格子是浅灰和白，几乎没有颜色；人物和箭头都有深蓝描边把里面围起来
+neutral = (np.abs(r - g) < 18) & (np.abs(g - b) < 18) & (np.abs(r - b) < 18) & (r > 110)
+
+# 从四条边往里漫水，只吃连通的"无色浅色"，描边挡住，箭头里面的白不会被吃掉
+bg = np.zeros((h, w), bool)
+q = deque((y, x) for y in range(h) for x in (0, w - 1)) + deque((y, x) for x in range(w) for y in (0, h - 1))
+while q:
+    y, x = q.popleft()
+    if bg[y, x] or not neutral[y, x]:
+        continue
+    bg[y, x] = True
+    if y > 0: q.append((y - 1, x))
+    if y < h - 1: q.append((y + 1, x))
+    if x > 0: q.append((y, x - 1))
+    if x < w - 1: q.append((y, x + 1))
+
+# 被描边围住的小洞（比如呆毛圈里）漫水进不去；格子是深浅两种灰交替，按"一块无色区域里明暗差得多"认出来
+seen = bg.copy()
+for sy, sx in zip(*np.where(neutral & ~bg)):
+    if seen[sy, sx]:
+        continue
+    comp, q = [], deque([(sy, sx)])
+    seen[sy, sx] = True
+    while q:
+        y, x = q.popleft()
+        comp.append((y, x))
+        for ny, nx in ((y - 1, x), (y + 1, x), (y, x - 1), (y, x + 1)):
+            if 0 <= ny < h and 0 <= nx < w and not seen[ny, nx] and neutral[ny, nx]:
+                seen[ny, nx] = True
+                q.append((ny, nx))
+    vals = np.array([r[p] for p in comp])
+    if len(comp) > 200 and vals.std() > 15:
+        for p in comp:
+            bg[p] = True
+
+rgba = np.dstack([img, np.where(bg, 0, 255)]).astype(np.uint8)
+out = Image.fromarray(rgba, 'RGBA')
+ys, xs = np.where(~bg)
+box = (xs.min(), ys.min(), xs.max() + 1, ys.max() + 1)
+out = out.crop(box)
+
+# 箭头尖：最左边那一列里最靠上的不透明像素（箭头是整张图最左的东西）
+a = np.array(out)[..., 3]
+col = np.where(a.any(axis=0))[0][0]
+tip = (int(col), int(np.where(a[:, col])[0][0]))
+side = max(out.size)
+square = Image.new('RGBA', (side, side), (0, 0, 0, 0))
+square.paste(out, (0, 0))  # 左上对齐，箭头尖靠近左上角
+square.save(HERE / 'cursor-clean.png')
+print('cropped', out.size, 'tip in cropped', tip)
+
+
+def cur_file(img, size, hx, hy):
+    # 先存成 BMP 格式的 .ico，再把类型改成光标（2）、把热点写进 planes / bitcount 的位置。
+    # PNG 压缩的 .cur LoadImage 读不了，必须用 BMP
+    buf = io.BytesIO()
+    img.save(buf, 'ICO', sizes=[(size, size)], bitmap_format='bmp')
+    b = bytearray(buf.getvalue())
+    struct.pack_into('<H', b, 2, 2)
+    struct.pack_into('<HH', b, 6 + 4, hx, hy)
+    return bytes(b)
+
+
+for s in SIZES:
+    small = square.resize((s, s), Image.LANCZOS)
+    small.save(HERE / f'cursor-{s}.png')
+    hx, hy = round(tip[0] * s / side), round(tip[1] * s / side)
+    (HERE / f'cursor-{s}.cur').write_bytes(cur_file(small, s, hx, hy))
+    print(f'{s}px hotspot', hx, hy)

package/pet/boot.js

@@ -0,0 +1,6 @@
+// 生成的文件（tools/pack-plugin.mjs）：让 require('koffi') / require('ws') 找到 vendor 里的副本
+const path = require('node:path')
+const Module = require('node:module')
+process.env.NODE_PATH = [path.join(__dirname, 'vendor'), process.env.NODE_PATH].filter(Boolean).join(path.delimiter)
+Module._initPaths()
+require('./app/main.js')

package/pet/extension/background.js

@@ -0,0 +1,252 @@
+// 和本机 pointy 保持一条 WebSocket：Pointy按快捷键时来要选区，右键菜单时主动发过去。
+// 只在这两种时候读页面，平时不碰任何标签页。
+// Pointy默认端口；storage 里存了 port 就用那个（Pointy改过 POINTY_PORT 时）
+const DEFAULT_PORT = 47321
+const petUrl = async () => `ws://127.0.0.1:${(await chrome.storage.local.get('port')).port ?? DEFAULT_PORT}`
+const BROWSER = navigator.userAgent.includes('Edg/') ? 'edge' : 'chrome'
+const KEEPALIVE_MS = 20_000
+
+let ws
+let authed = false
+let keepalive
+
+const enc = new TextEncoder()
+const hex = (buf) => [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('')
+
+async function hmac(token, data) {
+  const key = await crypto.subtle.importKey('raw', enc.encode(token), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign'])
+  return hex(await crypto.subtle.sign('HMAC', key, enc.encode(data)))
+}
+
+const randomHex = () => hex(crypto.getRandomValues(new Uint8Array(16)))
+
+async function connect() {
+  if (ws && ws.readyState <= WebSocket.OPEN) return
+  const { id, token } = await chrome.storage.local.get(['id', 'token'])
+  // 还没配对：自己去问 Pointy，Pointy 弹窗让用户点"允许"
+  if (!id || !token) { if (Date.now() >= pairQuietUntil) await pair(false); return }
+  const nonce = randomHex()
+  const sock = new WebSocket(await petUrl())
+  ws = sock
+  authed = false
+  sock.onmessage = async (ev) => {
+    const msg = JSON.parse(ev.data)
+    if (msg.type === 'challenge') {
+      sock.send(JSON.stringify({ type: 'hello', role: 'extension', id, browser: BROWSER, version: chrome.runtime.getManifest().version, nonce, proof: await hmac(token, `ext|${msg.nonce}`) }))
+    } else if (msg.type === 'hello-ok') {
+      if (msg.proof !== await hmac(token, `pet|${nonce}`)) return sock.close()
+      authed = true
+      clearInterval(keepalive)
+      // 有流量的 WebSocket 会让 MV3 service worker 一直活着
+      keepalive = setInterval(() => sock.readyState === WebSocket.OPEN && sock.send('{"type":"ping"}'), KEEPALIVE_MS)
+    } else if (authed && msg.type === 'grab') {
+      const result = await grabActiveTab()
+      sock.send(JSON.stringify({ type: 'grab-result', reqId: msg.reqId, ...result }))
+    } else if (authed && msg.type === 'pick-abort') {
+      if (pickTab) chrome.tabs.sendMessage(pickTab.id, { type: 'pick-abort' }).catch(() => {})
+      pickReq = undefined
+    } else if (authed && msg.type === 'focus-tab') {
+      focusTab(msg.origin)
+    } else if (authed && msg.type === 'reload') {
+      // 磁盘上的扩展比正在跑的新（Pointy 看 manifest 版本号判断），自己重载一次，不用去 chrome://extensions 点刷新
+      chrome.runtime.reload()
+    } else if (authed && msg.type === 'pick-release') {
+      const tabId = heldTabs.get(msg.reqId)
+      heldTabs.delete(msg.reqId)
+      if (tabId !== undefined) chrome.tabs.sendMessage(tabId, { type: 'pick-release', reqId: msg.reqId }).catch(() => {})
+    } else if (authed && msg.type === 'pick-start') {
+      const ok = await startPick(msg.cursor, msg.urlHint)
+      if (!ok) sock.send(JSON.stringify({ type: 'pick-unavailable', reqId: msg.reqId }))
+      else {
+        pickReq = msg.reqId
+        // 回报注入到了哪个标签页，Pointy记日志，方便查"高亮框跑哪去了"
+        sock.send(JSON.stringify({ type: 'pick-started', reqId: msg.reqId, tabTitle: pickTab?.title, tabUrl: pickTab?.url }))
+      }
+    }
+  }
+  sock.onclose = () => {
+    if (ws === sock) { ws = undefined; authed = false }
+    clearInterval(keepalive)
+  }
+}
+
+/** 在页面里执行：取选区和它前后的文字。密码框直接拒绝。 */
+function extractSelection() {
+  const active = document.activeElement
+  if (active && active.tagName === 'INPUT' && active.type === 'password') return { blocked: '焦点在密码框里' }
+  const editable = active && (active.tagName === 'TEXTAREA' || (active.tagName === 'INPUT' && /^(text|search|url|email|)$/.test(active.type)))
+  if (editable && active.selectionStart !== active.selectionEnd) {
+    const { value, selectionStart: s, selectionEnd: e } = active
+    return { text: value.slice(s, e).slice(0, 20000), before: value.slice(Math.max(0, s - 400), s).trim(), after: value.slice(e, e + 400).trim() }
+  }
+  const sel = window.getSelection()
+  const text = sel ? sel.toString() : ''
+  let before = ''
+  let after = ''
+  if (text && sel.rangeCount) {
+    let block = sel.getRangeAt(0).commonAncestorContainer
+    if (block.nodeType !== 1) block = block.parentElement
+    block = block.closest('p,li,td,blockquote,pre,article,section,div') || block
+    const full = block.innerText || ''
+    const needle = text.trim().slice(0, 60)
+    const at = needle ? full.indexOf(needle) : -1
+    if (at >= 0) {
+      before = full.slice(Math.max(0, at - 400), at).trim()
+      after = full.slice(at + text.trim().length, at + text.trim().length + 400).trim()
+    }
+    if (!before) before = (block.previousElementSibling?.innerText || '').slice(-400).trim()
+    if (!after) after = (block.nextElementSibling?.innerText || '').slice(0, 400).trim()
+  }
+  return { text: text.slice(0, 20000), before, after }
+}
+
+async function grabActiveTab(tab) {
+  tab ??= (await chrome.tabs.query({ active: true, lastFocusedWindow: true }))[0]
+  if (!tab) return {}
+  const base = { url: tab.url, pageTitle: tab.title, browser: BROWSER }
+  // chrome:// 之类的页面不让注入脚本，只带网址和标题
+  if (!/^(https?|file):/.test(tab.url ?? '')) return base
+  try {
+    const frames = await chrome.scripting.executeScript({ target: { tabId: tab.id, allFrames: true }, func: extractSelection })
+    const blocked = frames.find(f => f.result?.blocked)
+    if (blocked) return { ...base, blocked: blocked.result.blocked }
+    const hit = frames.find(f => f.result?.text) ?? frames.find(f => f.frameId === 0)
+    return { ...base, ...(hit?.result ?? {}) }
+  } catch {
+    return base
+  }
+}
+
+let pickReq
+let pickTab
+const heldTabs = new Map() // reqId -> 还留着高亮框的标签页 id，卡片关掉时Pointy发 pick-release 过来
+
+/** 往当前标签页注入指认脚本；chrome:// 之类的页面注入不了就返回 false，Pointy会改用系统层面的指认 */
+async function startPick(cursor, urlHint) {
+  // 正常是晃鼠标时眼前的那个标签页；Pointy也可以指定网址（测试时用，不必把窗口切到前台）
+  const [tab] = urlHint
+    ? (await chrome.tabs.query({})).filter(t => t.url === urlHint || t.url?.startsWith(urlHint))
+    : await chrome.tabs.query({ active: true, lastFocusedWindow: true })
+  if (!tab || !/^(https?|file):/.test(tab.url ?? '')) return false
+  try {
+    await chrome.scripting.executeScript({ target: { tabId: tab.id }, func: (c) => { window.__pointyStart = c }, args: [cursor ?? null] })
+    await chrome.scripting.executeScript({ target: { tabId: tab.id }, files: ['pick.js'] })
+    pickTab = tab
+    return true
+  } catch {
+    return false
+  }
+}
+
+/** 截当前可见区域，按元素位置裁出来；返回 data URL */
+async function cropElement(tab, rect, viewport) {
+  const shot = await chrome.tabs.captureVisibleTab(tab.windowId, { format: 'png' })
+  const bitmap = await createImageBitmap(await (await fetch(shot)).blob())
+  const scale = bitmap.width / viewport.width
+  const pad = 8
+  const sx = Math.max(0, Math.floor((rect.x - pad) * scale))
+  const sy = Math.max(0, Math.floor((rect.y - pad) * scale))
+  const sw = Math.min(bitmap.width - sx, Math.ceil((rect.width + pad * 2) * scale))
+  const sh = Math.min(bitmap.height - sy, Math.ceil((rect.height + pad * 2) * scale))
+  if (sw < 4 || sh < 4) return undefined
+  // 太大的缩到 1600 宽以内，够模型看清
+  const k = Math.min(1, 1600 / sw)
+  const canvas = new OffscreenCanvas(Math.round(sw * k), Math.round(sh * k))
+  canvas.getContext('2d').drawImage(bitmap, sx, sy, sw, sh, 0, 0, canvas.width, canvas.height)
+  const blob = await canvas.convertToBlob({ type: 'image/jpeg', quality: 0.85 })
+  const bytes = new Uint8Array(await blob.arrayBuffer())
+  let bin = ''
+  for (let i = 0; i < bytes.length; i += 0x8000) bin += String.fromCharCode(...bytes.subarray(i, i + 0x8000))
+  return `data:image/jpeg;base64,${btoa(bin)}`
+}
+
+async function onPickResult(msg, sender) {
+  const reqId = pickReq
+  pickReq = undefined
+  if (!authed || reqId === undefined) return
+  if (msg.cancelled || !msg.result) return ws.send(JSON.stringify({ type: 'pick-cancel', reqId }))
+  const tab = sender.tab ?? pickTab
+  // 图片、视频、画布这类光看文字不知道是什么的，截那一块一起发（只在内存里传，不存盘）
+  const r = msg.result
+  const visual = r.kind === 'image' || r.kind === 'video' || r.tag === 'canvas' || ((r.text ?? '').trim().length < 30 && r.images?.length > 0)
+  const image = visual ? await cropElement(tab, r.rect, r.viewport).catch(() => undefined) : undefined
+  // 截完了，让页面把高亮框重新显示出来，卡片开着的时候一直留着
+  heldTabs.set(reqId, tab.id)
+  chrome.tabs.sendMessage(tab.id, { type: 'pick-hold', reqId }).catch(() => {})
+  ws.send(JSON.stringify({ type: 'picked', reqId, url: tab.url, pageTitle: tab.title, browser: BROWSER, element: r, image }))
+}
+
+// web 版 DSH：Pointy 卡片上点"在 DSH 中打开"时，把开着它的标签页切到前面
+async function focusTab(origin) {
+  if (typeof origin !== 'string' || !/^https?:\/\/[^/]+$/.test(origin)) return
+  const tabs = await chrome.tabs.query({})
+  const tab = tabs.find((t) => t.url?.startsWith(origin + '/'))
+  // 没开着就不新开：web 版首次打开要带 token 的地址，扩展拿不到
+  if (!tab) return
+  await chrome.tabs.update(tab.id, { active: true })
+  await chrome.windows.update(tab.windowId, { focused: true })
+}
+
+// 用户点了"不允许"或 Pointy 正忙着问别的：过一阵再自动问；弹窗里手动点"连接"不受这个限制
+const PAIR_QUIET_MS = 10 * 60_000
+let pairQuietUntil = 0
+let pairing
+
+function pair(manual) {
+  if (pairing) return pairing
+  pairing = (async () => {
+    const url = await petUrl()
+    return new Promise((resolve) => {
+      const sock = new WebSocket(url)
+      let timer
+      let settled = false
+      const finish = (result) => { if (settled) return; settled = true; clearTimeout(timer); pairing = undefined; resolve(result); try { sock.close() } catch {} }
+      // Pointy 在等用户点的时候每 10 秒发一条 pair-wait；超过这个没动静就当它不在了
+      const arm = () => { clearTimeout(timer); timer = setTimeout(() => finish({ ok: false, error: 'Pointy 没响应' }), 20_000) }
+      arm()
+      sock.onerror = () => finish({ ok: false, error: '连不上 Pointy：确认 DSH 开着、Pointy 插件已启用' })
+      sock.onclose = () => finish({ ok: false, error: '连接断了' })
+      sock.onmessage = async (ev) => {
+        const msg = JSON.parse(ev.data)
+        arm()
+        if (msg.type === 'challenge') sock.send(JSON.stringify({ type: 'pair-request', browser: BROWSER, manual }))
+        else if (msg.type === 'paired') {
+          await chrome.storage.local.set({ id: msg.id, token: msg.token })
+          finish({ ok: true })
+          connect()
+        } else if (msg.type === 'pair-denied' || msg.type === 'pair-busy') {
+          pairQuietUntil = Date.now() + PAIR_QUIET_MS
+          finish({ ok: false, error: msg.type === 'pair-busy' ? 'Pointy 正在问别的，稍后再试' : '在 Pointy 那边没有允许' })
+        }
+      }
+    })
+  })()
+  return pairing
+}
+
+chrome.runtime.onInstalled.addListener(() => {
+  chrome.contextMenus.create({ id: 'ask-pet', title: '问 Pointy：“%s”', contexts: ['selection'] })
+  chrome.contextMenus.create({ id: 'ask-pet-page', title: '问 Pointy 这个页面', contexts: ['page'] })
+})
+
+chrome.contextMenus.onClicked.addListener(async (info, tab) => {
+  await connect()
+  const result = await grabActiveTab(tab)
+  if (!result.text && info.selectionText) result.text = info.selectionText
+  if (authed) ws.send(JSON.stringify({ type: 'selection', ...result }))
+})
+
+chrome.runtime.onMessage.addListener((msg, sender, reply) => {
+  if (msg.type === 'pick-result') { onPickResult(msg, sender); return }
+  if (msg.type === 'pair') { pair(true).then(reply); return true }
+  if (msg.type === 'status') {
+    chrome.storage.local.get(['id']).then(({ id }) => reply({ paired: !!id, connected: authed }))
+    return true
+  }
+  if (msg.type === 'unpair') { chrome.storage.local.remove(['id', 'token']).then(() => { ws?.close(); reply({ ok: true }) }); return true }
+})
+
+chrome.alarms.create('reconnect', { periodInMinutes: 1 })
+chrome.alarms.onAlarm.addListener(() => connect())
+chrome.runtime.onStartup.addListener(() => connect())
+connect()

package/pet/extension/manifest.json

@@ -0,0 +1,12 @@
+{
+  "manifest_version": 3,
+  "name": "Pointy 指哪问哪",
+  "version": "0.4.0",
+  "description": "晃一下鼠标指认网页上的内容，或右键「问 Pointy」，把指到的内容、网址和标题交给本机的 Pointy。只在被叫到时读页面。",
+  "permissions": ["storage", "scripting", "contextMenus", "alarms"],
+  "host_permissions": ["<all_urls>"],
+  "background": { "service_worker": "background.js" },
+  "action": { "default_title": "Pointy", "default_popup": "popup.html" },
+  "icons": { "128": "icon.png" },
+  "web_accessible_resources": [{ "resources": ["cursor-32.png", "cursor-48.png", "cursor-64.png"], "matches": ["<all_urls>"] }]
+}