dsh-inner-browser-use 0.27.0 → 0.34.28

package/read.js

@@ -257,13 +257,20 @@ function snapshot(mode, budget, max) {
     const missing = []
     for (const el of document.querySelectorAll('img,canvas')) {
       const b = box(el)
-      if (b.width < 64 || b.height < 64 || b.width * b.height < 8000) continue
+      /* A picture that failed to arrive shrinks to an error icon, so its own
+         declared size is what the page meant to show. */
+      const askW = Number(el.getAttribute?.('width')) || 0
+      const askH = Number(el.getAttribute?.('height')) || 0
+      const wide = Math.max(b.width, askW)
+      const high = Math.max(b.height, askH)
+      if (wide < 64 || high < 64 || wide * high < 8000) continue
       const s = getComputedStyle(el)
       if (s.visibility === 'hidden' || s.display === 'none') continue
       /* A picture slot that never got its picture is the commonest way a
          generated image goes missing, and it is silent by default. */
       if (el.tagName === 'IMG' && el.complete && el.naturalWidth === 0) {
-        missing.push({ width: Math.round(b.width), height: Math.round(b.height), name: String(el.getAttribute('alt') ?? '').replace(/\s+/g, ' ').trim().slice(0, 60) })
+        missing.push({ width: Math.round(wide), height: Math.round(high) })
+        missing[missing.length - 1].name = String(el.getAttribute('alt') ?? '').replace(/\s+/g, ' ').trim().slice(0, 60)
         continue
       }
       const w = el.tagName === 'CANVAS' ? el.width : el.naturalWidth || Math.round(b.width)
@@ -300,6 +307,38 @@ function snapshot(mode, budget, max) {
     }
     return said
   })()
+  /* What no amount of text will tell the reader: the screen shows something
+     that has no words in the tree. Counted here so every read can say it. */
+  const blind = (() => {
+    const out = { pictures: 0, canvases: 0, videos: 0, frames: 0 }
+    const speaks = (el) => {
+      const said = String(el.getAttribute?.('alt') ?? '').trim() || String(el.getAttribute?.('aria-label') ?? '').trim() || String(el.getAttribute?.('title') ?? '').trim()
+      if (said.length > 1) return true
+      let near = el.closest?.('figure, picture')
+      if (near) {
+        const words = String(near.querySelector?.('figcaption')?.textContent ?? '').trim()
+        if (words.length > 1) return true
+      }
+      return false
+    }
+    const shows = (el) => {
+      const b = box(el)
+      if (b.width < 48 || b.height < 48) return false
+      if (!inView(b)) return false
+      const st = getComputedStyle(el)
+      return st.visibility !== 'hidden' && st.display !== 'none' && Number(st.opacity) > 0.05
+    }
+    for (const el of document.querySelectorAll('img')) {
+      if (!shows(el) || speaks(el)) continue
+      out.pictures += 1
+    }
+    for (const el of document.querySelectorAll('canvas')) if (shows(el)) out.canvases += 1
+    for (const el of document.querySelectorAll('video')) if (shows(el)) out.videos += 1
+    /* A tracking pixel of a frame is not something the reader is missing: only
+       frames that occupy the screen are worth a sentence. */
+    for (const el of document.querySelectorAll('iframe')) if (shows(el)) out.frames += 1
+    return out
+  })()
   return JSON.stringify({
     url: location.href, // ui-copy-ignore
     title: document.title, // ui-copy-ignore
@@ -313,6 +352,7 @@ function snapshot(mode, budget, max) {
     })(),
     controls: picked,
     picture,
+    blind,
   })
 })()`
 }
@@ -378,8 +418,8 @@ export function pictures(ref, count) {
     if (el === null || el === undefined || seen.has(el)) return
     seen.add(el)
     const b = el.getBoundingClientRect()
-    const w = kind === 'canvas' ? el.width : el.naturalWidth || b.width
-    const h = kind === 'canvas' ? el.height : el.naturalHeight || b.height
+    const w = kind === 'canvas' ? el.width : kind === 'video' ? el.videoWidth || b.width : el.naturalWidth || b.width
+    const h = kind === 'canvas' ? el.height : kind === 'video' ? el.videoHeight || b.height : el.naturalHeight || b.height
     if (w < 48 || h < 48) return
     if (b.width < 8 || b.height < 8) return
     const s = getComputedStyle(el)
@@ -392,9 +432,13 @@ export function pictures(ref, count) {
     const tag = node.tagName
     if (tag === 'IMG') add(node, 'img')
     else if (tag === 'CANVAS') add(node, 'canvas')
+    /* A paused video is the frame the person is looking at; it never arrives as
+       an image file, so it has to be drawn out of the element. */
+    else if (tag === 'VIDEO' && (node.videoWidth || 0) > 0) add(node, 'video')
   }
   if (where && where.tagName === 'IMG') add(where, 'img')
   if (where && where.tagName === 'CANVAS') add(where, 'canvas')
+  if (where && where.tagName === 'VIDEO' && (where.videoWidth || 0) > 0) add(where, 'video')
   picked.sort((a, b) => b.area - a.area)
   /* A picture nobody can look at still has to be describable: how much of it is
      one colour tells a blank chart from a chart. Sampled at 32x32, which is
@@ -428,8 +472,15 @@ export function pictures(ref, count) {
     let data = ''
     let type = 'image/png'
     try {
-      if (item.kind === 'canvas') {
-        data = item.el.toDataURL('image/png')
+      if (item.kind === 'canvas' || item.kind === 'video') {
+        /* Same path for both: a video frame is copied onto a board and read as
+           a picture. Nothing here needs the video to be playing. */
+        const board = document.createElement('canvas')
+        board.width = Math.max(1, Math.round(item.w))
+        board.height = Math.max(1, Math.round(item.h))
+        const pen = board.getContext('2d')
+        pen.drawImage(item.el, 0, 0, board.width, board.height)
+        data = board.toDataURL('image/png')
       } else {
         const url = item.el.currentSrc || item.el.src || ''
         const answer = await fetch(url, { credentials: 'include' })
@@ -612,15 +663,15 @@ function routeOf(page, frame, slot) {
  * @param {any} page - sidecar page.
  * @param {string} mode - read mode.
  * @param {number} budget - characters of text to keep.
- * @returns {Promise<{text: string, controls: any[]}>} what the frames add.
+ * @returns {Promise<{text: string, controls: any[], frames: number}>} what the frames add.
  */
 async function readFrames(page, mode, budget) {
-  if (typeof page.frameList !== 'function') return { text: '', controls: [] }
+  if (typeof page.frameList !== 'function') return { text: '', controls: [], frames: 0 }
   let frames = []
   try {
     frames = await page.frameList()
   } catch (error) {
-    return { text: '', controls: [] }
+    return { text: '', controls: [], frames: 0 }
   }
   const children = frames.slice(1)
   const controls = []
@@ -642,7 +693,7 @@ async function readFrames(page, mode, budget) {
       continue
     }
   }
-  return { text, controls }
+  return { text, controls, frames: children.length }
 }
 
 /** Decode a ref into the frame it lives in and the number that frame knows. */
@@ -699,15 +750,51 @@ export async function readPage(page, options = {}) {
     scroll: read?.scroll,
     focused: read?.focused ?? null,
     controls: Array.isArray(read?.controls) ? read.controls : [],
+    blind: blindSpots(read?.blind, framed.frames),
   }
 }
 
+/**
+ * Merge what the page counted with what the frame reader managed to see.
+ * @param {any} said - counts from the page.
+ * @param {number} [read] - frames that answered.
+ * @returns {{pictures: number, canvases: number, videos: number, frames: number}} what text cannot show.
+ */
+export function blindSpots(said, read) {
+  const n = (value) => (Number.isFinite(Number(value)) ? Math.max(0, Math.round(Number(value))) : 0)
+  /* The page knows which frames take up the screen; the frame reader knows how
+     many exist, tracking pixels included. The screen is what the reader is
+     missing, so its count wins whenever the page could count at all. */
+  const frames = said !== undefined && said !== null && said.frames !== undefined ? n(said.frames) : n(read)
+  return { pictures: n(said?.pictures), canvases: n(said?.canvases), videos: n(said?.videos), frames }
+}
+
 /**
  * Resolve a ref into a point, at the moment of the action.
  * @param {any} page - sidecar page.
  * @param {number} ref - ref from a read.
  * @returns {Promise<{gone: boolean, x?: number, y?: number, name?: string, kind?: string, off?: boolean, hidden?: boolean, disabled?: boolean}>} where it is now.
  */
+/**
+ * Whether a number still means what it meant when it was read.
+ *
+ * A page can keep the same element and change its words: "Send" becomes
+ * "Stop", a row is reused for another order. Pressing the old number then
+ * does something the person never asked for.
+ * @param {any} known - the control the read handed out.
+ * @param {any} spot - what the number resolves to now.
+ * @returns {string} what changed, or '' when it is the same control.
+ */
+export function differs(known, spot) {
+  if (known === undefined || known === null || spot === undefined || spot === null) return ''
+  const before = String(known.name ?? '').replace(/\s+/g, ' ').trim().slice(0, 40)
+  const now = String(spot.name ?? '').replace(/\s+/g, ' ').trim().slice(0, 40)
+  if (before === '' || now === '' || before === now) return ''
+  const head = Math.min(12, before.length, now.length)
+  if (head >= 4 && (before.slice(0, head) === now.slice(0, head) || before.includes(now.slice(0, head)) || now.includes(before.slice(0, head)))) return ''
+  return `that number now points at "${now}" instead of "${before}" — read the page again and use the new number.`
+}
+
 export async function locate(page, ref) {
   const split = splitFrameRef(ref)
   let route = null
@@ -747,6 +834,10 @@ export function renderPage(read, options = {}) {
   const mode = String(options.mode ?? 'text')
   const maxChars = Number(options.maxChars ?? 6000)
   const lines = [read.url]
+  /* A browser error page is not a site with nothing on it: it is a site that
+     never arrived. Saying so is the only way the reader stops looking for the
+     content that would have been there. */
+  if (String(read.url).startsWith('chrome-error://')) lines.push('the page did not load — sidecar_nav to open the address again, or sidecar_open if the browser is gone')
   if (read.title !== '') lines.push(`# ${read.title}`)
   const shown = read.controls.filter((control) => control.view !== false || mode === 'full')
   const hidden = read.controls.length - shown.length
@@ -766,6 +857,16 @@ export function renderPage(read, options = {}) {
     lines.push(body)
     if (read.textChars > body.length) lines.push(`… ${read.textChars - body.length} more characters`)
   }
+  if (read.blind !== undefined && read.blind !== null) {
+    const bits = []
+    if (read.blind.pictures > 0) bits.push(`${read.blind.pictures} picture(s) with no words`)
+    if (read.blind.canvases > 0) bits.push(`${read.blind.canvases} drawing surface(s)`)
+    if (read.blind.videos > 0) bits.push(`${read.blind.videos} video(s)`)
+    if (read.blind.frames > 0) bits.push(`${read.blind.frames} embedded frame(s)`)
+    /* Only when something is actually out of reach: a line saying "nothing is
+       hidden" on every read is a line the reader learns to skip. */
+    if (bits.length > 0) lines.push(`text cannot show: ${bits.join(', ')} — sidecar_grab takes the picture, sidecar_shot shows the screen`)
+  }
   if (read.picture && Array.isArray(read.picture.missing)) {
     /* Something was meant to be here and is not: say that instead of nothing. */
     for (const gone of read.picture.missing.slice(0, 2)) {

package/review.js

@@ -0,0 +1,213 @@
+/**
+ * A review round: hand a repository address to the chat in the column, take its
+ * review back, and remember which commit the review was about. The log is what
+ * makes the next round checkable — advice given about a commit that has since
+ * moved on is stale, and saying so costs one line instead of a reread.
+ */
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+/** Where rounds are kept when the caller does not say. */
+export const ROUND_DIR = join(process.env.DSH_HOME ?? join(process.env.HOME ?? '.', '.dsh'), 'inner-browser-use', 'rounds')
+
+/** How long an answer may be, so a chatty model cannot flood the log. */
+const ANSWER_CAP = 6000
+
+/**
+ * @param {string} text - one line of configuration.
+ * @returns {string} the value, or an empty string.
+ */
+function clean(text) {
+  return String(text ?? '').replace(/\s+/g, ' ').trim()
+}
+
+/**
+ * The message to send: enough for a reviewer that has never seen this project,
+ * and no more. The address carries the code, so the message does not paste any.
+ * @param {{repo?: string, commit?: string, focus?: string}} [what] - what to review.
+ * @returns {string} the message.
+ */
+export function brief(what = {}) {
+  const repo = clean(what.repo)
+  /* What the reviewer said last time and what the code did about it: a point
+     already answered is not raised again, so a round does not re-litigate. */
+  const handled = Array.isArray(what.already) ? what.already.filter((mark) => clean(mark?.status) !== 'open') : []
+  const commit = clean(what.commit)
+  const focus = clean(what.focus)
+  const lines = ['Please review the latest code and tell me what to change.']
+  if (repo !== '') lines.push(`Repository: ${repo}`)
+  if (commit !== '') lines.push(`Look at commit ${commit} (the newest one) and the files it touches.`)
+  else lines.push('Look at the newest commit and the files it touches.')
+  if (focus !== '') lines.push(`Focus on: ${focus}`)
+  /* The reviewer cannot open the repository, so the change travels with the
+     request. Capped: a whole diff would bury the question it is attached to. */
+  const diff = String(what.diff ?? '').trim()
+  if (diff !== '') {
+    const cap = 5000
+    lines.push('The change under review, inline (a diff):')
+    lines.push(diff.length <= cap ? diff : `${diff.slice(0, cap)}\n… (trimmed)`)
+  }
+  if (handled.length > 0) {
+    lines.push('Already handled from your previous review, do not raise these again:')
+    for (const mark of handled.slice(0, 12)) {
+      const note = clean(mark.evidence) === '' ? '' : ` (${clean(mark.evidence).slice(0, 120)})`
+      lines.push(`- ${clean(mark.text).slice(0, 200)} — ${clean(mark.status)}${note}`)
+    }
+  }
+  lines.push('Answer with a short list: what is wrong, and the smallest change that fixes it. Name the file for each point.')
+  return lines.join('\n')
+}
+
+/**
+ * The suggestions in an answer, one line each: reviewers are asked for a list,
+ * and the list is what the next round acts on.
+ * @param {string} answer - what the reviewer said.
+ * @returns {string[]} the points, in the order given.
+ */
+export function points(answer) {
+  /* A page read flattens an answer into one line, so a list arrives as
+     "intro - first point - second point". Split on the markers a reviewer uses
+     for items, keep only pieces that carry a marker, and drop the echo of the
+     question when the page repeats it above the answer. */
+  const out = []
+  const seen = new Set()
+  for (const raw of String(answer ?? '').split(/\r?\n/)) {
+    const parts = raw.split(/\s+[-\u2022\u2013\u2014]\s+|\s+(?=\d+[.)]\s)/).map((part) => part.trim()).filter((part) => part !== '')
+    if (parts.length === 1 && !/^(?:[-*\u2022]|\d+[.)])\s+/.test(parts[0])) continue
+    for (let at = 0; at < parts.length; at += 1) {
+      let line = parts[at]
+      if (at === 0 && !/^(?:[-*\u2022]|\d+[.)])\s+/.test(line)) continue
+      line = line.replace(/^(?:Answer:\s*)/i, '').replace(/^(?:[-*\u2022]|\d+[.)])\s+/, '').trim()
+      if (/^(?:Please review|Repository:|Look at commit|Focus on:|Answer with)/i.test(line)) continue
+      if (line.length < 8 || line.length > 400) continue
+      const key = line.toLowerCase()
+      if (seen.has(key)) continue
+      seen.add(key)
+      out.push(line)
+    }
+  }
+  return out.slice(0, 40)
+}
+
+/**
+ * Write one round down. Append-only: a round is history, not state to rewrite.
+ * @param {{dir?: string, commit?: string, question?: string, answer?: string, tookMs?: number}} entry - the round.
+ * @returns {{file: string, round: any}} where it went and what was written.
+ */
+export function recordRound(entry = {}) {
+  const dir = clean(entry.dir) === '' ? ROUND_DIR : clean(entry.dir)
+  mkdirSync(dir, { recursive: true })
+  const file = join(dir, 'rounds.ndjson')
+  const round = {
+    at: new Date().toISOString(),
+    commit: clean(entry.commit),
+    question: clean(entry.question).slice(0, 400),
+    answer: String(entry.answer ?? '').replace(/\s+/g, ' ').trim().slice(0, ANSWER_CAP),
+    points: points(entry.answer),
+    tookMs: Math.max(0, Math.trunc(Number(entry.tookMs ?? 0)) || 0),
+  }
+  appendFileSync(file, `${JSON.stringify(round)}\n`)
+  return { file, round }
+}
+
+/**
+ * The last round played, for a round that wants to know what it was told before.
+ * @param {{dir?: string}} [where] - which log.
+ * @returns {any|null} the last round, or null when none was played.
+ */
+export function lastRound(where = {}) {
+  const dir = clean(where.dir) === '' ? ROUND_DIR : clean(where.dir)
+  const file = join(dir, 'rounds.ndjson')
+  if (!existsSync(file)) return null
+  const rows = readFileSync(file, 'utf8').split('\n').filter((row) => row.trim() !== '')
+  if (rows.length === 0) return null
+  /* Verdicts are separate rows, appended after the round they judge: find the
+     newest round, then fold in the marks written since. */
+  /** @type {any[]} */
+  const rows2 = []
+  for (const row of rows) {
+    try {
+      rows2.push(JSON.parse(row))
+    } catch {
+      /* a torn line costs its own row, not the log */
+    }
+  }
+  let at = -1
+  for (let i = rows2.length - 1; i >= 0; i -= 1) {
+    if (rows2[i]?.kind !== 'verdict') {
+      at = i
+      break
+    }
+  }
+  if (at < 0) return null
+  const round = rows2[at]
+  const latest = new Map()
+  for (const row of rows2.slice(at + 1)) {
+    if (row?.kind !== 'verdict' || clean(row.roundAt) !== clean(round.at)) continue
+    for (const mark of Array.isArray(row.marks) ? row.marks : []) if (Number.isInteger(mark?.point)) latest.set(mark.point, mark)
+  }
+  return { ...round, marks: [...latest.values()].sort((a, b) => a.point - b.point) }
+}
+
+/** What a point can be, once the code has looked at it. */
+export const MARK_STATUS = ['fixed', 'verified', 'invalid', 'open']
+
+/**
+ * Say what happened to the points of the last round. The verdict is history
+ * too: it is appended, and the next round reads it so a settled point is not
+ * raised twice.
+ * @param {{dir?: string, marks?: any[]}} entry - point numbers (1-based) and their outcome.
+ * @returns {{file: string, round: any, marks: any[]}|null} what was written, or null when there is no round.
+ */
+export function markRound(entry = {}) {
+  const dir = clean(entry.dir) === '' ? ROUND_DIR : clean(entry.dir)
+  const round = lastRound({ dir })
+  if (round === null) return null
+  const said = Array.isArray(round.points) ? round.points : []
+  /** @type {any[]} */
+  const marks = []
+  for (const raw of Array.isArray(entry.marks) ? entry.marks : []) {
+    const point = Math.trunc(Number(raw?.point ?? 0))
+    if (point < 1 || point > said.length) continue
+    const status = MARK_STATUS.includes(clean(raw?.status)) ? clean(raw.status) : 'open'
+    marks.push({ point, text: said[point - 1], status, evidence: clean(raw?.evidence).slice(0, 200), at: new Date().toISOString() })
+  }
+  if (marks.length === 0) return null
+  const file = join(dir, 'rounds.ndjson')
+  mkdirSync(dir, { recursive: true })
+  appendFileSync(file, `${JSON.stringify({ at: new Date().toISOString(), kind: 'verdict', roundAt: round.at, commit: round.commit, marks })}\n`)
+  return { file, round, marks }
+}
+
+/**
+ * One line for the model: what the reviewer said last, and whether the code has
+ * moved since. A review of an old commit is worth reading and not worth obeying.
+ * @param {{dir?: string, commit?: string}} [now] - the commit the code is at now.
+ * @returns {string} the note, or an empty string when no round was played.
+ */
+export function lastRoundNote(now = {}) {
+  const last = lastRound(now)
+  if (last === null) return ''
+  const commit = clean(now.commit)
+  const same = commit === '' || last.commit === '' || last.commit === commit
+  const points = Array.isArray(last.points) ? last.points : []
+  const head = points.length === 0 ? 'the reviewer gave no list' : `${points.length} point${points.length === 1 ? '' : 's'}`
+  const when = same ? 'about this commit' : `about ${String(last.commit).slice(0, 12)}, and the code has moved since`
+  return `Last review (${last.at.slice(0, 16).replace('T', ' ')}): ${head}, ${when}.`
+}
+
+/**
+ * The reviewer's last points as a list to act on.
+ * @param {{dir?: string}} [where] - which log.
+ * @returns {string[]} the points.
+ */
+export function lastPoints(where = {}) {
+  const last = lastRound(where)
+  return last !== null && Array.isArray(last.points) ? last.points : []
+}
+
+/** @returns {string} the directory the log lives in. */
+export function roundDir() {
+  return dirname(join(ROUND_DIR, 'rounds.ndjson'))
+}