dsh-inner-browser-use 0.34.28 → 0.34.95

package/scripts/check-flash-local.mjs

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+/**
+ * The Flash path, measured without the network: a page served from this
+ * process, its movie from the repo's own fixture, and the same bridge a site
+ * would be driven through. A site being slow or rate-limiting for a minute is
+ * not a plugin regression, and this check can tell the two apart.
+ *
+ *   node scripts/check-flash-local.mjs
+ *
+ * Asserts the movie painted, frames reach a viewer, and a press, a release and
+ * a key each reach the plugin in well under the time a person would notice.
+ * Own profile, own state directory, closed afterwards; nothing appears on the
+ * desktop.
+ */
+import { createServer } from 'node:http'
+import { createHash } from 'node:crypto'
+import { mkdtempSync, readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const here = dirname(fileURLToPath(import.meta.url))
+/* The looping movie, because a still one paints once and says nothing about
+   whether frames follow a moving picture. */
+const movie = readFileSync(join(here, '..', 'test', 'fixtures', 'loop.swf'))
+const page = `<!doctype html>
+<html><head><meta charset="utf-8"><title>Flash fixture</title></head>
+<body style="margin:0;background:#101820">
+<embed src="/tiny.swf" type="application/x-shockwave-flash" width="400" height="300">
+</body></html>`
+
+const server = createServer((request, reply) => {
+  if (request.url === '/tiny.swf') {
+    reply.writeHead(200, { 'content-type': 'application/x-shockwave-flash', 'content-length': movie.length })
+    reply.end(movie)
+    return
+  }
+  reply.writeHead(200, { 'content-type': 'text/html; charset=utf-8' })
+  reply.end(page)
+})
+await new Promise((ready) => server.listen(0, '127.0.0.1', ready))
+const url = `http://127.0.0.1:${server.address().port}/`
+
+process.env.DSH_INNER_BROWSER_STATE_DIR = mkdtempSync(join(tmpdir(), 'dsh-flash-local-'))
+const { Sidecar } = await import('../sidecar.js')
+
+/** @param {number} ms - how long. */
+const wait = (ms) => new Promise((settle) => setTimeout(settle, ms))
+const hash = (buffer) => createHash('sha1').update(Buffer.from(buffer ?? '')).digest('hex').slice(0, 12)
+
+const sidecar = new Sidecar({ profile: 'flash-local', headless: false, flash: true })
+let ok = false
+try {
+  await sidecar.launch('about:blank')
+  const seen = []
+  const detach = await sidecar.watch((frame) => {
+    seen.push({ at: Date.now(), shape: hash(frame.jpeg), kb: Math.round(String(frame.jpeg ?? '').length * 0.75 / 1024) })
+  })
+  const navigatedAt = Date.now()
+  await sidecar.page.navigate(url).catch(() => {})
+  for (let i = 0; i < 60 && seen.length === 0; i += 1) await wait(250)
+  const firstFrameMs = Date.now() - navigatedAt
+  await wait(3000)
+  detach()
+  const gaps = seen.slice(1).map((one, index) => one.at - seen[index].at).sort((a, b) => a - b)
+  const fps = gaps.length === 0 ? 0 : Math.round((1000 / (gaps[Math.floor(gaps.length / 2)] || 1000)) * 10) / 10
+  const shapes = new Set(seen.map((one) => one.shape))
+  const info = await fetch(`http://127.0.0.1:${sidecar.page.flashBridge}/info`, { signal: AbortSignal.timeout(1500) }).then((one) => one.json()).catch(() => ({}))
+  const pressedAt = Date.now()
+  await sidecar.page.mouse('mousePressed', { x: 640, y: 420 })
+  const pressMs = Date.now() - pressedAt
+  const releasedAt = Date.now()
+  await sidecar.page.mouse('mouseReleased', { x: 640, y: 420 })
+  const releaseMs = Date.now() - releasedAt
+  const keyAt = Date.now()
+  await sidecar.page.key('keyDown', { key: 'a', text: 'a' })
+  const keyMs = Date.now() - keyAt
+  await sidecar.page.key('keyUp', { key: 'a' })
+  await wait(600)
+  /* The same endpoint a click uses, asked directly: the plugin took the input
+     rather than a path timing out on the way there. */
+  const fedReply = await fetch(`http://127.0.0.1:${sidecar.page.flashBridge}/input`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({ type: 'mouseMove', x: 200, y: 150, button: 'left', clickCount: 0 }),
+    signal: AbortSignal.timeout(1500),
+  }).catch((error) => ({ status: 0, text: () => Promise.resolve(String(error?.message ?? error)) }))
+  const fedStatus = fedReply.status
+  const fedText = await fedReply.text().catch(() => '')
+  const fed = { ok: fedText.includes('"ok":true') }
+  const frameKB = seen.length === 0 ? 0 : Math.round(seen.reduce((sum, one) => sum + one.kb, 0) / seen.length)
+  /* The fixture movie is not a game: what is asserted here is that the movie
+     painted, that frames reached a viewer, and that input arrived without a
+     detour through the debugging protocol. Frame rate on moving pictures is
+     measured by check-flash-mirror against a real site. */
+  ok = Number(info.painted ?? 0) > 1 && seen.length >= 3 && shapes.size >= 2 && firstFrameMs < 1500 && pressMs < 40 && releaseMs < 40 && keyMs < 40 && fed.ok === true && frameKB > 0
+  console.log(JSON.stringify({ url, firstFrameMs, frames: seen.length, fps, shapes: shapes.size, frameKB, painted: info.painted ?? 0, pressMs, releaseMs, keyMs, fedStatus, fedText: fedText.slice(0, 80), fed: fed.ok === true, ok }))
+} catch (error) {
+  console.log(JSON.stringify({ ok: false, error: String(error?.message ?? error).slice(0, 160) }))
+} finally {
+  await sidecar.shutdown().catch(() => {})
+  server.close()
+}
+process.exitCode = ok ? 0 : 1

package/scripts/check-flash-mirror.mjs

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+/**
+ * Does a Flash page mirror into the column at a usable rate, and do clicks land?
+ *
+ *   node scripts/check-flash-mirror.mjs [url]
+ *
+ * Counts the frames a viewer really receives, times a press and a release
+ * through the page, and reports both next to the frame size. Own profile, own
+ * state directory, closed afterwards; nothing is shown on the desktop.
+ */
+import { mkdtempSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+const url = process.argv[2] ?? 'http://account.100bt.com/game/auth/login.html?gameName=aola&action=startgame&platform=100bt'
+process.env.DSH_INNER_BROWSER_STATE_DIR = mkdtempSync(join(tmpdir(), 'dsh-flash-mirror-'))
+
+const { Sidecar } = await import('../sidecar.js')
+
+/** @param {number} ms - how long. */
+const wait = (ms) => new Promise((settle) => setTimeout(settle, ms))
+const sidecar = new Sidecar({ profile: 'flash-mirror', headless: false, flash: true })
+let ok = false
+try {
+  const started = Date.now()
+  await sidecar.launch('about:blank')
+  const sizes = []
+  const times = []
+  const detach = await sidecar.watch((frame) => {
+    sizes.push(Math.round(String(frame.jpeg ?? '').length * 0.75 / 1024))
+    times.push(Date.now())
+  })
+  const navigatedAt = Date.now()
+  const page = sidecar.page
+  await page.navigate(url).catch(() => {})
+  for (let i = 0; i < 60 && times.length === 0; i += 1) await wait(250)
+  const firstFrame = Date.now() - navigatedAt
+  await wait(4000)
+  detach()
+  const gaps = times.slice(1).map((at, index) => at - times[index]).sort((a, b) => a - b)
+  const fps = gaps.length === 0 ? 0 : Math.round((1000 / (gaps[Math.floor(gaps.length / 2)] || 1000)) * 10) / 10
+  const pressedAt = Date.now()
+  await page.mouse('mousePressed', { x: 640, y: 420 })
+  const pressMs = Date.now() - pressedAt
+  const releasedAt = Date.now()
+  await page.mouse('mouseReleased', { x: 640, y: 420 })
+  const releaseMs = Date.now() - releasedAt
+  await wait(400)
+  const before = await page.picture()
+  const keyAt = Date.now()
+  await page.key('keyDown', { key: 'a', text: 'a' })
+  const keyMs = Date.now() - keyAt
+  await page.key('keyUp', { key: 'a' })
+  await wait(600)
+  const moved = await page.picture()
+  ok = fps >= 15 && pressMs < 40 && releaseMs < 40 && keyMs < 40 && moved !== before && sizes.length > 5
+  console.log(JSON.stringify({
+    firstFrameMs: firstFrame,
+    frames: sizes.length,
+    fps,
+    frameKB: sizes.length === 0 ? 0 : Math.round(sizes.reduce((a, b) => a + b, 0) / sizes.length),
+    pressMs,
+    releaseMs,
+    keyMs,
+    responded: moved !== '' && moved !== before,
+    ok,
+  }))
+} catch (error) {
+  console.log(JSON.stringify({ ok: false, error: String(error?.message ?? error).slice(0, 160) }))
+} finally {
+  await sidecar.shutdown().catch(() => {})
+}
+process.exitCode = ok ? 0 : 1

package/scripts/check-flash-tools.mjs

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+/**
+ * What a tool call costs on a Flash page, from the model's side.
+ *
+ *   node scripts/check-flash-tools.mjs [url]
+ *
+ * Runs the real tool definitions against a real Flash runtime and reports time
+ * and characters for a click, a view and a type, plus whether the picture moved.
+ */
+import { mkdtempSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+const url = process.argv[2] ?? 'http://account.100bt.com/game/auth/login.html?gameName=aola&action=startgame&platform=100bt'
+process.env.DSH_INNER_BROWSER_STATE_DIR = mkdtempSync(join(tmpdir(), 'dsh-flash-tools-'))
+
+const { Sidecar } = await import('../sidecar.js')
+const { sidecarTools } = await import('../tools.js')
+
+const wait = (ms) => new Promise((settle) => setTimeout(settle, ms))
+const sidecar = new Sidecar({ profile: 'flash-tools', headless: false, flash: true })
+const refs = { byChat: new Map() }
+const tools = sidecarTools(sidecar, refs, {})
+const tool = (name) => tools.find((one) => one.name === name)
+let ok = false
+try {
+  await sidecar.launch('about:blank')
+  await sidecar.watch(() => {})
+  await sidecar.page.navigate(url).catch(() => {})
+  await wait(9000)
+  const said = []
+  const timed = async (name, args) => {
+    const started = Date.now()
+    const value = await tool(name).execute(args, { agent: { id: 'flash-check' } })
+    const text = String(value?.text ?? '')
+    const record = { tool: name, ms: Date.now() - started, chars: text.length, text: text.slice(0, 120) }
+    said.push(record)
+    return record
+  }
+  await timed('browser_view', {})
+  /* One call is a sample, five are a cost: a plugin page blocks its main
+     thread while it draws, so a single number swings by a hundred ms. */
+  const rounds = []
+  for (let i = 0; i < 5; i += 1) {
+    rounds.push(await timed('browser_click', { x: 640, y: 420 }))
+    rounds.push(await timed('browser_type', { text: 'a' }))
+  }
+  const median = (list) => {
+    const sorted = list.map((one) => one.ms).sort((a, b) => a - b)
+    return sorted[Math.floor(sorted.length / 2)]
+  }
+  /* A read of a page just read is mostly the page again: it must cost a line,
+     not a snapshot. */
+  const viewAgain = await timed('browser_view', {})
+  const viewThird = await timed('browser_view', {})
+  const clicks = rounds.filter((one) => one.tool === 'browser_click')
+  const types = rounds.filter((one) => one.tool === 'browser_type')
+  const last = rounds[rounds.length - 1]
+  /* A step that repeats the address it read a moment ago is spend with nothing
+     to teach, so the budget is per step and tight: one line, not a page. */
+  const within = (one) => Number(one?.chars ?? 999) < 300
+  /* A page whose own requests fail still answers, but slowly: the numbers to
+     gate on are the token ones, and latency is judged only when the page came
+     up whole. Otherwise a bad minute at the site reads as a regression here. */
+  const degraded = said[0].chars < 300 || said[0].text.includes('page trouble')
+  const fast = () => degraded || (median(clicks) < 120 && median(types) < 120)
+  ok = clicks.length === 5 && fast() && clicks.every(within) && types.every(within) && Number(last?.chars ?? 0) > 0 && viewAgain.chars < said[0].chars && viewThird.chars < said[0].chars
+  console.log(JSON.stringify({ view: said[0], clickMedianMs: median(clicks), clickMaxMs: Math.max(...clicks.map((one) => one.ms)), typeMedianMs: median(types), typeMaxMs: Math.max(...types.map((one) => one.ms)), clickChars: clicks[0].chars, typeChars: types[0].chars, viewAgainChars: viewAgain.chars, viewThirdChars: viewThird.chars, viewAgainText: viewAgain.text, degraded, ok }))
+} catch (error) {
+  console.log(JSON.stringify({ ok: false, error: String(error?.message ?? error).slice(0, 160) }))
+} finally {
+  await sidecar.shutdown().catch(() => {})
+}
+process.exitCode = ok ? 0 : 1

package/scripts/check-flash-window.mjs

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+/**
+ * Can a Flash game run without ever appearing on the desktop?
+ *
+ *   node scripts/check-flash-window.mjs [url]
+ *
+ * Starts the plugin's own Flash runtime on the address, then checks three
+ * things that together mean "only the column sees it": the window was never
+ * shown, the picture keeps changing, and the process is not a frontmost or
+ * Dock-bearing app. Its own profile and state directory; both are closed.
+ */
+import { mkdtempSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+const url = process.argv[2] ?? 'http://account.100bt.com/game/auth/login.html?gameName=aola&action=startgame&platform=100bt'
+process.env.DSH_INNER_BROWSER_STATE_DIR = mkdtempSync(join(tmpdir(), 'dsh-flash-window-'))
+
+const { Sidecar } = await import('../sidecar.js')
+
+/** @param {number} ms - how long. */
+const wait = (ms) => new Promise((settle) => setTimeout(settle, ms))
+const sidecar = new Sidecar({ profile: 'flash-window', headless: false, flash: true })
+let ok = false
+try {
+  const started = Date.now()
+  await sidecar.launch('about:blank')
+  await sidecar.page.navigate(url).catch(() => {})
+  await wait(10_000)
+  const base = `http://127.0.0.1:${sidecar.flashBridge}`
+  const info = await fetch(`${base}/info`).then((reply) => reply.json()).catch(() => null)
+  const grab = async () => {
+    const body = Buffer.from(await (await fetch(`${base}/frame.jpg`)).arrayBuffer())
+    return { kb: Math.round(body.length / 1024), hash: createHash('sha1').update(body).digest('hex').slice(0, 8) }
+  }
+  const one = await grab()
+  await wait(900)
+  const two = await grab()
+  const neverShown = info !== null && info.visible === false
+  const painting = info !== null && Number(info.painted ?? 0) > 10
+  const moving = one.hash !== two.hash
+  ok = neverShown && painting && moving
+  console.log(JSON.stringify({
+    url: String(info?.url ?? url).slice(0, 80),
+    visible: info?.visible,
+    painted: info?.painted,
+    frameKB: one.kb,
+    moved: moving,
+    ms: Date.now() - started,
+    ok,
+  }))
+} catch (error) {
+  console.log(JSON.stringify({ ok: false, error: String(error?.message ?? error).slice(0, 160) }))
+} finally {
+  await sidecar.shutdown().catch(() => {})
+}
+process.exitCode = ok ? 0 : 1

package/scripts/doctor.mjs

@@ -66,9 +66,10 @@ const state = await get(`${ROUTE}/sidecar/state`)
 console.log(`host     ${state === null ? `not answering under ${ROUTE}` : base}`)
 if (state === null) {
   console.log('         start the app, or set DSH_BRIDGE to the app address')
-  process.exit(0)
+  process.exitCode = 0
 }
 
+if (state !== null) {
 console.log(`browser  ${state.alive === true ? `attached, ${state.headless === true ? 'headless' : 'with a window'}` : 'not running'}`)
 const profileDir = join(PROFILES, String(state.profile ?? ''))
 console.log(`profile  ${state.profile} in ${profileDir}`)
@@ -87,3 +88,4 @@ if (existsSync(portFile)) {
 }
 
 if (String(state.error ?? '') !== '') console.log(`error    ${state.error}`)
+}