minimal-agent 0.10.0 → 0.12.0

package/src/tools/computeruse/vision/png.js

@@ -0,0 +1,274 @@
+/**
+ * ============================================================
+ *  vision/png.ts —— 纯 Node PNG 编解码（零依赖）
+ * ------------------------------------------------------------
+ *  为什么自己写而不是装个库：
+ *    绿色版是 `bun --compile` 单文件夹、插 U 盘即用，多一个原生依赖就多一份
+ *    跨平台构建与法务负担。而我们真正需要的只有「解出像素 / 拼回 PNG」——
+ *    PNG 的压缩层就是 zlib（Node 内置），剩下的是 chunk 解析和 5 种行滤波器，
+ *    加起来两百行。OpenCV / sharp 那种量级的依赖买不到额外价值。
+ *
+ *  支持范围（够用即可，不做全功能编解码器）：
+ *    - 位深 8（screencapture / DXGI / ScreenCaptureKit 的输出都是 8bit）
+ *    - 颜色类型 0 灰度 / 2 RGB / 3 调色板 / 4 灰度+A / 6 RGBA
+ *    - **不支持隔行（Adam7）** —— 屏幕截图不会用它；遇到直接抛错，不静默出错图
+ *    - 解码统一输出 RGBA8，调用方不必关心源格式
+ *    - 编码可选 RGBA(6) 或 RGB(2)：发给模型的图不需要 alpha，去掉省 25% 字节
+ *
+ *  编码固定用滤波器 0（None）：屏幕截图的大片纯色区域交给 zlib 已经压得很好，
+ *  逐行选最优滤波器的收益（几个百分点）不值得那份复杂度和出错面。
+ * ============================================================
+ */
+import { deflateSync, inflateSync } from 'node:zlib';
+const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+/** 每种颜色类型的采样数（每像素几个通道）。 */
+const SAMPLES_PER_PIXEL = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 };
+// ============================================================
+//  CRC32（PNG chunk 校验）
+// ============================================================
+const CRC_TABLE = (() => {
+    const table = new Uint32Array(256);
+    for (let n = 0; n < 256; n += 1) {
+        let c = n;
+        for (let k = 0; k < 8; k += 1) {
+            c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+        }
+        table[n] = c >>> 0;
+    }
+    return table;
+})();
+function crc32(buf) {
+    let c = 0xffffffff;
+    for (let i = 0; i < buf.length; i += 1) {
+        c = CRC_TABLE[(c ^ buf[i]) & 0xff] ^ (c >>> 8);
+    }
+    return (c ^ 0xffffffff) >>> 0;
+}
+/** 解出 RGBA8 像素。源图是灰度/调色板/RGB 都统一展开。 */
+export function decodePng(buffer) {
+    if (buffer.length < 8 || !buffer.subarray(0, 8).equals(SIGNATURE)) {
+        throw new Error('不是 PNG 文件（签名不匹配）');
+    }
+    let header = null;
+    let palette = null;
+    let transparency = null;
+    const idatParts = [];
+    let offset = 8;
+    while (offset + 8 <= buffer.length) {
+        const length = buffer.readUInt32BE(offset);
+        const type = buffer.toString('ascii', offset + 4, offset + 8);
+        const dataStart = offset + 8;
+        const data = buffer.subarray(dataStart, dataStart + length);
+        if (type === 'IHDR') {
+            header = {
+                width: data.readUInt32BE(0),
+                height: data.readUInt32BE(4),
+                bitDepth: data[8],
+                colorType: data[9],
+                interlace: data[12],
+            };
+            if (header.bitDepth !== 8) {
+                throw new Error(`只支持 8 位色深，实际 ${header.bitDepth}`);
+            }
+            if (header.interlace !== 0) {
+                throw new Error('不支持隔行 PNG（Adam7）');
+            }
+            if (SAMPLES_PER_PIXEL[header.colorType] === undefined) {
+                throw new Error(`不支持的颜色类型 ${header.colorType}`);
+            }
+        }
+        else if (type === 'PLTE') {
+            palette = Buffer.from(data);
+        }
+        else if (type === 'tRNS') {
+            transparency = Buffer.from(data);
+        }
+        else if (type === 'IDAT') {
+            idatParts.push(Buffer.from(data));
+        }
+        else if (type === 'IEND') {
+            break;
+        }
+        offset = dataStart + length + 4; // +4 跳过 CRC
+    }
+    if (!header)
+        throw new Error('PNG 缺少 IHDR');
+    if (idatParts.length === 0)
+        throw new Error('PNG 缺少 IDAT');
+    const raw = inflateSync(Buffer.concat(idatParts));
+    const samples = SAMPLES_PER_PIXEL[header.colorType];
+    const scanlines = unfilter(raw, header.width, header.height, samples);
+    const data = toRgba(scanlines, header, samples, palette, transparency);
+    return { width: header.width, height: header.height, data };
+}
+/**
+ * 逐行去滤波。PNG 的每条扫描线前有 1 字节滤波器类型，且滤波是**就地递推**的
+ * （当前行依赖已复原的上一行），所以必须顺序处理、不能并行。
+ */
+function unfilter(raw, width, height, samples) {
+    const stride = width * samples;
+    const out = Buffer.allocUnsafe(stride * height);
+    let pos = 0;
+    for (let y = 0; y < height; y += 1) {
+        const filter = raw[pos];
+        pos += 1;
+        const line = raw.subarray(pos, pos + stride);
+        pos += stride;
+        const cur = out.subarray(y * stride, (y + 1) * stride);
+        const prev = y > 0 ? out.subarray((y - 1) * stride, y * stride) : null;
+        for (let i = 0; i < stride; i += 1) {
+            const rawByte = line[i];
+            const a = i >= samples ? cur[i - samples] : 0; // 左
+            const b = prev ? prev[i] : 0; // 上
+            const c = prev && i >= samples ? prev[i - samples] : 0; // 左上
+            switch (filter) {
+                case 0:
+                    cur[i] = rawByte;
+                    break;
+                case 1:
+                    cur[i] = (rawByte + a) & 0xff;
+                    break;
+                case 2:
+                    cur[i] = (rawByte + b) & 0xff;
+                    break;
+                case 3:
+                    cur[i] = (rawByte + ((a + b) >> 1)) & 0xff;
+                    break;
+                case 4:
+                    cur[i] = (rawByte + paeth(a, b, c)) & 0xff;
+                    break;
+                default: throw new Error(`未知的 PNG 行滤波器 ${filter}（第 ${y} 行）`);
+            }
+        }
+    }
+    return out;
+}
+function paeth(a, b, c) {
+    const p = a + b - c;
+    const pa = Math.abs(p - a);
+    const pb = Math.abs(p - b);
+    const pc = Math.abs(p - c);
+    if (pa <= pb && pa <= pc)
+        return a;
+    return pb <= pc ? b : c;
+}
+/** 各种颜色类型统一展开成 RGBA8 —— 上层从此不必关心源格式。 */
+function toRgba(scanlines, header, samples, palette, transparency) {
+    const { width, height, colorType } = header;
+    const out = new Uint8Array(width * height * 4);
+    for (let i = 0, px = 0; px < width * height; px += 1) {
+        const s = px * samples;
+        let r;
+        let g;
+        let b;
+        let a = 255;
+        switch (colorType) {
+            case 0:
+                r = g = b = scanlines[s];
+                break;
+            case 2:
+                r = scanlines[s];
+                g = scanlines[s + 1];
+                b = scanlines[s + 2];
+                break;
+            case 3: {
+                const idx = scanlines[s];
+                if (!palette)
+                    throw new Error('调色板 PNG 缺少 PLTE');
+                r = palette[idx * 3];
+                g = palette[idx * 3 + 1];
+                b = palette[idx * 3 + 2];
+                if (transparency && idx < transparency.length)
+                    a = transparency[idx];
+                break;
+            }
+            case 4:
+                r = g = b = scanlines[s];
+                a = scanlines[s + 1];
+                break;
+            default: // 6
+                r = scanlines[s];
+                g = scanlines[s + 1];
+                b = scanlines[s + 2];
+                a = scanlines[s + 3];
+        }
+        out[i] = r;
+        out[i + 1] = g;
+        out[i + 2] = b;
+        out[i + 3] = a;
+        i += 4;
+    }
+    return out;
+}
+export function encodePng(image, options = {}) {
+    const { width, height, data } = image;
+    if (data.length !== width * height * 4) {
+        throw new Error(`像素数据长度不符：期望 ${width * height * 4}，实际 ${data.length}`);
+    }
+    const channels = options.channels ?? 4;
+    // 每行前置一个 0（滤波器 None），其余原样
+    const stride = width * channels;
+    const raw = Buffer.allocUnsafe((stride + 1) * height);
+    for (let y = 0; y < height; y += 1) {
+        const rowStart = y * (stride + 1);
+        raw[rowStart] = 0;
+        if (channels === 4) {
+            Buffer.from(data.buffer, data.byteOffset + y * stride, stride).copy(raw, rowStart + 1);
+        }
+        else {
+            // RGB：逐像素丢掉 alpha 通道。
+            let out = rowStart + 1;
+            let src = y * width * 4;
+            for (let x = 0; x < width; x += 1) {
+                raw[out] = data[src];
+                raw[out + 1] = data[src + 1];
+                raw[out + 2] = data[src + 2];
+                out += 3;
+                src += 4;
+            }
+        }
+    }
+    const ihdr = Buffer.alloc(13);
+    ihdr.writeUInt32BE(width, 0);
+    ihdr.writeUInt32BE(height, 4);
+    ihdr[8] = 8; // bit depth
+    ihdr[9] = channels === 4 ? 6 : 2; // RGBA / RGB
+    ihdr[10] = 0; // deflate
+    ihdr[11] = 0; // adaptive filtering
+    ihdr[12] = 0; // no interlace
+    return Buffer.concat([
+        SIGNATURE,
+        chunk('IHDR', ihdr),
+        chunk('IDAT', deflateSync(raw, { level: options.level ?? 6 })),
+        chunk('IEND', Buffer.alloc(0)),
+    ]);
+}
+function chunk(type, data) {
+    const out = Buffer.allocUnsafe(data.length + 12);
+    out.writeUInt32BE(data.length, 0);
+    out.write(type, 4, 'ascii');
+    data.copy(out, 8);
+    out.writeUInt32BE(crc32(out.subarray(4, 8 + data.length)), 8 + data.length);
+    return out;
+}
+/**
+ * 把裸灰度帧（helper 的 `gray480` 出口）包成 PNG。
+ *
+ * 灰度帧是原生侧为了做帧差而下采样的产物，用于 `replay`：它回答「刚才发生了
+ * 什么」而不是「上面写的什么」，所以不需要颜色。
+ */
+export function encodeGrayscalePng(gray, width, height, options = {}) {
+    if (gray.length !== width * height) {
+        throw new Error(`灰度数据长度不符：期望 ${width * height}，实际 ${gray.length}`);
+    }
+    const rgba = new Uint8Array(width * height * 4);
+    for (let i = 0, p = 0; p < gray.length; p += 1) {
+        const value = gray[p];
+        rgba[i] = value;
+        rgba[i + 1] = value;
+        rgba[i + 2] = value;
+        rgba[i + 3] = 255;
+        i += 4;
+    }
+    return encodePng({ width, height, data: rgba }, { channels: 3, ...options });
+}

package/src/tools/computeruse/vision/png.ts

@@ -0,0 +1,317 @@
+/**
+ * ============================================================
+ *  vision/png.ts —— 纯 Node PNG 编解码（零依赖）
+ * ------------------------------------------------------------
+ *  为什么自己写而不是装个库：
+ *    绿色版是 `bun --compile` 单文件夹、插 U 盘即用，多一个原生依赖就多一份
+ *    跨平台构建与法务负担。而我们真正需要的只有「解出像素 / 拼回 PNG」——
+ *    PNG 的压缩层就是 zlib（Node 内置），剩下的是 chunk 解析和 5 种行滤波器，
+ *    加起来两百行。OpenCV / sharp 那种量级的依赖买不到额外价值。
+ *
+ *  支持范围（够用即可，不做全功能编解码器）：
+ *    - 位深 8（screencapture / DXGI / ScreenCaptureKit 的输出都是 8bit）
+ *    - 颜色类型 0 灰度 / 2 RGB / 3 调色板 / 4 灰度+A / 6 RGBA
+ *    - **不支持隔行（Adam7）** —— 屏幕截图不会用它；遇到直接抛错，不静默出错图
+ *    - 解码统一输出 RGBA8，调用方不必关心源格式
+ *    - 编码可选 RGBA(6) 或 RGB(2)：发给模型的图不需要 alpha，去掉省 25% 字节
+ *
+ *  编码固定用滤波器 0（None）：屏幕截图的大片纯色区域交给 zlib 已经压得很好，
+ *  逐行选最优滤波器的收益（几个百分点）不值得那份复杂度和出错面。
+ * ============================================================
+ */
+
+import { deflateSync, inflateSync } from 'node:zlib';
+
+/** RGBA8 像素缓冲。所有 vision 模块内部统一用它。 */
+export interface RgbaImage {
+  width: number;
+  height: number;
+  /** 长度必须是 width × height × 4。 */
+  data: Uint8Array;
+}
+
+const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+
+/** 每种颜色类型的采样数（每像素几个通道）。 */
+const SAMPLES_PER_PIXEL: Record<number, number> = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 };
+
+// ============================================================
+//  CRC32（PNG chunk 校验）
+// ============================================================
+
+const CRC_TABLE = (() => {
+  const table = new Uint32Array(256);
+  for (let n = 0; n < 256; n += 1) {
+    let c = n;
+    for (let k = 0; k < 8; k += 1) {
+      c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    }
+    table[n] = c >>> 0;
+  }
+  return table;
+})();
+
+function crc32(buf: Buffer): number {
+  let c = 0xffffffff;
+  for (let i = 0; i < buf.length; i += 1) {
+    c = CRC_TABLE[(c ^ buf[i]!) & 0xff]! ^ (c >>> 8);
+  }
+  return (c ^ 0xffffffff) >>> 0;
+}
+
+// ============================================================
+//  解码
+// ============================================================
+
+interface PngHeader {
+  width: number;
+  height: number;
+  bitDepth: number;
+  colorType: number;
+  interlace: number;
+}
+
+/** 解出 RGBA8 像素。源图是灰度/调色板/RGB 都统一展开。 */
+export function decodePng(buffer: Buffer): RgbaImage {
+  if (buffer.length < 8 || !buffer.subarray(0, 8).equals(SIGNATURE)) {
+    throw new Error('不是 PNG 文件（签名不匹配）');
+  }
+
+  let header: PngHeader | null = null;
+  let palette: Buffer | null = null;
+  let transparency: Buffer | null = null;
+  const idatParts: Buffer[] = [];
+
+  let offset = 8;
+  while (offset + 8 <= buffer.length) {
+    const length = buffer.readUInt32BE(offset);
+    const type = buffer.toString('ascii', offset + 4, offset + 8);
+    const dataStart = offset + 8;
+    const data = buffer.subarray(dataStart, dataStart + length);
+
+    if (type === 'IHDR') {
+      header = {
+        width: data.readUInt32BE(0),
+        height: data.readUInt32BE(4),
+        bitDepth: data[8]!,
+        colorType: data[9]!,
+        interlace: data[12]!,
+      };
+      if (header.bitDepth !== 8) {
+        throw new Error(`只支持 8 位色深，实际 ${header.bitDepth}`);
+      }
+      if (header.interlace !== 0) {
+        throw new Error('不支持隔行 PNG（Adam7）');
+      }
+      if (SAMPLES_PER_PIXEL[header.colorType] === undefined) {
+        throw new Error(`不支持的颜色类型 ${header.colorType}`);
+      }
+    } else if (type === 'PLTE') {
+      palette = Buffer.from(data);
+    } else if (type === 'tRNS') {
+      transparency = Buffer.from(data);
+    } else if (type === 'IDAT') {
+      idatParts.push(Buffer.from(data));
+    } else if (type === 'IEND') {
+      break;
+    }
+
+    offset = dataStart + length + 4; // +4 跳过 CRC
+  }
+
+  if (!header) throw new Error('PNG 缺少 IHDR');
+  if (idatParts.length === 0) throw new Error('PNG 缺少 IDAT');
+
+  const raw = inflateSync(Buffer.concat(idatParts));
+  const samples = SAMPLES_PER_PIXEL[header.colorType]!;
+  const scanlines = unfilter(raw, header.width, header.height, samples);
+  const data = toRgba(scanlines, header, samples, palette, transparency);
+
+  return { width: header.width, height: header.height, data };
+}
+
+/**
+ * 逐行去滤波。PNG 的每条扫描线前有 1 字节滤波器类型，且滤波是**就地递推**的
+ * （当前行依赖已复原的上一行），所以必须顺序处理、不能并行。
+ */
+function unfilter(raw: Buffer, width: number, height: number, samples: number): Buffer {
+  const stride = width * samples;
+  const out = Buffer.allocUnsafe(stride * height);
+  let pos = 0;
+
+  for (let y = 0; y < height; y += 1) {
+    const filter = raw[pos];
+    pos += 1;
+    const line = raw.subarray(pos, pos + stride);
+    pos += stride;
+
+    const cur = out.subarray(y * stride, (y + 1) * stride);
+    const prev = y > 0 ? out.subarray((y - 1) * stride, y * stride) : null;
+
+    for (let i = 0; i < stride; i += 1) {
+      const rawByte = line[i]!;
+      const a = i >= samples ? cur[i - samples]! : 0; // 左
+      const b = prev ? prev[i]! : 0; // 上
+      const c = prev && i >= samples ? prev[i - samples]! : 0; // 左上
+
+      switch (filter) {
+        case 0: cur[i] = rawByte; break;
+        case 1: cur[i] = (rawByte + a) & 0xff; break;
+        case 2: cur[i] = (rawByte + b) & 0xff; break;
+        case 3: cur[i] = (rawByte + ((a + b) >> 1)) & 0xff; break;
+        case 4: cur[i] = (rawByte + paeth(a, b, c)) & 0xff; break;
+        default: throw new Error(`未知的 PNG 行滤波器 ${filter}（第 ${y} 行）`);
+      }
+    }
+  }
+  return out;
+}
+
+function paeth(a: number, b: number, c: number): number {
+  const p = a + b - c;
+  const pa = Math.abs(p - a);
+  const pb = Math.abs(p - b);
+  const pc = Math.abs(p - c);
+  if (pa <= pb && pa <= pc) return a;
+  return pb <= pc ? b : c;
+}
+
+/** 各种颜色类型统一展开成 RGBA8 —— 上层从此不必关心源格式。 */
+function toRgba(
+  scanlines: Buffer,
+  header: PngHeader,
+  samples: number,
+  palette: Buffer | null,
+  transparency: Buffer | null,
+): Uint8Array {
+  const { width, height, colorType } = header;
+  const out = new Uint8Array(width * height * 4);
+
+  for (let i = 0, px = 0; px < width * height; px += 1) {
+    const s = px * samples;
+    let r: number;
+    let g: number;
+    let b: number;
+    let a = 255;
+
+    switch (colorType) {
+      case 0: r = g = b = scanlines[s]!; break;
+      case 2: r = scanlines[s]!; g = scanlines[s + 1]!; b = scanlines[s + 2]!; break;
+      case 3: {
+        const idx = scanlines[s]!;
+        if (!palette) throw new Error('调色板 PNG 缺少 PLTE');
+        r = palette[idx * 3]!;
+        g = palette[idx * 3 + 1]!;
+        b = palette[idx * 3 + 2]!;
+        if (transparency && idx < transparency.length) a = transparency[idx]!;
+        break;
+      }
+      case 4: r = g = b = scanlines[s]!; a = scanlines[s + 1]!; break;
+      default: // 6
+        r = scanlines[s]!; g = scanlines[s + 1]!; b = scanlines[s + 2]!; a = scanlines[s + 3]!;
+    }
+
+    out[i] = r; out[i + 1] = g; out[i + 2] = b; out[i + 3] = a;
+    i += 4;
+  }
+  return out;
+}
+
+// ============================================================
+//  编码
+// ============================================================
+
+export interface EncodePngOptions {
+  /** zlib 压缩级别（默认 6；截图用 3 明显更快且体积接近）。 */
+  level?: number;
+  /**
+   * 通道数。3 = RGB（颜色类型 2），4 = RGBA（颜色类型 6）。默认 4。
+   *
+   * 发给模型的图**一律用 3**：多模态 provider 要么把 alpha 合成掉、要么直接
+   * 丢弃（属未定义行为），带上它只是白传 25% 的字节。需要「透明感」时必须在
+   * `raster.ts::fadeOutside` 里自己合成到确定的背景色，而不是指望 alpha。
+   */
+  channels?: 3 | 4;
+}
+
+export function encodePng(image: RgbaImage, options: EncodePngOptions = {}): Buffer {
+  const { width, height, data } = image;
+  if (data.length !== width * height * 4) {
+    throw new Error(`像素数据长度不符：期望 ${width * height * 4}，实际 ${data.length}`);
+  }
+  const channels = options.channels ?? 4;
+
+  // 每行前置一个 0（滤波器 None），其余原样
+  const stride = width * channels;
+  const raw = Buffer.allocUnsafe((stride + 1) * height);
+  for (let y = 0; y < height; y += 1) {
+    const rowStart = y * (stride + 1);
+    raw[rowStart] = 0;
+    if (channels === 4) {
+      Buffer.from(data.buffer, data.byteOffset + y * stride, stride).copy(raw, rowStart + 1);
+    } else {
+      // RGB：逐像素丢掉 alpha 通道。
+      let out = rowStart + 1;
+      let src = y * width * 4;
+      for (let x = 0; x < width; x += 1) {
+        raw[out] = data[src]!;
+        raw[out + 1] = data[src + 1]!;
+        raw[out + 2] = data[src + 2]!;
+        out += 3;
+        src += 4;
+      }
+    }
+  }
+
+  const ihdr = Buffer.alloc(13);
+  ihdr.writeUInt32BE(width, 0);
+  ihdr.writeUInt32BE(height, 4);
+  ihdr[8] = 8; // bit depth
+  ihdr[9] = channels === 4 ? 6 : 2; // RGBA / RGB
+  ihdr[10] = 0; // deflate
+  ihdr[11] = 0; // adaptive filtering
+  ihdr[12] = 0; // no interlace
+
+  return Buffer.concat([
+    SIGNATURE,
+    chunk('IHDR', ihdr),
+    chunk('IDAT', deflateSync(raw, { level: options.level ?? 6 })),
+    chunk('IEND', Buffer.alloc(0)),
+  ]);
+}
+
+function chunk(type: string, data: Buffer): Buffer {
+  const out = Buffer.allocUnsafe(data.length + 12);
+  out.writeUInt32BE(data.length, 0);
+  out.write(type, 4, 'ascii');
+  data.copy(out, 8);
+  out.writeUInt32BE(crc32(out.subarray(4, 8 + data.length)), 8 + data.length);
+  return out;
+}
+
+/**
+ * 把裸灰度帧（helper 的 `gray480` 出口）包成 PNG。
+ *
+ * 灰度帧是原生侧为了做帧差而下采样的产物，用于 `replay`：它回答「刚才发生了
+ * 什么」而不是「上面写的什么」，所以不需要颜色。
+ */
+export function encodeGrayscalePng(
+  gray: Uint8Array,
+  width: number,
+  height: number,
+  options: EncodePngOptions = {},
+): Buffer {
+  if (gray.length !== width * height) {
+    throw new Error(`灰度数据长度不符：期望 ${width * height}，实际 ${gray.length}`);
+  }
+  const rgba = new Uint8Array(width * height * 4);
+  for (let i = 0, p = 0; p < gray.length; p += 1) {
+    const value = gray[p]!;
+    rgba[i] = value;
+    rgba[i + 1] = value;
+    rgba[i + 2] = value;
+    rgba[i + 3] = 255;
+    i += 4;
+  }
+  return encodePng({ width, height, data: rgba }, { channels: 3, ...options });
+}