kelra 0.0.0-stage → 0.1.0

package/package.json

@@ -1,6 +1,40 @@
 {
   "name": "kelra",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "type": "module",
+  "main": "dist/cli.js",
+  "bin": {
+    "kelra": "dist/cli.js"
+  },
+  "scripts": {
+    "prepack": "cd ../.. && npm run build && test -f packages/helper/dist/cli.js && test -f packages/helper/bin/kelra-ocr",
+    "build:ocr": "mkdir -p bin && DD=\"${DEVELOPER_DIR:-$(xcode-select -p)}\" && SDK=\"$DD/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk\" && SW=\"$DD/Toolchains/XcodeDefault.xctoolchain/usr/bin/swiftc\" && \"$SW\" -O -target arm64-apple-macos13 swift/ocr.swift -o bin/kelra-ocr-arm64 -sdk \"$SDK\" && \"$SW\" -O -target x86_64-apple-macos13 swift/ocr.swift -o bin/kelra-ocr-x64 -sdk \"$SDK\" && lipo -create bin/kelra-ocr-arm64 bin/kelra-ocr-x64 -output bin/kelra-ocr && rm bin/kelra-ocr-arm64 bin/kelra-ocr-x64"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.12.0",
+    "pngjs": "^7.0.0",
+    "zod": "^3.24.0"
+  },
+  "devDependencies": {
+    "@types/pngjs": "^6.0.5",
+    "@kelra/protocol": "*"
+  },
+  "files": [
+    "dist",
+    "!dist/.tsbuildinfo",
+    "!dist/**/*.map",
+    "swift",
+    "bin",
+    "README.md",
+    "AGENT_GUIDE.md"
+  ],
+  "description": "MCP server that gives AI agents precise iOS UI measurements: measure a reference, build, compare, fix.",
+  "os": [
+    "darwin"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "license": "SEE LICENSE IN LICENSE",
+  "homepage": "https://kelra.net"
+}

package/swift/ocr.swift

@@ -0,0 +1,216 @@
+// kelra-ocr: reads text lines from an image with Apple Vision and reports raw measurements.
+// usage: kelra-ocr <image.png> <pixelsPerPoint>
+// prints a JSON array: text, confidence, rectangle in points, the stroke thickness and ink height measured in the image,
+// and for each system-font weight the size whose width matches plus that candidate's own stroke thickness.
+// It makes no decisions: which weight and size a line really is gets decided by the kelra service.
+import Foundation
+import Vision
+import AppKit
+import CoreText
+
+struct Candidate: Codable { var weight: String; var sizePt: Double; var coverage: Double; var stemPx: Double }
+struct RawLine: Codable {
+    var text: String
+    var confidence: Float
+    var x: Double, y: Double, w: Double, h: Double
+    var cropStemPx: Double
+    var inkHeightPt: Double
+    var cropCoverage: Double
+    var candidates: [Candidate]
+}
+
+let weights: [(String, NSFont.Weight)] = [("regular", .regular), ("medium", .medium), ("semibold", .semibold), ("bold", .bold)]
+
+func fail(_ msg: String) -> Never {
+    FileHandle.standardError.write((msg + "\n").data(using: .utf8)!)
+    exit(1)
+}
+
+let args = CommandLine.arguments
+guard args.count >= 3, let scale = Double(args[2]), scale > 0 else { fail("usage: kelra-ocr <image> <pixelsPerPoint>") }
+guard let src = CGImageSourceCreateWithURL(URL(fileURLWithPath: args[1]) as CFURL, nil),
+      let image = CGImageSourceCreateImageAtIndex(src, 0, nil) else { fail("cannot read image \(args[1])") }
+let W = image.width, H = image.height
+
+// luminance of the whole image, row-major, top row first
+var gray = [UInt8](repeating: 0, count: W * H)
+gray.withUnsafeMutableBytes { buf in
+    let ctx = CGContext(data: buf.baseAddress, width: W, height: H, bitsPerComponent: 8, bytesPerRow: W,
+                        space: CGColorSpaceCreateDeviceGray(), bitmapInfo: CGImageAlphaInfo.none.rawValue)!
+    ctx.draw(image, in: CGRect(x: 0, y: 0, width: W, height: H))
+}
+
+/// how much of a box is covered by ink: 0 (empty) to 1 (solid), relative to the strongest contrast in the box
+func cropCoverage(x: Int, y: Int, w: Int, h: Int) -> Double {
+    guard w > 2, h > 2 else { return 0 }
+    var border = [Int]()
+    for i in 0..<w { border.append(Int(gray[y * W + x + i])); border.append(Int(gray[(y + h - 1) * W + x + i])) }
+    for j in 0..<h { border.append(Int(gray[(y + j) * W + x])); border.append(Int(gray[(y + j) * W + x + w - 1])) }
+    border.sort()
+    let bg = Double(border[border.count / 2])
+    var farthest = 0.0
+    for j in 0..<h { for i in 0..<w { farthest = max(farthest, abs(Double(gray[(y + j) * W + x + i]) - bg)) } }
+    guard farthest > 20 else { return 0 }
+    var sum = 0.0
+    for j in 0..<h { for i in 0..<w { sum += min(1, abs(Double(gray[(y + j) * W + x + i]) - bg) / farthest) } }
+    return sum / Double(w * h)
+}
+
+/// Height in pixels of the drawn ink (first to last row that has any), not of Vision's looser text box.
+func inkHeight(_ buf: [UInt8], stride: Int, x: Int, y: Int, w: Int, h: Int) -> Double {
+    guard w > 2, h > 2 else { return 0 }
+    var border = [Int]()
+    for i in 0..<w { border.append(Int(buf[y * stride + x + i])); border.append(Int(buf[(y + h - 1) * stride + x + i])) }
+    border.sort()
+    let bg = Double(border[border.count / 2])
+    var farthest = 0.0
+    for j in 0..<h { for i in 0..<w { farthest = max(farthest, abs(Double(buf[(y + j) * stride + x + i]) - bg)) } }
+    guard farthest > 20 else { return 0 }
+    var top = -1, bottom = -1
+    for j in 0..<h {
+        let hasInk = (0..<w).contains { i in abs(Double(buf[(y + j) * stride + x + i]) - bg) > 0.35 * farthest }
+        if hasInk { if top < 0 { top = j }; bottom = j }
+    }
+    return top < 0 ? 0 : Double(bottom - top + 1)
+}
+
+/// Typical stroke thickness in pixels: the median, over rows and ink runs, of each run's darkness-weighted length.
+/// Measured the same way on the screenshot crop and on rendered candidates, so no assumption about the other text is needed.
+func stemWidth(_ buf: [UInt8], stride: Int, x: Int, y: Int, w: Int, h: Int) -> Double {
+    guard w > 2, h > 2 else { return 0 }
+    var border = [Int]()
+    for i in 0..<w { border.append(Int(buf[y * stride + x + i])); border.append(Int(buf[(y + h - 1) * stride + x + i])) }
+    for j in 0..<h { border.append(Int(buf[(y + j) * stride + x])); border.append(Int(buf[(y + j) * stride + x + w - 1])) }
+    border.sort()
+    let bg = Double(border[border.count / 2])
+    var farthest = 0.0
+    for j in 0..<h { for i in 0..<w { farthest = max(farthest, abs(Double(buf[(y + j) * stride + x + i]) - bg)) } }
+    guard farthest > 20 else { return 0 }
+    var runs = [Double]()
+    for j in 0..<h {
+        var sum = 0.0, len = 0
+        for i in 0...w {
+            let d = i < w ? min(1, abs(Double(buf[(y + j) * stride + x + i]) - bg) / farthest) : 0
+            if d > 0.2 { sum += d; len += 1 } else if len > 0 { runs.append(sum); sum = 0; len = 0 }
+        }
+    }
+    guard !runs.isEmpty else { return 0 }
+    runs.sort()
+    return runs[runs.count / 2]
+}
+
+func textWidth(_ s: String, size: CGFloat, weight: NSFont.Weight) -> CGFloat {
+    (s as NSString).size(withAttributes: [.font: NSFont.systemFont(ofSize: size, weight: weight)]).width
+}
+
+/// the font size, in points, at which the string is `targetPt` wide
+func solveSize(_ s: String, targetPt: Double, weight: NSFont.Weight) -> Double {
+    var size = 17.0
+    for _ in 0..<8 {
+        let w = Double(textWidth(s, size: CGFloat(size), weight: weight))
+        if w <= 0 { break }
+        size *= targetPt / w
+    }
+    return size
+}
+
+/// coverage of the string rendered in a system font, in a box of the same pixel size
+func renderCoverage(_ s: String, sizePt: Double, weight: NSFont.Weight, boxW: Int, boxH: Int) -> (coverage: Double, stem: Double) {
+    guard boxW > 2, boxH > 2 else { return (0, 0) }
+    var buf = [UInt8](repeating: 255, count: boxW * boxH)
+    var total = 0.0
+    buf.withUnsafeMutableBytes { p in
+        let ctx = CGContext(data: p.baseAddress, width: boxW, height: boxH, bitsPerComponent: 8, bytesPerRow: boxW,
+                            space: CGColorSpaceCreateDeviceGray(), bitmapInfo: CGImageAlphaInfo.none.rawValue)!
+        let font = NSFont.systemFont(ofSize: CGFloat(sizePt * scale), weight: weight)
+        let attr = NSAttributedString(string: s, attributes: [.font: font, .foregroundColor: NSColor.black])
+        let line = CTLineCreateWithAttributedString(attr)
+        var asc: CGFloat = 0, desc: CGFloat = 0, lead: CGFloat = 0
+        let lw = CTLineGetTypographicBounds(line, &asc, &desc, &lead)
+        ctx.textPosition = CGPoint(x: (CGFloat(boxW) - CGFloat(lw)) / 2, y: (CGFloat(boxH) - (asc + desc)) / 2 + desc)
+        CTLineDraw(line, ctx)
+    }
+    for v in buf { total += (255 - Double(v)) / 255 }
+    return (total / Double(boxW * boxH), stemWidth(buf, stride: boxW, x: 0, y: 0, w: boxW, h: boxH))
+}
+
+let request = VNRecognizeTextRequest()
+request.recognitionLevel = .accurate
+request.usesLanguageCorrection = false  // UI strings like "£3.45" or "Sep 29" must not be "corrected"
+do { try VNImageRequestHandler(cgImage: image, options: [:]).perform([request]) } catch { fail("vision failed: \(error)") }
+
+/// column ranges (relative to the box) of the word-token groups in a box. Each boundary between tokens is matched to the
+/// ink gap nearest to where it should fall by character position, so a gap inside a word (a comma) is not mistaken for a space.
+func tokenGroups(x: Int, y: Int, w: Int, h: Int, tokenLengths: [Int]) -> [(Int, Int)] {
+    var border = [Int]()
+    for i in 0..<w { border.append(Int(gray[y * W + x + i])); border.append(Int(gray[(y + h - 1) * W + x + i])) }
+    border.sort()
+    let bg = Double(border[border.count / 2])
+    var farthest = 0.0
+    for j in 0..<h { for i in 0..<w { farthest = max(farthest, abs(Double(gray[(y + j) * W + x + i]) - bg)) } }
+    guard farthest > 20 else { return [] }
+    let ink = (0..<w).map { i -> Bool in (0..<h).contains { j in abs(Double(gray[(y + j) * W + x + i]) - bg) > 0.35 * farthest } }
+    guard let first = ink.firstIndex(of: true), let last = ink.lastIndex(of: true) else { return [] }
+    var gaps = [(start: Int, len: Int)](); var i = first
+    while i <= last { if !ink[i] { var j = i; while j <= last && !ink[j] { j += 1 }; gaps.append((i, j - i)); i = j } else { i += 1 } }
+    let total = Double(tokenLengths.reduce(0, +) + tokenLengths.count - 1)
+    var cuts = [(start: Int, len: Int)](), used = Set<Int>(), chars = 0.0
+    for k in 0..<(tokenLengths.count - 1) {
+        chars += Double(tokenLengths[k]) + 0.5
+        let expected = Double(first) + chars / total * Double(last - first)
+        guard let g = gaps.enumerated().filter({ !used.contains($0.offset) }).min(by: { abs(Double($0.element.start) + Double($0.element.len) / 2 - expected) < abs(Double($1.element.start) + Double($1.element.len) / 2 - expected) }),
+              abs(Double(g.element.start) + Double(g.element.len) / 2 - expected) < 0.22 * Double(last - first) else { return [] }
+        used.insert(g.offset); cuts.append(g.element)
+    }
+    cuts.sort { $0.start < $1.start }
+    var groups = [(Int, Int)](); var from = first
+    for c in cuts { groups.append((from, c.start - 1)); from = c.start + c.len }
+    groups.append((from, last))
+    return groups
+}
+
+func isWord(_ t: Substring) -> Bool { t.contains { $0.isLetter || $0.isNumber } }
+
+struct Pending { var text: String; var conf: Float; var px: Int, py: Int, pw: Int, ph: Int; var crop: Double; var cropStem: Double; var inkH: Double; var fits: [Candidate] }
+var pending = [Pending]()
+for obs in request.results ?? [] {
+    guard let cand = obs.topCandidates(1).first else { continue }
+    let bb = obs.boundingBox  // normalized, origin at bottom-left
+    var text = cand.string
+    var px = Int((bb.minX * Double(W)).rounded()), pw = Int((bb.width * Double(W)).rounded())
+    let ph = Int((bb.height * Double(H)).rounded())
+    let py = Int(((1 - bb.maxY) * Double(H)).rounded())
+    guard px >= 0, py >= 0, px + pw <= W, py + ph <= H, pw > 2, ph > 2 else { continue }
+    // a trailing chevron or a neighboring icon is swept into the line: keep only the groups of word tokens
+    let tokens = cand.string.split(separator: " ", omittingEmptySubsequences: true)
+    if let first = tokens.firstIndex(where: isWord), let last = tokens.lastIndex(where: isWord), (first > 0 || last < tokens.count - 1) {
+        let groups = tokenGroups(x: px, y: py, w: pw, h: ph, tokenLengths: tokens.map { $0.count })
+        if groups.count == tokens.count {
+            let from = groups[first].0, to = groups[last].1
+            text = tokens[first...last].joined(separator: " ")
+            px += from; pw = to - from + 1
+        }
+    }
+    let targetPt = Double(pw) / scale
+    let crop = cropCoverage(x: px, y: py, w: pw, h: ph)
+    var fits = [Candidate]()
+    let cropStem = stemWidth(gray, stride: W, x: px, y: py, w: pw, h: ph)
+    for (name, w) in weights {
+        let size = solveSize(text, targetPt: targetPt, weight: w)
+        let r = renderCoverage(text, sizePt: size, weight: w, boxW: pw, boxH: ph)
+        fits.append(Candidate(weight: name, sizePt: (size * 1000).rounded() / 1000, coverage: (r.coverage * 1000).rounded() / 1000, stemPx: (r.stem * 100).rounded() / 100))
+    }
+    pending.append(Pending(text: text, conf: cand.confidence, px: px, py: py, pw: pw, ph: ph, crop: crop, cropStem: cropStem, inkH: inkHeight(gray, stride: W, x: px, y: py, w: pw, h: ph), fits: fits))
+}
+
+var lines = [RawLine]()
+for p in pending {
+    lines.append(RawLine(text: p.text, confidence: p.conf,
+                         x: (Double(p.px) / scale * 10).rounded() / 10, y: (Double(p.py) / scale * 10).rounded() / 10,
+                         w: (Double(p.pw) / scale * 10).rounded() / 10, h: (Double(p.ph) / scale * 10).rounded() / 10,
+                         cropStemPx: (p.cropStem * 100).rounded() / 100, inkHeightPt: (p.inkH / scale * 100).rounded() / 100,
+                         cropCoverage: (p.crop * 1000).rounded() / 1000, candidates: p.fits))
+}
+lines.sort { ($0.y, $0.x) < ($1.y, $1.x) }
+let enc = JSONEncoder(); enc.outputFormatting = [.prettyPrinted, .sortedKeys]
+print(String(data: try! enc.encode(lines), encoding: .utf8)!)