@insidepics/expo-apple-intelligence 0.1.0

package/ios/ExpoAppleIntelligenceModule.swift

@@ -0,0 +1,352 @@
+import ExpoModulesCore
+import Vision
+
+#if canImport(UIKit)
+import UIKit
+#endif
+
+public class ExpoAppleIntelligenceModule: Module {
+  public func definition() -> ModuleDefinition {
+    Name("ExpoAppleIntelligence")
+
+    // MARK: - Batch analysis
+
+    AsyncFunction("analyzeImage") { (imagePath: String, promise: Promise) in
+      guard let (cgImage, width, height) = VisionHelpers.loadImage(imagePath) else {
+        promise.reject("ERR_LOAD_IMAGE", "Could not load image at \(imagePath)")
+        return
+      }
+
+      DispatchQueue.global(qos: .userInitiated).async {
+        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
+
+        let faceLandmarksReq = VNDetectFaceLandmarksRequest()
+        let faceQualityReq = VNDetectFaceCaptureQualityRequest()
+        let textReq = VNRecognizeTextRequest()
+        textReq.recognitionLevel = .accurate
+        let classifyReq = VNClassifyImageRequest()
+        let barcodeReq = VNDetectBarcodesRequest()
+        let bodyPoseReq = VNDetectHumanBodyPoseRequest()
+        let handPoseReq = VNDetectHumanHandPoseRequest()
+        handPoseReq.maximumHandCount = 4
+        let featurePrintReq = VNGenerateImageFeaturePrintRequest()
+        let attentionReq = VNGenerateAttentionBasedSaliencyImageRequest()
+        let objectnessReq = VNGenerateObjectnessBasedSaliencyImageRequest()
+        let animalReq = VNRecognizeAnimalsRequest()
+        let rectangleReq = VNDetectRectanglesRequest()
+        rectangleReq.maximumObservations = 10
+        let horizonReq = VNDetectHorizonRequest()
+
+        var requests: [VNRequest] = [
+          faceLandmarksReq, faceQualityReq, textReq, classifyReq, barcodeReq,
+          bodyPoseReq, handPoseReq, featurePrintReq, attentionReq, objectnessReq,
+          animalReq, rectangleReq, horizonReq,
+        ]
+
+        var aestheticsReq: VNRequest? = nil
+        if #available(iOS 18.0, macOS 15.0, *) {
+          let req = VNCalculateImageAestheticsScoresRequest()
+          aestheticsReq = req
+          requests.append(req)
+        }
+
+        do { try handler.perform(requests) } catch {
+          promise.reject("ERR_ANALYSIS", error.localizedDescription)
+          return
+        }
+
+        let faces = VisionProcessors.processFaces(landmarks: faceLandmarksReq.results ?? [], quality: faceQualityReq.results ?? [])
+        let text = VisionProcessors.processText(textReq.results ?? [])
+        let labels = VisionProcessors.processClassification(classifyReq.results ?? [])
+        let barcodes = VisionProcessors.processBarcodes(barcodeReq.results ?? [])
+        let bodyPoses = VisionProcessors.processBodyPoses(bodyPoseReq.results ?? [])
+        let handPoses = VisionProcessors.processHandPoses(handPoseReq.results ?? [])
+        let featurePrint = VisionProcessors.processFeaturePrint(featurePrintReq.results?.first)
+        let attentionRegions = VisionProcessors.processSaliency(attentionReq.results ?? [])
+        let objectnessRegions = VisionProcessors.processSaliency(objectnessReq.results ?? [])
+        let animals = VisionProcessors.processAnimals(animalReq.results ?? [])
+        let rectangles = VisionProcessors.processRectangles(rectangleReq.results ?? [])
+        let horizon = VisionProcessors.processHorizon(horizonReq.results?.first)
+
+        var aesthetics: [String: Any]? = nil
+        if #available(iOS 18.0, macOS 15.0, *) {
+          aesthetics = VisionModern.processAesthetics(aestheticsReq?.results)
+        }
+
+        var result: [String: Any] = [
+          "faces": faces, "text": text, "labels": labels, "barcodes": barcodes,
+          "bodyPoses": bodyPoses, "handPoses": handPoses, "rectangles": rectangles,
+          "animals": animals,
+          "featurePrint": featurePrint ?? [:] as [String: Any],
+          "aesthetics": aesthetics ?? [:] as [String: Any],
+          "saliency": ["attentionRegions": attentionRegions, "objectnessRegions": objectnessRegions],
+          "horizon": horizon ?? [:] as [String: Any],
+          "lensSmudge": [:] as [String: Any],
+          "document": [:] as [String: Any],
+          "imageWidth": Double(width), "imageHeight": Double(height),
+        ]
+        #if os(iOS)
+        result["platform"] = "ios"
+        #else
+        result["platform"] = "macos"
+        #endif
+
+        #if os(iOS)
+        if #available(iOS 26.0, *) {
+          Task {
+            let lensSmudge = await VisionModern.detectLensSmudge(cgImage: cgImage)
+            let document = await VisionModern.recognizeDocument(cgImage: cgImage)
+            var final26 = result
+            if let ls = lensSmudge { final26["lensSmudge"] = ls }
+            if let doc = document { final26["document"] = doc }
+            promise.resolve(final26)
+          }
+        } else {
+          promise.resolve(result)
+        }
+        #elseif os(macOS)
+        if #available(macOS 26.0, *) {
+          Task {
+            let lensSmudge = await VisionModern.detectLensSmudge(cgImage: cgImage)
+            let document = await VisionModern.recognizeDocument(cgImage: cgImage)
+            var final26 = result
+            if let ls = lensSmudge { final26["lensSmudge"] = ls }
+            if let doc = document { final26["document"] = doc }
+            promise.resolve(final26)
+          }
+        } else {
+          promise.resolve(result)
+        }
+        #endif
+      }
+    }
+
+    // MARK: - Individual functions
+
+    AsyncFunction("detectFaces") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let lReq = VNDetectFaceLandmarksRequest()
+        let qReq = VNDetectFaceCaptureQualityRequest()
+        try handler.perform([lReq, qReq])
+        return VisionProcessors.processFaces(landmarks: lReq.results ?? [], quality: qReq.results ?? [])
+      }
+    }
+
+    AsyncFunction("recognizeText") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNRecognizeTextRequest()
+        req.recognitionLevel = .accurate
+        try handler.perform([req])
+        return VisionProcessors.processText(req.results ?? [])
+      }
+    }
+
+    AsyncFunction("classifyImage") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNClassifyImageRequest()
+        try handler.perform([req])
+        return VisionProcessors.processClassification(req.results ?? [])
+      }
+    }
+
+    AsyncFunction("detectBarcodes") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNDetectBarcodesRequest()
+        try handler.perform([req])
+        return VisionProcessors.processBarcodes(req.results ?? [])
+      }
+    }
+
+    AsyncFunction("detectBodyPoses") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNDetectHumanBodyPoseRequest()
+        try handler.perform([req])
+        return VisionProcessors.processBodyPoses(req.results ?? [])
+      }
+    }
+
+    AsyncFunction("detectHandPoses") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNDetectHumanHandPoseRequest()
+        req.maximumHandCount = 4
+        try handler.perform([req])
+        return VisionProcessors.processHandPoses(req.results ?? [])
+      }
+    }
+
+    AsyncFunction("generateFeaturePrint") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNGenerateImageFeaturePrintRequest()
+        try handler.perform([req])
+        return VisionProcessors.processFeaturePrint(req.results?.first) as Any
+      }
+    }
+
+    AsyncFunction("calculateAesthetics") { (imagePath: String, promise: Promise) in
+      guard #available(iOS 18.0, macOS 15.0, *) else {
+        promise.resolve(nil)
+        return
+      }
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNCalculateImageAestheticsScoresRequest()
+        try handler.perform([req])
+        return VisionModern.processAesthetics(req.results) as Any
+      }
+    }
+
+    AsyncFunction("detectSaliency") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let aReq = VNGenerateAttentionBasedSaliencyImageRequest()
+        let oReq = VNGenerateObjectnessBasedSaliencyImageRequest()
+        try handler.perform([aReq, oReq])
+        return [
+          "attentionRegions": VisionProcessors.processSaliency(aReq.results ?? []),
+          "objectnessRegions": VisionProcessors.processSaliency(oReq.results ?? []),
+        ]
+      }
+    }
+
+    AsyncFunction("detectAnimals") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNRecognizeAnimalsRequest()
+        try handler.perform([req])
+        return VisionProcessors.processAnimals(req.results ?? [])
+      }
+    }
+
+    AsyncFunction("detectRectangles") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNDetectRectanglesRequest()
+        req.maximumObservations = 10
+        try handler.perform([req])
+        return VisionProcessors.processRectangles(req.results ?? [])
+      }
+    }
+
+    AsyncFunction("detectHorizon") { (imagePath: String, promise: Promise) in
+      Self.runVision(imagePath, promise: promise) { handler in
+        let req = VNDetectHorizonRequest()
+        try handler.perform([req])
+        return VisionProcessors.processHorizon(req.results?.first) as Any
+      }
+    }
+
+    AsyncFunction("detectLensSmudge") { (imagePath: String, promise: Promise) in
+      if #available(iOS 26.0, macOS 26.0, *) {
+        guard let (cgImage, _, _) = VisionHelpers.loadImage(imagePath) else {
+          promise.reject("ERR_LOAD_IMAGE", "Could not load image at \(imagePath)")
+          return
+        }
+        Task {
+          let result = await VisionModern.detectLensSmudge(cgImage: cgImage)
+          promise.resolve(result)
+        }
+      } else {
+        promise.resolve(nil)
+      }
+    }
+
+    AsyncFunction("recognizeDocument") { (imagePath: String, promise: Promise) in
+      if #available(iOS 26.0, macOS 26.0, *) {
+        guard let (cgImage, _, _) = VisionHelpers.loadImage(imagePath) else {
+          promise.reject("ERR_LOAD_IMAGE", "Could not load image at \(imagePath)")
+          return
+        }
+        Task {
+          let result = await VisionModern.recognizeDocument(cgImage: cgImage)
+          promise.resolve(result)
+        }
+      } else {
+        promise.resolve(nil)
+      }
+    }
+
+    // MARK: - Image decoding
+
+    AsyncFunction("decodeImagePixels") { (imagePath: String, promise: Promise) in
+      guard let (cgImage, _, _) = VisionHelpers.loadImage(imagePath) else {
+        promise.reject("ERR_LOAD_IMAGE", "Could not load image at \(imagePath)")
+        return
+      }
+      DispatchQueue.global(qos: .userInitiated).async {
+        guard let data = VisionHelpers.decodePixels(cgImage) else {
+          promise.reject("ERR_DECODE", "Failed to decode pixels")
+          return
+        }
+        promise.resolve([
+          "pixels": data.base64EncodedString(),
+          "width": Double(cgImage.width),
+          "height": Double(cgImage.height),
+        ])
+      }
+    }
+
+    // MARK: - Foundation Models (iOS 26+)
+
+    Function("isFoundationModelAvailable") { () -> Bool in
+      if #available(iOS 26.0, *) {
+        return VisionModern.isFoundationModelAvailable()
+      }
+      return false
+    }
+
+    AsyncFunction("generateText") { (prompt: String, systemPrompt: String?, promise: Promise) in
+      if #available(iOS 26.0, *) {
+        Task {
+          let result = await VisionModern.generateText(prompt: prompt, systemPrompt: systemPrompt)
+          promise.resolve(result)
+        }
+      } else {
+        promise.resolve(nil)
+      }
+    }
+
+    // MARK: - Speech Transcription (iOS 26+)
+
+    AsyncFunction("transcribeAudio") { (audioPath: String, locale: String?, promise: Promise) in
+      if #available(iOS 26.0, *) {
+        Task {
+          let result = await VisionModern.transcribeAudio(audioPath: audioPath, locale: locale)
+          promise.resolve(result)
+        }
+      } else {
+        promise.resolve(nil)
+      }
+    }
+
+    // MARK: - Image Generation (iOS 18.4+)
+
+    AsyncFunction("generateImage") { (prompt: String, style: String?, promise: Promise) in
+      if #available(iOS 18.4, *) {
+        Task {
+          let result = await VisionModern.generateImage(prompt: prompt, style: style)
+          promise.resolve(result)
+        }
+      } else {
+        promise.resolve(nil)
+      }
+    }
+  }
+
+  // MARK: - Helper to reduce boilerplate
+
+  private static func runVision(
+    _ imagePath: String,
+    promise: Promise,
+    work: @escaping (VNImageRequestHandler) throws -> Any
+  ) {
+    guard let (cgImage, _, _) = VisionHelpers.loadImage(imagePath) else {
+      promise.reject("ERR_LOAD_IMAGE", "Could not load image at \(imagePath)")
+      return
+    }
+    DispatchQueue.global(qos: .userInitiated).async {
+      let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
+      do {
+        let result = try work(handler)
+        promise.resolve(result)
+      } catch {
+        promise.reject("ERR_VISION", error.localizedDescription)
+      }
+    }
+  }
+}

package/ios/VisionHelpers.swift

@@ -0,0 +1,62 @@
+import Vision
+
+#if canImport(UIKit)
+import UIKit
+#elseif canImport(AppKit)
+import AppKit
+#endif
+
+/// Shared helpers for image loading and coordinate serialization.
+enum VisionHelpers {
+  static func loadImage(_ path: String) -> (CGImage, CGFloat, CGFloat)? {
+    let url: URL
+    if path.hasPrefix("file://") {
+      guard let parsed = URL(string: path) else { return nil }
+      url = parsed
+    } else {
+      url = URL(fileURLWithPath: path)
+    }
+    guard let data = try? Data(contentsOf: url) else { return nil }
+
+    #if canImport(UIKit)
+    guard let uiImage = UIImage(data: data),
+          let cgImage = uiImage.cgImage else { return nil }
+    #elseif canImport(AppKit)
+    guard let nsImage = NSImage(data: data),
+          let cgImage = nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil) else { return nil }
+    #endif
+
+    return (cgImage, CGFloat(cgImage.width), CGFloat(cgImage.height))
+  }
+
+  /// Serialize CGRect as-is (normalized 0-1, bottom-left origin).
+  static func rawRect(_ rect: CGRect) -> [String: Double] {
+    ["x": Double(rect.origin.x), "y": Double(rect.origin.y),
+     "width": Double(rect.size.width), "height": Double(rect.size.height)]
+  }
+
+  /// Serialize CGPoint as-is (normalized 0-1, bottom-left origin).
+  static func rawPoint(_ point: CGPoint) -> [String: Double] {
+    ["x": Double(point.x), "y": Double(point.y)]
+  }
+
+  /// Decode a CGImage to raw RGBA pixel data.
+  static func decodePixels(_ cgImage: CGImage) -> Data? {
+    let w = cgImage.width
+    let h = cgImage.height
+    let bytesPerRow = w * 4
+    var pixels = [UInt8](repeating: 0, count: w * h * 4)
+
+    guard let ctx = CGContext(
+      data: &pixels,
+      width: w, height: h,
+      bitsPerComponent: 8,
+      bytesPerRow: bytesPerRow,
+      space: CGColorSpaceCreateDeviceRGB(),
+      bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+    ) else { return nil }
+
+    ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: w, height: h))
+    return Data(pixels)
+  }
+}

package/ios/VisionModern.swift

@@ -0,0 +1,239 @@
+import Vision
+import CoreImage
+import AVFAudio
+
+#if canImport(UIKit)
+import UIKit
+#endif
+
+#if canImport(FoundationModels)
+import FoundationModels
+#endif
+
+#if canImport(Speech)
+import Speech
+#endif
+
+#if canImport(ImagePlayground)
+import ImagePlayground
+#endif
+
+/// iOS 18+/macOS 15+ and iOS 26+/macOS 26+ Vision APIs (Swift-native, async/await).
+enum VisionModern {
+
+  // MARK: - Aesthetics (iOS 18+ / macOS 15+)
+
+  @available(iOS 18.0, macOS 15.0, *)
+  static func processAesthetics(_ results: [Any]?) -> [String: Any]? {
+    guard let obs = results?.first as? VNImageAestheticsScoresObservation else { return nil }
+    return ["overallScore": Double(obs.overallScore), "isUtility": obs.isUtility]
+  }
+
+  // MARK: - Lens Smudge (iOS 26+ / macOS 26+)
+
+  @available(iOS 26.0, macOS 26.0, *)
+  static func detectLensSmudge(cgImage: CGImage) async -> [String: Any]? {
+    let ciImage = CIImage(cgImage: cgImage)
+    let request = DetectLensSmudgeRequest()
+    guard let obs = try? await request.perform(on: ciImage) else { return nil }
+    return ["confidence": obs.confidence]
+  }
+
+  // MARK: - Document Recognition (iOS 26+ / macOS 26+)
+
+  @available(iOS 26.0, macOS 26.0, *)
+  static func recognizeDocument(cgImage: CGImage) async -> [String: Any]? {
+    let ciImage = CIImage(cgImage: cgImage)
+    let request = RecognizeDocumentsRequest()
+    guard let docObs = try? await request.perform(on: ciImage).first else { return nil }
+    let doc = docObs.document
+    return processDocument(doc)
+  }
+
+  @available(iOS 26.0, macOS 26.0, *)
+  private static func processDocument(_ doc: DocumentObservation.Container) -> [String: Any] {
+    let paragraphs: [[String: Any]] = doc.paragraphs.map { para in
+      var detectedData: [[String: Any]] = []
+      for item in para.detectedData {
+        detectedData.append([
+          "type": detectedDataType(item),
+          "value": "\(item.match)",
+          "boundingBox": VisionHelpers.rawRect(item.boundingRegion.boundingBox.cgRect),
+        ])
+      }
+      return [
+        "text": para.transcript,
+        "boundingBox": VisionHelpers.rawRect(para.boundingRegion.boundingBox.cgRect),
+        "detectedData": detectedData,
+      ]
+    }
+
+    let tables: [[String: Any]] = doc.tables.enumerated().map { (_, table) in
+      var cells: [[String: Any]] = []
+      for (rowIdx, row) in table.rows.enumerated() {
+        for (colIdx, cell) in row.enumerated() {
+          let text = cell.content.paragraphs.map { $0.transcript }.joined(separator: "\n")
+          cells.append(["text": text, "row": rowIdx, "column": colIdx])
+        }
+      }
+      return [
+        "cells": cells,
+        "boundingBox": VisionHelpers.rawRect(table.boundingRegion.boundingBox.cgRect),
+      ]
+    }
+
+    let lists: [[String: Any]] = doc.lists.map { list in
+      let items: [[String: Any]] = list.items.map { item in
+        [
+          "text": item.itemString,
+          "marker": item.markerString,
+        ]
+      }
+      return [
+        "items": items,
+        "boundingBox": VisionHelpers.rawRect(list.boundingRegion.boundingBox.cgRect),
+      ]
+    }
+
+    let barcodes: [[String: Any]] = doc.barcodes.map { barcode in
+      [
+        "value": barcode.payloadString ?? "",
+        "symbology": "\(barcode.symbology)",
+        "boundingBox": VisionHelpers.rawRect(barcode.boundingRegion.boundingBox.cgRect),
+      ]
+    }
+
+    return ["paragraphs": paragraphs, "tables": tables, "lists": lists, "barcodes": barcodes]
+  }
+
+  @available(iOS 26.0, macOS 26.0, *)
+  private static func detectedDataType(_ data: DocumentObservation.Container.DataDetectorMatch) -> String {
+    switch data.match.details {
+    case .emailAddress: return "email"
+    case .phoneNumber: return "phoneNumber"
+    case .link: return "url"
+    case .calendarEvent: return "calendarEvent"
+    case .postalAddress: return "postalAddress"
+    case .moneyAmount: return "moneyAmount"
+    case .measurement: return "measurement"
+    default: return "unknown"
+    }
+  }
+
+  // MARK: - Foundation Models (iOS 26+)
+
+  @available(iOS 26.0, *)
+  static func isFoundationModelAvailable() -> Bool {
+    #if canImport(FoundationModels)
+    return SystemLanguageModel.default.availability == .available
+    #else
+    return false
+    #endif
+  }
+
+  @available(iOS 26.0, *)
+  static func generateText(prompt: String, systemPrompt: String?) async -> String? {
+    #if canImport(FoundationModels)
+    guard SystemLanguageModel.default.availability == .available else { return nil }
+    do {
+      let session: LanguageModelSession
+      if let systemPrompt = systemPrompt {
+        session = LanguageModelSession(instructions: systemPrompt)
+      } else {
+        session = LanguageModelSession()
+      }
+      let response = try await session.respond(to: prompt)
+      return response.content
+    } catch {
+      return nil
+    }
+    #else
+    return nil
+    #endif
+  }
+
+  // MARK: - Speech Transcription (iOS 26+)
+
+  @available(iOS 26.0, *)
+  static func transcribeAudio(audioPath: String, locale: String?) async -> [String: Any]? {
+    #if canImport(Speech)
+    let url: URL
+    if audioPath.hasPrefix("file://") {
+      guard let parsed = URL(string: audioPath) else { return nil }
+      url = parsed
+    } else {
+      url = URL(fileURLWithPath: audioPath)
+    }
+
+    guard let audioFile = try? AVAudioFile(forReading: url) else { return nil }
+
+    let loc = locale != nil ? Locale(identifier: locale!) : .current
+
+    do {
+      let transcriber = SpeechTranscriber(locale: loc, preset: .transcription)
+      _ = try await SpeechAnalyzer(
+        inputAudioFile: audioFile,
+        modules: [transcriber],
+        finishAfterFile: true
+      )
+
+      var segments: [[String: Any]] = []
+      for try await result in transcriber.results {
+        let text = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines)
+        if !text.isEmpty {
+          segments.append(["text": text])
+        }
+      }
+
+      if segments.isEmpty { return nil }
+      return ["segments": segments]
+    } catch {
+      return nil
+    }
+    #else
+    return nil
+    #endif
+  }
+
+  // MARK: - Image Generation (iOS 18.4+)
+
+  @available(iOS 18.4, *)
+  static func generateImage(prompt: String, style: String?) async -> String? {
+    #if canImport(ImagePlayground)
+    do {
+      let creator = try await ImageCreator()
+
+      let imageStyle: ImagePlaygroundStyle
+      switch style {
+      case "illustration": imageStyle = .illustration
+      case "sketch": imageStyle = .sketch
+      default: imageStyle = .animation
+      }
+
+      var resultImage: CGImage? = nil
+      for try await image in creator.images(for: [.text(prompt)], style: imageStyle, limit: 1) {
+        resultImage = image.cgImage
+        break
+      }
+
+      guard let cgImage = resultImage else { return nil }
+
+      let fileName = UUID().uuidString + ".jpg"
+      let filePath = FileManager.default.temporaryDirectory.appendingPathComponent(fileName)
+
+      #if canImport(UIKit)
+      guard let data = UIImage(cgImage: cgImage).jpegData(compressionQuality: 0.9) else { return nil }
+      try data.write(to: filePath)
+      #else
+      return nil
+      #endif
+
+      return filePath.path
+    } catch {
+      return nil
+    }
+    #else
+    return nil
+    #endif
+  }
+}