@dr33m/react-native-litert-lm 0.5.0

package/ios/HybridLiteRTLM+Execute.swift

@@ -0,0 +1,287 @@
+//
+//  HybridLiteRTLM+Execute.swift
+//  react-native-litert-lm
+//
+//  Unified inference pipeline (execute). JS routes via src/inferenceRouting.ts;
+//  legacy send* methods below delegate here for direct native hybrid access.
+//
+
+import Foundation
+import UIKit
+import NitroModules
+import CLiteRTLM
+
+
+
+// MARK: - Preprocessed parts for thread-safety
+
+struct PreprocessedPart {
+    enum Kind {
+        case text(String)
+        case imagePath(String)
+        case imageData(Data)
+        case audioPath(String)
+        case audioData(Data)
+    }
+    let kind: Kind
+    let label: String
+}
+
+// MARK: - Execute pipeline
+
+extension HybridLiteRTLM {
+
+    /// Only used when legacy native *Async methods are invoked directly (not via createLLM JS proxy).
+    func executeVoid(
+        parts: [MultimodalPart],
+        onToken: @escaping (_ token: String, _ done: Bool) -> Void
+    ) throws -> Promise<Void> {
+        let voidPromise = Promise<Void>()
+        let resultPromise = try execute(parts: parts, onToken: onToken)
+        resultPromise
+            .then { _ in voidPromise.resolve() }
+            .catch { voidPromise.reject(withError: $0) }
+        return voidPromise
+    }
+
+    public func execute(parts: [MultimodalPart], onToken: ((_ token: String, _ done: Bool) -> Void)?) throws -> Promise<ExecuteResult> {
+        let promise = Promise<ExecuteResult>()
+
+        // Preprocess all JSI-bound data on the caller thread synchronously
+        var preprocessed: [PreprocessedPart] = []
+        for part in parts {
+            switch part.type {
+            case .text:
+                let txt = part.text ?? ""
+                preprocessed.append(PreprocessedPart(kind: .text(txt), label: txt))
+            case .image:
+                if let path = part.path {
+                    preprocessed.append(PreprocessedPart(kind: .imagePath(path), label: "[Image]"))
+                } else if let buf = part.imageBuffer {
+                    let data = buf.toData(copyIfNeeded: true)
+                    preprocessed.append(PreprocessedPart(kind: .imageData(data), label: "[Image]"))
+                }
+            case .audio:
+                if let path = part.path {
+                    preprocessed.append(PreprocessedPart(kind: .audioPath(path), label: "[Audio]"))
+                } else if let buf = part.audioBuffer {
+                    let data = buf.toData(copyIfNeeded: true)
+                    preprocessed.append(PreprocessedPart(kind: .audioData(data), label: "[Audio]"))
+                }
+            }
+        }
+
+        let userLabel = preprocessed.map { $0.label }.joined(separator: " ").trimmingCharacters(in: .whitespaces)
+
+        queue.async {
+            guard let conversation = self.conversation else {
+                promise.reject(withError: NSError(domain: "LiteRTLM", code: 400,
+                    userInfo: [NSLocalizedDescriptionKey: "LiteRTLM: No model loaded. Call loadModel() first."]))
+                return
+            }
+
+            let payload: (json: String, tempFiles: [String])
+            do { payload = try self.buildExecutePayload(preprocessed) }
+            catch { promise.reject(withError: error); return }
+
+            let msgJson = payload.json
+            let tempFiles = payload.tempFiles
+            let cleanup = { tempFiles.forEach { try? FileManager.default.removeItem(atPath: $0) } }
+
+            if let onToken = onToken {
+                self.runExecuteStreaming(
+                    conversation: conversation,
+                    msgJson: msgJson,
+                    userLabel: userLabel,
+                    onToken: onToken,
+                    promise: promise,
+                    cleanup: cleanup
+                )
+            } else {
+                self.runExecuteBlocking(
+                    conversation: conversation,
+                    msgJson: msgJson,
+                    userLabel: userLabel,
+                    promise: promise,
+                    cleanup: cleanup
+                )
+            }
+        }
+
+        return promise
+    }
+
+    // MARK: - Payload / media
+
+    private func validateMediaPath(_ path: String, label: String) throws {
+        if !FileManager.default.fileExists(atPath: path) {
+            throw NSError(
+                domain: "LiteRTLM", code: 404,
+                userInfo: [NSLocalizedDescriptionKey: "\(label) file not found: \(path)"]
+            )
+        }
+    }
+
+    private func buildExecutePayload(_ preprocessed: [PreprocessedPart]) throws -> (json: String, tempFiles: [String]) {
+        struct Desc { let kind: String; let text: String?; let file: String? }
+        var descs: [Desc] = []
+        var temps: [String] = []
+
+        do {
+            for part in preprocessed {
+                switch part.kind {
+                case .text(let text):
+                    descs.append(Desc(kind: "text", text: text, file: nil))
+                case .imagePath(let path):
+                    try validateMediaPath(path, label: "Image")
+                    let scaled = scaleImageIfNeeded(path)
+                    if scaled != path { temps.append(scaled) }
+                    descs.append(Desc(kind: "image", text: nil, file: scaled))
+                case .imageData(let data):
+                    let raw = try saveDataToTempFile(data, ext: "jpg")
+                    temps.append(raw)
+                    let scaled = scaleImageIfNeeded(raw)
+                    if scaled != raw { temps.append(scaled) }
+                    descs.append(Desc(kind: "image", text: nil, file: scaled))
+                case .audioPath(let path):
+                    try validateMediaPath(path, label: "Audio")
+                    descs.append(Desc(kind: "audio", text: nil, file: path))
+                case .audioData(let data):
+                    let raw = try saveDataToTempFile(data, ext: "wav")
+                    temps.append(raw)
+                    descs.append(Desc(kind: "audio", text: nil, file: raw))
+                }
+            }
+        } catch {
+            temps.forEach { try? FileManager.default.removeItem(atPath: $0) }
+            throw error
+        }
+
+        if descs.count == 1 && descs[0].kind == "text" {
+            let payload: [String: Any] = ["role": "user", "content": descs[0].text ?? ""]
+            let data = try JSONSerialization.data(withJSONObject: payload, options: [])
+            if let jsonString = String(data: data, encoding: .utf8) {
+                return (jsonString, temps)
+            }
+        }
+
+        var contents: [[String: String]] = []
+        for d in descs {
+            if d.kind == "text" {
+                contents.append(["type": "text", "text": d.text ?? ""])
+            } else if d.kind == "image" {
+                contents.append(["type": "image", "path": d.file ?? ""])
+            } else {
+                contents.append(["type": "audio", "path": d.file ?? ""])
+            }
+        }
+
+        let payload: [String: Any] = ["role": "user", "content": contents]
+        let data = try JSONSerialization.data(withJSONObject: payload, options: [])
+        guard let jsonString = String(data: data, encoding: .utf8) else {
+            throw NSError(domain: "LiteRTLM", code: 500, userInfo: [NSLocalizedDescriptionKey: "Failed to encode JSON payload"])
+        }
+        return (jsonString, temps)
+    }
+
+    private func scaleImageIfNeeded(_ imagePath: String, maxDimension: Int = 1024) -> String {
+        guard let image = UIImage(contentsOfFile: imagePath) else { return imagePath }
+        let w = Int(image.size.width), h = Int(image.size.height)
+        guard max(w, h) > maxDimension else { return imagePath }
+        let scale = CGFloat(maxDimension) / CGFloat(max(w, h))
+        let newSize = CGSize(width: CGFloat(w) * scale, height: CGFloat(h) * scale)
+        let renderer = UIGraphicsImageRenderer(size: newSize)
+        let scaled = renderer.image { _ in image.draw(in: CGRect(origin: .zero, size: newSize)) }
+        let tmp = (NSTemporaryDirectory() as NSString)
+            .appendingPathComponent("litert_scaled_\(UUID().uuidString).jpg")
+        if let data = scaled.jpegData(compressionQuality: 0.9) {
+            do {
+                try data.write(to: URL(fileURLWithPath: tmp))
+                return tmp
+            } catch {
+                NSLog("[LiteRTLM] Failed to write scaled image to temp file: \(error.localizedDescription)")
+                return imagePath
+            }
+        }
+        return imagePath
+    }
+
+    private func saveDataToTempFile(_ data: Data, ext: String) throws -> String {
+        let tmp = (NSTemporaryDirectory() as NSString)
+            .appendingPathComponent("litert_buf_\(UUID().uuidString).\(ext)")
+        try data.write(to: URL(fileURLWithPath: tmp))
+        return tmp
+    }
+
+    // MARK: - Streaming / blocking runners
+
+    private func runExecuteBlocking(
+        conversation: OpaquePointer,
+        msgJson: String,
+        userLabel: String,
+        promise: Promise<ExecuteResult>,
+        cleanup: @escaping () -> Void
+    ) {
+        let startTime = Date()
+        guard let response = litert_lm_conversation_send_message(conversation, msgJson, nil, nil) else {
+            cleanup()
+            promise.reject(withError: NSError(domain: "LiteRTLM", code: 500,
+                userInfo: [NSLocalizedDescriptionKey: "LiteRTLM: execute failed."]))
+            return
+        }
+        defer { litert_lm_json_response_delete(response) }
+
+        var result = ""
+        if let rs = litert_lm_json_response_get_string(response) {
+            result = extractTextFromResponse(String(cString: rs))
+                .trimmingCharacters(in: .whitespacesAndNewlines)
+        }
+
+        commitExecuteTurn(
+            userLabel: userLabel,
+            modelResponse: result,
+            startTime: startTime,
+            conversation: conversation,
+            tokenCount: 0
+        )
+        cleanup()
+        // iOS: tool calls not yet captured from C API — return empty array
+        promise.resolve(withResult: ExecuteResult(text: result, toolCalls: [], thinkingText: ""))
+    }
+
+
+
+    func commitExecuteTurn(
+        userLabel: String,
+        modelResponse: String,
+        startTime: Date,
+        conversation: OpaquePointer?,
+        tokenCount: Int
+    ) {
+        let totalTime = Date().timeIntervalSince(startTime)
+        var compTok = Double(tokenCount)
+        var tps = 0.0, ttft = 0.0
+        if let conversation = conversation,
+           let bi = litert_lm_conversation_get_benchmark_info(conversation) {
+            let turns = litert_lm_benchmark_info_get_num_decode_turns(bi)
+            if turns > 0 {
+                let li = turns - 1
+                tps = litert_lm_benchmark_info_get_decode_tokens_per_sec_at(bi, li)
+                compTok = Double(litert_lm_benchmark_info_get_decode_token_count_at(bi, li))
+            }
+            ttft = litert_lm_benchmark_info_get_time_to_first_token(bi)
+            litert_lm_benchmark_info_delete(bi)
+        }
+        let pt = Double(userLabel.count) / 4.0
+        if compTok == 0.0 { compTok = Double(modelResponse.count) / 4.0 }
+        lastStats = GenerationStats(
+            promptTokens: pt, completionTokens: compTok,
+            totalTokens: pt + compTok, timeToFirstToken: ttft,
+            totalTime: totalTime,
+            tokensPerSecond: tps > 0 ? tps : (compTok / totalTime)
+        )
+        history.append(Message(role: .user, content: userLabel))
+        history.append(Message(role: .model, content: modelResponse))
+    }
+}
+

package/ios/HybridLiteRTLM+Streaming.swift

@@ -0,0 +1,153 @@
+//
+//  HybridLiteRTLM+Streaming.swift
+//  react-native-litert-lm
+//
+//  Streaming context and callback runners for the unified execute pipeline.
+//
+
+import Foundation
+import NitroModules
+import CLiteRTLM
+
+class ExecuteStreamContext {
+    let userLabel: String
+    let startTime: Date
+    let onToken: (_ token: String, _ done: Bool) -> Void
+    let promise: Promise<ExecuteResult>
+    let parent: HybridLiteRTLM
+    let cleanup: () -> Void
+    var rawResponse: String = ""
+    var fullResponse: String = ""
+    var lastEmittedLength: Int = 0
+    var tokenCount: Int = 0
+
+    init(
+        userLabel: String,
+        startTime: Date,
+        onToken: @escaping (_ token: String, _ done: Bool) -> Void,
+        promise: Promise<ExecuteResult>,
+        parent: HybridLiteRTLM,
+        cleanup: @escaping () -> Void
+    ) {
+        self.userLabel = userLabel
+        self.startTime = startTime
+        self.onToken = onToken
+        self.promise = promise
+        self.parent = parent
+        self.cleanup = cleanup
+    }
+}
+
+extension HybridLiteRTLM {
+
+    func runExecuteStreaming(
+        conversation: OpaquePointer,
+        msgJson: String,
+        userLabel: String,
+        onToken: @escaping (_ token: String, _ done: Bool) -> Void,
+        promise: Promise<ExecuteResult>,
+        cleanup: @escaping () -> Void
+    ) {
+        let ctx = ExecuteStreamContext(
+            userLabel: userLabel, startTime: Date(),
+            onToken: onToken, promise: promise, parent: self,
+            cleanup: cleanup
+        )
+        let ptr = Unmanaged.passRetained(ctx).toOpaque()
+
+        let cb: LiteRtLmStreamCallback = { ptr, chunk, isFinal, errorMsg in
+            guard let ptr = ptr else { return }
+            let ctx = Unmanaged<ExecuteStreamContext>.fromOpaque(ptr).takeUnretainedValue()
+
+            if let errorMsg = errorMsg {
+                let msg = String(cString: errorMsg)
+                ctx.onToken("Error: \(msg)", true)
+                ctx.cleanup()
+                ctx.promise.reject(withError: NSError(domain: "LiteRTLM", code: 500,
+                    userInfo: [NSLocalizedDescriptionKey: msg]))
+                Unmanaged<ExecuteStreamContext>.fromOpaque(ptr).release()
+                return
+            }
+
+            if isFinal {
+                ctx.parent.finalizeExecuteStream(ctx: ctx, streamPtr: ptr)
+                return
+            }
+
+            if let chunk = chunk {
+                ctx.parent.emitExecuteStreamChunk(ctx: ctx, chunk: chunk)
+            }
+        }
+
+        let status = litert_lm_conversation_send_message_stream(
+            conversation, msgJson, nil, nil, cb, ptr)
+        if status != 0 {
+            Unmanaged<ExecuteStreamContext>.fromOpaque(ptr).release()
+            cleanup()
+            promise.reject(withError: NSError(domain: "LiteRTLM", code: Int(status),
+                userInfo: [NSLocalizedDescriptionKey: "LiteRTLM: execute streaming failed."]))
+        }
+    }
+
+    func finalizeExecuteStream(ctx: ExecuteStreamContext, streamPtr: UnsafeMutableRawPointer) {
+        let cleaned = stripControlTokens(ctx.rawResponse)
+            .trimmingCharacters(in: .whitespacesAndNewlines)
+        var finalText = cleaned
+        if !ctx.userLabel.isEmpty && finalText.hasPrefix(ctx.userLabel) {
+            finalText = String(finalText.dropFirst(ctx.userLabel.count))
+                .trimmingCharacters(in: .whitespacesAndNewlines)
+        }
+        if finalText.count > ctx.lastEmittedLength {
+            let si = finalText.index(finalText.startIndex, offsetBy: ctx.lastEmittedLength)
+            ctx.onToken(String(finalText[si...]), false)
+        }
+        ctx.fullResponse = finalText
+
+        queue.async {
+            self.commitExecuteTurn(
+                userLabel: ctx.userLabel,
+                modelResponse: ctx.fullResponse,
+                startTime: ctx.startTime,
+                conversation: self.conversation,
+                tokenCount: ctx.tokenCount
+            )
+            ctx.onToken("", true)
+            ctx.cleanup()
+            // iOS: tool calls not yet captured from C API — return empty array
+            ctx.promise.resolve(withResult: ExecuteResult(text: ctx.fullResponse, toolCalls: [], thinkingText: ""))
+            Unmanaged<ExecuteStreamContext>.fromOpaque(streamPtr).release()
+        }
+    }
+
+    func emitExecuteStreamChunk(ctx: ExecuteStreamContext, chunk: UnsafePointer<CChar>) {
+        let token = String(cString: chunk)
+        let raw = token.hasPrefix("{") && token.contains("\"role\"")
+            ? extractTextFromResponse(token) : token
+        ctx.rawResponse += raw
+        let cleaned = stripControlTokens(ctx.rawResponse)
+            .trimmingLeadingCharacters(in: .whitespacesAndNewlines)
+        var processed = cleaned
+        if !ctx.userLabel.isEmpty && processed.hasPrefix(ctx.userLabel) {
+            processed = String(processed.dropFirst(ctx.userLabel.count))
+                .trimmingLeadingCharacters(in: .whitespacesAndNewlines)
+        }
+        let safe = safeEmitLength(processed)
+        if safe > ctx.lastEmittedLength {
+            let chars = Array(processed)
+            ctx.onToken(String(chars[ctx.lastEmittedLength..<safe]), false)
+            ctx.lastEmittedLength = safe
+            ctx.tokenCount += 1
+        }
+    }
+}
+
+private extension String {
+    func trimmingLeadingCharacters(in characterSet: CharacterSet) -> String {
+        guard let index = firstIndex(where: { char in
+            !char.unicodeScalars.allSatisfy { characterSet.contains($0) }
+        }) else {
+            return ""
+        }
+        return String(self[index...])
+    }
+}