@capgo/capacitor-llm 8.0.19 → 8.1.0

package/ios/Sources/LiteRTLM/ExperimentalFlags.swift

@@ -0,0 +1,142 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import Foundation
+import OSLog
+
+/// A struct to manage flags for APIs that are not yet considered mature.
+///
+/// These flags guard experimental APIs that may still undergo significant changes.
+/// To use any experimental flags, first call `ExperimentalFlags.optIntoExperimentalAPIs()`.
+public struct ExperimentalFlags {
+  private static var optedIn = false
+
+  private static let logger = Logger(
+    subsystem: "com.google.odml.litertlm.swift",
+    category: "ExperimentalFlags"
+  )
+
+  public static func optIntoExperimentalAPIs() {
+    guard !optedIn else { return }
+    logger.info("EXPERIMENTAL: LiteRTLM: Opting into experimental APIs....")
+    optedIn = true
+  }
+
+  private static var _enableBenchmark: Bool = false
+
+  public static var enableBenchmark: Bool {
+    get { return _enableBenchmark }
+    set {
+      guard optedIn else {
+        logger.error("LiteRTLM: Must opt into experimental APIs before setting this flag.")
+        return
+      }
+      _enableBenchmark = newValue
+    }
+  }
+
+  private static var _convertCamelToSnakeCaseInToolDescription: Bool = true
+
+  /// Whether to convert the function and parameter names in to snake case for
+  /// tool calling.
+  ///
+  /// Swift idiomatic style uses camelCase for names. However, many large
+  /// language models are predominantly trained on datasets where snake_case is
+  /// common.
+  ///
+  /// By default, this API converts Swift camelCase names to snake_case to
+  /// potentially improve tool calling performance with models more familiar
+  /// with snake_case.
+  ///
+  /// Set this flag to `false` if your model is specifically trained with
+  /// camelCase tool descriptions to skip the conversion. Otherwise, the default
+  /// of `true` (using snake_case) is recommended.
+  ///
+  /// Note: This flag is read only when a new [Conversation] is created.
+  /// Changing this value will not affect any existing [Conversation] instances.
+  public static var convertCamelToSnakeCaseInToolDescription: Bool {
+    get { return _convertCamelToSnakeCaseInToolDescription }
+    set {
+      guard optedIn else {
+        logger.error("LiteRTLM: Must opt into experimental APIs before setting this flag.")
+        return
+      }
+      _convertCamelToSnakeCaseInToolDescription = newValue
+    }
+  }
+
+  private static var _enableConversationConstrainedDecoding: Bool = false
+
+  /// Whether to enable conversation constrained decoding. This is primarily
+  /// used for function calling.
+  ///
+  /// Note: This flag is read only when a new [Conversation] is created.
+  /// Changing this value will not affect any existing [Conversation] instances.
+  public static var enableConversationConstrainedDecoding: Bool {
+    get { return _enableConversationConstrainedDecoding }
+    set {
+      guard optedIn else {
+        logger.error("LiteRTLM: Must opt into experimental APIs before setting this flag.")
+        return
+      }
+      _enableConversationConstrainedDecoding = newValue
+    }
+  }
+
+  private static var _enableSpeculativeDecoding: Bool? = nil
+
+  /// Whether to enable speculative decoding.
+  ///
+  /// If true, enable speculative decoding; an error will be thrown if the model does not support it.
+  /// If false, disable speculative decoding.
+  ///
+  /// Note: This flag is read only when a new [Engine] is created. Changing this value will not
+  /// affect any existing [Engine] or [Conversation] instances.
+  public static var enableSpeculativeDecoding: Bool? {
+    get { return _enableSpeculativeDecoding }
+    set {
+      guard optedIn else {
+        logger.error("LiteRTLM: Must opt into experimental APIs before setting this flag.")
+        return
+      }
+      _enableSpeculativeDecoding = newValue
+    }
+  }
+
+  private static var _visualTokenBudget: Int32? = nil
+
+  /// The visual token budget.
+  ///
+  /// The number of visual tokens that the model can generate for a single image. If null, there is
+  /// no budget limit and the engine use as much as needed.
+  ///
+  /// Currently, this is only supported by Gemma4. If this flag is set for a non-Gemma4 model, it
+  /// will result in a no-ops. The Gemma4 budget options are 70, 140, 280, 560, or 1120 tokens. See
+  /// https://ai.google.dev/gemma/docs/capabilities/vision#variable-resolution for more details.
+  ///
+  /// Note: This flag takes effect immediately and change alter the behaivor of created
+  /// [Conversation].
+  public static var visualTokenBudget: Int32? {
+    get { return _visualTokenBudget }
+    set {
+      guard optedIn else {
+        logger.error("LiteRTLM: Must opt into experimental APIs before setting this flag.")
+        return
+      }
+      _visualTokenBudget = newValue
+    }
+  }
+
+  // Prevent initializing the struct
+  private init() {}
+}

package/ios/Sources/LiteRTLM/LiteRTLMError.swift

@@ -0,0 +1,156 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Errors thrown by the LiteRT-LM Swift API.
+public enum LiteRTLMError: Error, LocalizedError, Equatable {
+  case engine(EngineError)
+  case conversation(ConversationError)
+  case config(ConfigError)
+  case tool(ToolError)
+  case message(MessageError)
+
+  public var errorDescription: String? {
+    switch self {
+    case .engine(let error): return error.errorDescription
+    case .conversation(let error): return error.errorDescription
+    case .config(let error): return error.errorDescription
+    case .tool(let error): return error.errorDescription
+    case .message(let error): return error.errorDescription
+    }
+  }
+
+  /// Specific errors related to the `Engine`.
+  public enum EngineError: Error, LocalizedError, Equatable {
+    case alreadyInitialized
+    case failedToCreateSettings
+    case failedToCreateEngine
+    case notInitialized
+    case failedToCreateSessionConfig
+    case failedToCreateConversationConfig
+    case failedToCreateConversation
+
+    public var errorDescription: String? {
+      switch self {
+      case .alreadyInitialized:
+        return "Engine is already initialized."
+      case .failedToCreateSettings:
+        return "Failed to create engine settings."
+      case .failedToCreateEngine:
+        return "Failed to create engine."
+      case .notInitialized:
+        return "Engine is not initialized."
+      case .failedToCreateSessionConfig:
+        return "Failed to create session config."
+      case .failedToCreateConversationConfig:
+        return "Failed to create conversation config."
+      case .failedToCreateConversation:
+        return "Failed to create conversation."
+      }
+    }
+  }
+
+  /// Specific errors related to `Conversation`.
+  public enum ConversationError: Error, LocalizedError, Equatable {
+    case notAlive
+    case failedToSerializeMessage
+    case invalidResponse(String)
+    case recurringToolCallLimitExceeded(limit: Int)
+    case failedToStartStream(status: Int)
+    case invalidJson(String)
+    case toolExecutionError(name: String, error: String)
+    case benchmarkNotEnabled
+    case benchmarkInfoUnavailable
+
+    public var errorDescription: String? {
+      switch self {
+      case .notAlive:
+        return "Conversation is not alive."
+      case .failedToSerializeMessage:
+        return "Failed to serialize message to JSON string."
+      case .invalidResponse(let details):
+        return "Invalid response from native layer: \(details)"
+      case .recurringToolCallLimitExceeded(let limit):
+        return "Exceeded recurring tool call limit of \(limit)"
+      case .failedToStartStream(let status):
+        return "Failed to start stream. Status: \(status)"
+      case .invalidJson(let details):
+        return "Invalid JSON: \(details)"
+      case .toolExecutionError(let name, let error):
+        return "Error processing tool call \(name): \(error)"
+      case .benchmarkNotEnabled:
+        return """
+          Benchmark flag is not enabled. Please enable the flag by setting setting \
+          ExperimentalFlags.enableBenchmark to true before initializing the Engine.
+          """
+      case .benchmarkInfoUnavailable:
+        return "Failed to get benchmark info."
+      }
+    }
+  }
+
+  public enum ConfigError: Error, LocalizedError, Equatable {
+    case invalidMaxNumTokens
+    case invalidMaxNumImages(count: Int)
+    case invalidTopK
+    case invalidTopP
+    case invalidTemperature
+    case multipleSystemMessages
+
+    public var errorDescription: String? {
+      switch self {
+      case .invalidMaxNumTokens:
+        return "maxNumTokens must be positive or nil (use the default from model or engine)."
+      case .invalidMaxNumImages:
+        return "maxNumImages must be non-negative or nil (use the default from model or engine)."
+      case .invalidTopK:
+        return "topK should be positive."
+      case .invalidTopP:
+        return "topP not between 0 and 1"
+      case .invalidTemperature:
+        return "temperature should be non-negative"
+      case .multipleSystemMessages:
+        return "Cannot set both systemMessage and have system messages in initialMessages."
+      }
+    }
+  }
+
+  /// Specific errors related to tools.
+  public enum ToolError: Error, LocalizedError, Equatable {
+    case notFound(name: String)
+
+    public var errorDescription: String? {
+      switch self {
+      case .notFound(let name):
+        return "Tool '\(name)' not found."
+      }
+    }
+  }
+
+  /// Specific errors related to messages.
+  public enum MessageError: Error, LocalizedError, Equatable {
+    case failedToConvertToJson
+    case invalidContent
+
+    public var errorDescription: String? {
+      switch self {
+      case .failedToConvertToJson:
+        return "Failed to convert Message to JSON string."
+      case .invalidContent:
+        return "No content found in JSON string. Cannot create Message."
+      }
+    }
+  }
+}

package/ios/Sources/LiteRTLM/Message.swift

@@ -0,0 +1,225 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import OSLog
+
+/// The role of the message in a conversation.
+public enum Role: String {
+  case system
+  case user
+  case model
+  case tool
+}
+
+/// Represents a content in the `Message` of the conversation.
+///
+/// Example Usage:
+/// ```swift
+/// let textContent = Content.text("What's the capital of France?")
+/// let imageContent = Content.imageData(imageData)
+/// let audioContent = Content.audioFile(path: "/path/to/audio.wav")
+/// ```
+public enum Content {
+  /// Text.
+  case text(String)
+  /// Image provided as raw bytes.
+  case imageData(Data)
+  /// Image provided by an absolute file path.
+  case imageFile(String)
+  /// Audio provided as raw bytes.
+  case audioData(Data)
+  /// Audio provided by an absolute file path.
+  case audioFile(String)
+
+  /// Convert to JSON format. Used internally.
+  var toJson: [String: String] {
+    switch self {
+    case .text(let text):
+      return ["type": "text", "text": text]
+    case .imageData(let bytes):
+      return ["type": "image", "blob": bytes.base64EncodedString()]
+    case .imageFile(let absPath):
+      return ["type": "image", "path": absPath]
+    case .audioData(let bytes):
+      return ["type": "audio", "blob": bytes.base64EncodedString()]
+    case .audioFile(let absPath):
+      return ["type": "audio", "path": absPath]
+    }
+  }
+}
+
+/// Represents a message in the conversation. A message can contain multiple `Content`s.
+///
+/// Example Usage:
+/// ```swift
+/// let textMessage = Message("What's the capital of France?")
+/// let modelMessage = Message("The capital of France is Paris.", role: .model)
+/// let audioContentMessage = Message(of: someAudioContent)
+/// let multiContentMessage = Message(of: someText, someImageContent)
+/// let multiContentMessageFromArray = Message(contents: [someText, someImageContent])
+/// ```
+public struct Message {
+
+  private let logger = Logger(
+    subsystem: "com.google.odml.litertlm.swift",
+    category: "Message"
+  )
+
+  /// The role of the message.
+  public let role: Role
+
+  /// The contents of the message.
+  public let contents: Contents
+
+  /// The channels of the message.
+  public let channels: [String: String]
+
+  // TODO: (b/459796564): Update constructor to `Message.role("messageTextHere")`
+  /// Creates a `Message` from a text string.
+  /// - Parameter text: The text content of the message.
+  public init(_ text: String, role: Role = .user, channels: [String: String] = [:]) {
+    self.init(contents: Contents(contents: [.text(text)]), role: role, channels: channels)
+  }
+
+  /// Creates a `Message` from one or more `Content`s.
+  /// - Parameter contents: The list of contents for this message.
+  public init(of contents: Content..., role: Role = .user) {
+    precondition(!contents.isEmpty, "Contents should not be empty.")
+    self.contents = Contents(contents: contents)
+    self.role = role
+    self.channels = [:]
+  }
+
+  /// Creates a `Message` from a list of `Content`s.
+  /// - Parameter contents: The list of contents for this message.
+  public init(contents: [Content], role: Role = .user, channels: [String: String] = [:]) {
+    precondition(
+      !contents.isEmpty || !channels.isEmpty, "Contents and channels should not both be empty.")
+    self.contents = Contents(contents: contents)
+    self.role = role
+    self.channels = channels
+  }
+
+  /// Creates a `Message` from a `Contents` object.
+  /// - Parameter contents: The contents object for this message.
+  public init(contents: Contents, role: Role = .user, channels: [String: String] = [:]) {
+    precondition(
+      !contents.isEmpty || !channels.isEmpty, "Contents and channels should not both be empty.")
+    self.contents = contents
+    self.role = role
+    self.channels = channels
+  }
+
+  /// Convert to JSON format. Used internally.
+  var toJson: [String: Any] {
+    var dict: [String: Any] = ["role": role.rawValue]
+    if !contents.isEmpty {
+      dict["content"] = contents.toJson
+    }
+    if !channels.isEmpty {
+      dict["channels"] = channels
+    }
+    return dict
+  }
+
+  /// Convenience property to get the text content of the message.
+  /// Returns a string by concatenating all text content items, separated by a space.
+  /// Returns an empty string if no text content exists.
+  public var toString: String {
+    return contents.toString
+  }
+
+  /// A computed property that returns the JSON string representation
+  /// of the Message, or nil if conversion fails.
+  var jsonString: String {
+    get throws {
+      let jsonData = try JSONSerialization.data(withJSONObject: self.toJson, options: [])
+      guard let resultString = String(data: jsonData, encoding: .utf8) else {
+        throw LiteRTLMError.message(.failedToConvertToJson)
+      }
+      return resultString
+    }
+  }
+}
+
+/// Represents a collection of `Content` items in the `Message` of the conversation.
+///
+/// Example Usage:
+/// ```swift
+/// let contents = Contents(contents: [.text("Hello")])
+/// ```
+public struct Contents: RandomAccessCollection {
+  public typealias Element = Content
+  public typealias Index = Int
+
+  /// The list of underlying content items.
+  public var contents: [Content]
+
+  public var startIndex: Int { contents.startIndex }
+  public var endIndex: Int { contents.endIndex }
+
+  public subscript(position: Int) -> Content {
+    return contents[position]
+  }
+
+  /// Creates a `Contents` from a list of `Content`s.
+  public init(contents: [Content]) {
+    self.contents = contents
+  }
+
+  /// Creates an empty `Contents`.
+  public static func empty() -> Contents {
+    return Contents(contents: [])
+  }
+
+  /// Creates a `Contents` from a text string.
+  public static func of(_ text: String) -> Contents {
+    return Contents(contents: [.text(text)])
+  }
+
+  /// Creates a `Contents` from one or more `Content`s.
+  public static func of(_ contents: Content...) -> Contents {
+    return Contents(contents: contents)
+  }
+
+  /// Creates a `Contents` from a list of `Content`s.
+  public static func of(_ contents: [Content]) -> Contents {
+    return Contents(contents: contents)
+  }
+
+  /// Convert to JSON format. Used internally.
+  var toJson: [[String: String]] {
+    return contents.map { $0.toJson }
+  }
+
+  /// A computed property that returns the JSON string representation
+  /// of the Contents array.
+  var jsonString: String {
+    get throws {
+      let jsonData = try JSONSerialization.data(withJSONObject: self.toJson, options: [])
+      guard let resultString = String(data: jsonData, encoding: .utf8) else {
+        throw LiteRTLMError.message(.failedToConvertToJson)
+      }
+      return resultString
+    }
+  }
+
+  /// Returns a string by concatenating all text content items, separated by a space.
+  public var toString: String {
+    contents.compactMap { content in
+      if case .text(let str) = content { return str } else { return nil }
+    }.joined(separator: " ")
+  }
+}