@dvai-bridge/ios-llama-core 4.0.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +341 -34
- package/Package.swift +71 -71
- package/ios/Sources/DVAILlamaCore/AudioDecoder.swift +112 -112
- package/ios/Sources/DVAILlamaCore/ContentPartsTranslator.swift +232 -232
- package/ios/Sources/DVAILlamaCore/ImageDecoder.swift +91 -91
- package/ios/Sources/DVAILlamaCore/LlamaCppBridgeProtocol.swift +59 -59
- package/ios/Sources/DVAILlamaCore/LlamaHandlers.swift +422 -422
- package/ios/Sources/DVAILlamaCore/ModelDownloader.swift +445 -445
- package/ios/Sources/DVAILlamaCore/PluginState.swift +158 -158
- package/ios/Sources/DVAILlamaCoreObjC/LlamaCppBridge.mm +649 -649
- package/ios/Sources/DVAILlamaCoreObjC/include/LlamaCppBridge.h +101 -101
- package/ios/Tests/DVAILlamaCoreTests/AudioDecoderTest.swift +46 -46
- package/ios/Tests/DVAILlamaCoreTests/ContentPartsTranslatorTest.swift +361 -361
- package/ios/Tests/DVAILlamaCoreTests/ImageDecoderTest.swift +139 -139
- package/ios/Tests/DVAILlamaCoreTests/LlamaCppBridgeTest.swift +131 -131
- package/ios/Tests/DVAILlamaCoreTests/LlamaHandlersTest.swift +515 -515
- package/ios/Tests/DVAILlamaCoreTests/ModelDownloaderTest.swift +89 -89
- package/ios/Tests/DVAILlamaCoreTests/PluginStateTest.swift +51 -51
- package/package.json +3 -3
- package/README.md +0 -199
|
@@ -1,91 +1,91 @@
|
|
|
1
|
-
import Foundation
|
|
2
|
-
|
|
3
|
-
/// Errors thrown by `ImageDecoder.resolve(url:)` when the input URL string
|
|
4
|
-
/// can't be turned into image bytes.
|
|
5
|
-
enum ImageSourceError: Error {
|
|
6
|
-
case malformedDataURL(String)
|
|
7
|
-
case invalidScheme(String)
|
|
8
|
-
case httpError(status: Int)
|
|
9
|
-
case base64DecodeFailed
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
/// Resolves any of the three image URL schemes accepted by the DVAI bridge
|
|
13
|
-
/// (`data:`, `https:`/`http:`, `file:`) into the raw encoded image bytes
|
|
14
|
-
/// (PNG/JPEG/etc.). The bytes are returned as-is — actual format decoding
|
|
15
|
-
/// is performed downstream by `mtmd_helper_eval` inside llama.cpp.
|
|
16
|
-
struct ImageDecoder {
|
|
17
|
-
/// Resolve any supported URL scheme into raw image bytes.
|
|
18
|
-
///
|
|
19
|
-
/// - `data:` URLs are parsed for an optional `;base64` token and
|
|
20
|
-
/// decoded accordingly (URL-encoded payloads are also supported).
|
|
21
|
-
/// - `https:` / `http:` URLs are fetched via `URLSession` with a 30s
|
|
22
|
-
/// timeout; non-2xx responses throw `httpError`.
|
|
23
|
-
/// - `file:` URLs are read off disk via `Data(contentsOf:)`.
|
|
24
|
-
/// - Any other scheme throws `invalidScheme`.
|
|
25
|
-
static func resolve(url: String) async throws -> Data {
|
|
26
|
-
if url.hasPrefix("data:") {
|
|
27
|
-
return try resolveDataURL(url)
|
|
28
|
-
}
|
|
29
|
-
guard let parsed = URL(string: url) else {
|
|
30
|
-
throw ImageSourceError.invalidScheme(url)
|
|
31
|
-
}
|
|
32
|
-
switch parsed.scheme?.lowercased() {
|
|
33
|
-
case "https", "http":
|
|
34
|
-
return try await resolveHTTP(parsed)
|
|
35
|
-
case "file":
|
|
36
|
-
return try Data(contentsOf: parsed)
|
|
37
|
-
case let other?:
|
|
38
|
-
throw ImageSourceError.invalidScheme(other)
|
|
39
|
-
case nil:
|
|
40
|
-
throw ImageSourceError.invalidScheme(url)
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
/// Parse a `data:[<mediatype>][;base64],<payload>` URL into raw bytes.
|
|
45
|
-
/// Strict: a missing comma is treated as malformed (we don't try to
|
|
46
|
-
/// guess intent).
|
|
47
|
-
private static func resolveDataURL(_ url: String) throws -> Data {
|
|
48
|
-
// RFC 2397: data:[<mediatype>][;base64],<data>
|
|
49
|
-
// Empty header and/or empty body are well-formed and produce an
|
|
50
|
-
// empty Data result (e.g. `data:,` returns `Data()`).
|
|
51
|
-
guard let commaIdx = url.firstIndex(of: ",") else {
|
|
52
|
-
throw ImageSourceError.malformedDataURL(url)
|
|
53
|
-
}
|
|
54
|
-
// Skip the leading "data:" (5 chars) and isolate the header / body.
|
|
55
|
-
let prefixEnd = url.index(url.startIndex, offsetBy: 5)
|
|
56
|
-
let header = url[prefixEnd..<commaIdx]
|
|
57
|
-
let body = String(url[url.index(after: commaIdx)...])
|
|
58
|
-
if header.contains(";base64") {
|
|
59
|
-
guard let decoded = Data(base64Encoded: body) else {
|
|
60
|
-
throw ImageSourceError.base64DecodeFailed
|
|
61
|
-
}
|
|
62
|
-
return decoded
|
|
63
|
-
}
|
|
64
|
-
// Non-base64: payload is percent-encoded text per RFC 2397.
|
|
65
|
-
let decodedString = body.removingPercentEncoding ?? body
|
|
66
|
-
return Data(decodedString.utf8)
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/// Fetch over HTTP(S) with a 30-second timeout. Uses the older
|
|
70
|
-
/// dataTask + continuation pattern so we still work on iOS 14
|
|
71
|
-
/// (the package's deployment target); `URLSession.data(for:)` is
|
|
72
|
-
/// iOS 15+.
|
|
73
|
-
private static func resolveHTTP(_ url: URL) async throws -> Data {
|
|
74
|
-
var request = URLRequest(url: url)
|
|
75
|
-
request.timeoutInterval = 30
|
|
76
|
-
return try await withCheckedThrowingContinuation { continuation in
|
|
77
|
-
let task = URLSession.shared.dataTask(with: request) { data, response, error in
|
|
78
|
-
if let error = error {
|
|
79
|
-
continuation.resume(throwing: error)
|
|
80
|
-
return
|
|
81
|
-
}
|
|
82
|
-
if let http = response as? HTTPURLResponse, !(200...299).contains(http.statusCode) {
|
|
83
|
-
continuation.resume(throwing: ImageSourceError.httpError(status: http.statusCode))
|
|
84
|
-
return
|
|
85
|
-
}
|
|
86
|
-
continuation.resume(returning: data ?? Data())
|
|
87
|
-
}
|
|
88
|
-
task.resume()
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
}
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Errors thrown by `ImageDecoder.resolve(url:)` when the input URL string
|
|
4
|
+
/// can't be turned into image bytes.
|
|
5
|
+
enum ImageSourceError: Error {
|
|
6
|
+
case malformedDataURL(String)
|
|
7
|
+
case invalidScheme(String)
|
|
8
|
+
case httpError(status: Int)
|
|
9
|
+
case base64DecodeFailed
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/// Resolves any of the three image URL schemes accepted by the DVAI bridge
|
|
13
|
+
/// (`data:`, `https:`/`http:`, `file:`) into the raw encoded image bytes
|
|
14
|
+
/// (PNG/JPEG/etc.). The bytes are returned as-is — actual format decoding
|
|
15
|
+
/// is performed downstream by `mtmd_helper_eval` inside llama.cpp.
|
|
16
|
+
struct ImageDecoder {
|
|
17
|
+
/// Resolve any supported URL scheme into raw image bytes.
|
|
18
|
+
///
|
|
19
|
+
/// - `data:` URLs are parsed for an optional `;base64` token and
|
|
20
|
+
/// decoded accordingly (URL-encoded payloads are also supported).
|
|
21
|
+
/// - `https:` / `http:` URLs are fetched via `URLSession` with a 30s
|
|
22
|
+
/// timeout; non-2xx responses throw `httpError`.
|
|
23
|
+
/// - `file:` URLs are read off disk via `Data(contentsOf:)`.
|
|
24
|
+
/// - Any other scheme throws `invalidScheme`.
|
|
25
|
+
static func resolve(url: String) async throws -> Data {
|
|
26
|
+
if url.hasPrefix("data:") {
|
|
27
|
+
return try resolveDataURL(url)
|
|
28
|
+
}
|
|
29
|
+
guard let parsed = URL(string: url) else {
|
|
30
|
+
throw ImageSourceError.invalidScheme(url)
|
|
31
|
+
}
|
|
32
|
+
switch parsed.scheme?.lowercased() {
|
|
33
|
+
case "https", "http":
|
|
34
|
+
return try await resolveHTTP(parsed)
|
|
35
|
+
case "file":
|
|
36
|
+
return try Data(contentsOf: parsed)
|
|
37
|
+
case let other?:
|
|
38
|
+
throw ImageSourceError.invalidScheme(other)
|
|
39
|
+
case nil:
|
|
40
|
+
throw ImageSourceError.invalidScheme(url)
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/// Parse a `data:[<mediatype>][;base64],<payload>` URL into raw bytes.
|
|
45
|
+
/// Strict: a missing comma is treated as malformed (we don't try to
|
|
46
|
+
/// guess intent).
|
|
47
|
+
private static func resolveDataURL(_ url: String) throws -> Data {
|
|
48
|
+
// RFC 2397: data:[<mediatype>][;base64],<data>
|
|
49
|
+
// Empty header and/or empty body are well-formed and produce an
|
|
50
|
+
// empty Data result (e.g. `data:,` returns `Data()`).
|
|
51
|
+
guard let commaIdx = url.firstIndex(of: ",") else {
|
|
52
|
+
throw ImageSourceError.malformedDataURL(url)
|
|
53
|
+
}
|
|
54
|
+
// Skip the leading "data:" (5 chars) and isolate the header / body.
|
|
55
|
+
let prefixEnd = url.index(url.startIndex, offsetBy: 5)
|
|
56
|
+
let header = url[prefixEnd..<commaIdx]
|
|
57
|
+
let body = String(url[url.index(after: commaIdx)...])
|
|
58
|
+
if header.contains(";base64") {
|
|
59
|
+
guard let decoded = Data(base64Encoded: body) else {
|
|
60
|
+
throw ImageSourceError.base64DecodeFailed
|
|
61
|
+
}
|
|
62
|
+
return decoded
|
|
63
|
+
}
|
|
64
|
+
// Non-base64: payload is percent-encoded text per RFC 2397.
|
|
65
|
+
let decodedString = body.removingPercentEncoding ?? body
|
|
66
|
+
return Data(decodedString.utf8)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/// Fetch over HTTP(S) with a 30-second timeout. Uses the older
|
|
70
|
+
/// dataTask + continuation pattern so we still work on iOS 14
|
|
71
|
+
/// (the package's deployment target); `URLSession.data(for:)` is
|
|
72
|
+
/// iOS 15+.
|
|
73
|
+
private static func resolveHTTP(_ url: URL) async throws -> Data {
|
|
74
|
+
var request = URLRequest(url: url)
|
|
75
|
+
request.timeoutInterval = 30
|
|
76
|
+
return try await withCheckedThrowingContinuation { continuation in
|
|
77
|
+
let task = URLSession.shared.dataTask(with: request) { data, response, error in
|
|
78
|
+
if let error = error {
|
|
79
|
+
continuation.resume(throwing: error)
|
|
80
|
+
return
|
|
81
|
+
}
|
|
82
|
+
if let http = response as? HTTPURLResponse, !(200...299).contains(http.statusCode) {
|
|
83
|
+
continuation.resume(throwing: ImageSourceError.httpError(status: http.statusCode))
|
|
84
|
+
return
|
|
85
|
+
}
|
|
86
|
+
continuation.resume(returning: data ?? Data())
|
|
87
|
+
}
|
|
88
|
+
task.resume()
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -1,59 +1,59 @@
|
|
|
1
|
-
// Internal/LlamaCppBridgeProtocol.swift
|
|
2
|
-
import Foundation
|
|
3
|
-
#if !COCOAPODS
|
|
4
|
-
import DVAILlamaCoreObjC
|
|
5
|
-
#endif
|
|
6
|
-
|
|
7
|
-
/// Test seam over the ObjC++ `LlamaCppBridge`. Concrete `LlamaCppBridge`
|
|
8
|
-
/// conforms via the extension below; `LlamaHandlers` takes this protocol so
|
|
9
|
-
/// unit tests can substitute a canned-response fake without loading a real
|
|
10
|
-
/// GGUF model. Mirrors the `ImageDecoderProtocol` pattern used by Task 35's
|
|
11
|
-
/// `ContentPartsTranslator`.
|
|
12
|
-
///
|
|
13
|
-
/// The inference methods use Swift's automatic NSError-bridging — they
|
|
14
|
-
/// match the `(NSString *) … error:(NSError **)` ObjC selector and so are
|
|
15
|
-
/// imported as `throws -> String` / `throws -> [NSNumber]`.
|
|
16
|
-
protocol LlamaCppBridgeProtocol: AnyObject {
|
|
17
|
-
var isLoaded: Bool { get }
|
|
18
|
-
func completePrompt(
|
|
19
|
-
_ prompt: String,
|
|
20
|
-
maxTokens: Int32,
|
|
21
|
-
temperature: Float,
|
|
22
|
-
topP: Float
|
|
23
|
-
) throws -> String
|
|
24
|
-
func embedding(_ text: String) throws -> [NSNumber]
|
|
25
|
-
|
|
26
|
-
// Phase 2A Pass 2: real multimodal projector (mmproj) lifecycle +
|
|
27
|
-
// chat-template + multimodal completion.
|
|
28
|
-
var isMmprojLoaded: Bool { get }
|
|
29
|
-
func loadMmproj(atPath path: String) throws
|
|
30
|
-
func unloadMmproj()
|
|
31
|
-
/// Whether the loaded model declares an audio encoder (mtmd_support_audio).
|
|
32
|
-
/// Always false when mmproj is not loaded.
|
|
33
|
-
func hasAudioEncoder() -> Bool
|
|
34
|
-
|
|
35
|
-
/// Apply `llama_chat_apply_template`. `templateOverride` nil/empty falls
|
|
36
|
-
/// back to the model's bundled chat template. Each message dict must have
|
|
37
|
-
/// `role` and `content` string entries. Returns the rendered prompt string.
|
|
38
|
-
func applyChatTemplate(
|
|
39
|
-
_ templateOverride: String?,
|
|
40
|
-
messages: [[String: String]],
|
|
41
|
-
addAssistant: Bool
|
|
42
|
-
) throws -> String
|
|
43
|
-
|
|
44
|
-
/// Multimodal completion. The prompt must contain N `<__media__>` markers
|
|
45
|
-
/// matching `media.count`; bytes are auto-detected as image vs audio
|
|
46
|
-
/// (image: PNG/JPEG/etc.; audio: WAV/MP3/FLAC).
|
|
47
|
-
func completeMultimodalPrompt(
|
|
48
|
-
_ prompt: String,
|
|
49
|
-
media: [Data],
|
|
50
|
-
maxTokens: Int32,
|
|
51
|
-
temperature: Float,
|
|
52
|
-
topP: Float
|
|
53
|
-
) throws -> String
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// Concrete `LlamaCppBridge` (ObjC class) gets the four new methods via its
|
|
57
|
-
// imported ObjC selectors; the existing ones (completePrompt, embedding,
|
|
58
|
-
// loadMmproj, isMmprojLoaded) already conform from Pass 1.
|
|
59
|
-
extension LlamaCppBridge: LlamaCppBridgeProtocol {}
|
|
1
|
+
// Internal/LlamaCppBridgeProtocol.swift
|
|
2
|
+
import Foundation
|
|
3
|
+
#if !COCOAPODS
|
|
4
|
+
import DVAILlamaCoreObjC
|
|
5
|
+
#endif
|
|
6
|
+
|
|
7
|
+
/// Test seam over the ObjC++ `LlamaCppBridge`. Concrete `LlamaCppBridge`
|
|
8
|
+
/// conforms via the extension below; `LlamaHandlers` takes this protocol so
|
|
9
|
+
/// unit tests can substitute a canned-response fake without loading a real
|
|
10
|
+
/// GGUF model. Mirrors the `ImageDecoderProtocol` pattern used by Task 35's
|
|
11
|
+
/// `ContentPartsTranslator`.
|
|
12
|
+
///
|
|
13
|
+
/// The inference methods use Swift's automatic NSError-bridging — they
|
|
14
|
+
/// match the `(NSString *) … error:(NSError **)` ObjC selector and so are
|
|
15
|
+
/// imported as `throws -> String` / `throws -> [NSNumber]`.
|
|
16
|
+
protocol LlamaCppBridgeProtocol: AnyObject {
|
|
17
|
+
var isLoaded: Bool { get }
|
|
18
|
+
func completePrompt(
|
|
19
|
+
_ prompt: String,
|
|
20
|
+
maxTokens: Int32,
|
|
21
|
+
temperature: Float,
|
|
22
|
+
topP: Float
|
|
23
|
+
) throws -> String
|
|
24
|
+
func embedding(_ text: String) throws -> [NSNumber]
|
|
25
|
+
|
|
26
|
+
// Phase 2A Pass 2: real multimodal projector (mmproj) lifecycle +
|
|
27
|
+
// chat-template + multimodal completion.
|
|
28
|
+
var isMmprojLoaded: Bool { get }
|
|
29
|
+
func loadMmproj(atPath path: String) throws
|
|
30
|
+
func unloadMmproj()
|
|
31
|
+
/// Whether the loaded model declares an audio encoder (mtmd_support_audio).
|
|
32
|
+
/// Always false when mmproj is not loaded.
|
|
33
|
+
func hasAudioEncoder() -> Bool
|
|
34
|
+
|
|
35
|
+
/// Apply `llama_chat_apply_template`. `templateOverride` nil/empty falls
|
|
36
|
+
/// back to the model's bundled chat template. Each message dict must have
|
|
37
|
+
/// `role` and `content` string entries. Returns the rendered prompt string.
|
|
38
|
+
func applyChatTemplate(
|
|
39
|
+
_ templateOverride: String?,
|
|
40
|
+
messages: [[String: String]],
|
|
41
|
+
addAssistant: Bool
|
|
42
|
+
) throws -> String
|
|
43
|
+
|
|
44
|
+
/// Multimodal completion. The prompt must contain N `<__media__>` markers
|
|
45
|
+
/// matching `media.count`; bytes are auto-detected as image vs audio
|
|
46
|
+
/// (image: PNG/JPEG/etc.; audio: WAV/MP3/FLAC).
|
|
47
|
+
func completeMultimodalPrompt(
|
|
48
|
+
_ prompt: String,
|
|
49
|
+
media: [Data],
|
|
50
|
+
maxTokens: Int32,
|
|
51
|
+
temperature: Float,
|
|
52
|
+
topP: Float
|
|
53
|
+
) throws -> String
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Concrete `LlamaCppBridge` (ObjC class) gets the four new methods via its
|
|
57
|
+
// imported ObjC selectors; the existing ones (completePrompt, embedding,
|
|
58
|
+
// loadMmproj, isMmprojLoaded) already conform from Pass 1.
|
|
59
|
+
extension LlamaCppBridge: LlamaCppBridgeProtocol {}
|