@dvai-bridge/ios-llama-core 4.0.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +341 -34
- package/Package.swift +71 -71
- package/ios/Sources/DVAILlamaCore/AudioDecoder.swift +112 -112
- package/ios/Sources/DVAILlamaCore/ContentPartsTranslator.swift +232 -232
- package/ios/Sources/DVAILlamaCore/ImageDecoder.swift +91 -91
- package/ios/Sources/DVAILlamaCore/LlamaCppBridgeProtocol.swift +59 -59
- package/ios/Sources/DVAILlamaCore/LlamaHandlers.swift +422 -422
- package/ios/Sources/DVAILlamaCore/ModelDownloader.swift +445 -445
- package/ios/Sources/DVAILlamaCore/PluginState.swift +158 -158
- package/ios/Sources/DVAILlamaCoreObjC/LlamaCppBridge.mm +649 -649
- package/ios/Sources/DVAILlamaCoreObjC/include/LlamaCppBridge.h +101 -101
- package/ios/Tests/DVAILlamaCoreTests/AudioDecoderTest.swift +46 -46
- package/ios/Tests/DVAILlamaCoreTests/ContentPartsTranslatorTest.swift +361 -361
- package/ios/Tests/DVAILlamaCoreTests/ImageDecoderTest.swift +139 -139
- package/ios/Tests/DVAILlamaCoreTests/LlamaCppBridgeTest.swift +131 -131
- package/ios/Tests/DVAILlamaCoreTests/LlamaHandlersTest.swift +515 -515
- package/ios/Tests/DVAILlamaCoreTests/ModelDownloaderTest.swift +89 -89
- package/ios/Tests/DVAILlamaCoreTests/PluginStateTest.swift +51 -51
- package/package.json +3 -3
- package/README.md +0 -199
|
@@ -1,112 +1,112 @@
|
|
|
1
|
-
import Foundation
|
|
2
|
-
import AVFoundation
|
|
3
|
-
|
|
4
|
-
/// Supported audio encodings accepted by `AudioDecoder.decode(data:format:)`.
|
|
5
|
-
///
|
|
6
|
-
/// `pcm16` is treated as already-decoded raw little-endian 16 kHz mono PCM16
|
|
7
|
-
/// and returned unchanged. All other formats are decoded via
|
|
8
|
-
/// `AVAudioFile` + `AVAudioConverter` to that same target format.
|
|
9
|
-
enum AudioFormat: String {
|
|
10
|
-
case pcm16, wav, mp3, m4a, aac, flac
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
/// Decodes supported audio formats to 16 kHz mono PCM16 little-endian samples
|
|
14
|
-
/// suitable for feeding into a multimodal projector.
|
|
15
|
-
struct AudioDecoder {
|
|
16
|
-
/// Decode `data` (encoded in `format`) to 16 kHz mono PCM16 LE samples.
|
|
17
|
-
/// Pass-through for `.pcm16`.
|
|
18
|
-
static func decode(data: Data, format: AudioFormat) async throws -> Data {
|
|
19
|
-
switch format {
|
|
20
|
-
case .pcm16:
|
|
21
|
-
return data
|
|
22
|
-
case .wav, .mp3, .m4a, .aac, .flac:
|
|
23
|
-
return try await decodeViaAVAudioFile(data: data)
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
private static func decodeViaAVAudioFile(data: Data) async throws -> Data {
|
|
28
|
-
// AVAudioFile requires a file URL, so write to a temp file first.
|
|
29
|
-
let tmpURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
|
30
|
-
try data.write(to: tmpURL)
|
|
31
|
-
defer { try? FileManager.default.removeItem(at: tmpURL) }
|
|
32
|
-
|
|
33
|
-
let inputFile = try AVAudioFile(forReading: tmpURL)
|
|
34
|
-
guard let outputFormat = AVAudioFormat(
|
|
35
|
-
commonFormat: .pcmFormatInt16,
|
|
36
|
-
sampleRate: 16000,
|
|
37
|
-
channels: 1,
|
|
38
|
-
interleaved: true
|
|
39
|
-
) else {
|
|
40
|
-
throw NSError(
|
|
41
|
-
domain: "AudioDecoder",
|
|
42
|
-
code: 1,
|
|
43
|
-
userInfo: [NSLocalizedDescriptionKey: "Unable to create output format"]
|
|
44
|
-
)
|
|
45
|
-
}
|
|
46
|
-
guard let converter = AVAudioConverter(from: inputFile.processingFormat, to: outputFormat) else {
|
|
47
|
-
throw NSError(
|
|
48
|
-
domain: "AudioDecoder",
|
|
49
|
-
code: 2,
|
|
50
|
-
userInfo: [NSLocalizedDescriptionKey: "Unable to create converter"]
|
|
51
|
-
)
|
|
52
|
-
}
|
|
53
|
-
guard
|
|
54
|
-
let inputBuf = AVAudioPCMBuffer(pcmFormat: inputFile.processingFormat, frameCapacity: 4096),
|
|
55
|
-
let outputBuf = AVAudioPCMBuffer(pcmFormat: outputFormat, frameCapacity: 4096)
|
|
56
|
-
else {
|
|
57
|
-
throw NSError(
|
|
58
|
-
domain: "AudioDecoder",
|
|
59
|
-
code: 3,
|
|
60
|
-
userInfo: [NSLocalizedDescriptionKey: "Unable to allocate buffers"]
|
|
61
|
-
)
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
var result = Data()
|
|
65
|
-
while inputFile.framePosition < inputFile.length {
|
|
66
|
-
try inputFile.read(into: inputBuf)
|
|
67
|
-
|
|
68
|
-
// The input callback may be invoked multiple times per
|
|
69
|
-
// `convert(to:error:withInputFrom:)` call. Without this guard the
|
|
70
|
-
// same `inputBuf` would be re-emitted to the converter and we'd
|
|
71
|
-
// double-count samples / corrupt output.
|
|
72
|
-
var consumed = false
|
|
73
|
-
var error: NSError?
|
|
74
|
-
converter.convert(to: outputBuf, error: &error) { _, status in
|
|
75
|
-
if consumed {
|
|
76
|
-
status.pointee = .endOfStream
|
|
77
|
-
return nil
|
|
78
|
-
}
|
|
79
|
-
consumed = true
|
|
80
|
-
status.pointee = .haveData
|
|
81
|
-
return inputBuf
|
|
82
|
-
}
|
|
83
|
-
if let error = error { throw error }
|
|
84
|
-
|
|
85
|
-
if let int16Data = outputBuf.int16ChannelData {
|
|
86
|
-
let frameLength = Int(outputBuf.frameLength)
|
|
87
|
-
if frameLength > 0 {
|
|
88
|
-
let bytes = UnsafeRawPointer(int16Data[0])
|
|
89
|
-
result.append(Data(bytes: bytes, count: frameLength * 2))
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// Drain: tell the converter we're done so any buffered tail samples
|
|
95
|
-
// (e.g. AAC priming / codec lookahead) are flushed to outputBuf.
|
|
96
|
-
var drainError: NSError?
|
|
97
|
-
let drainStatus = converter.convert(to: outputBuf, error: &drainError) { _, status in
|
|
98
|
-
status.pointee = .endOfStream
|
|
99
|
-
return nil
|
|
100
|
-
}
|
|
101
|
-
if drainStatus == .haveData, drainError == nil, let int16Data = outputBuf.int16ChannelData {
|
|
102
|
-
let frameLength = Int(outputBuf.frameLength)
|
|
103
|
-
if frameLength > 0 {
|
|
104
|
-
let bytes = UnsafeRawPointer(int16Data[0])
|
|
105
|
-
result.append(Data(bytes: bytes, count: frameLength * 2))
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
// drainError on flush is acceptable — some codecs return an error when there's nothing left.
|
|
109
|
-
|
|
110
|
-
return result
|
|
111
|
-
}
|
|
112
|
-
}
|
|
1
|
+
import Foundation
|
|
2
|
+
import AVFoundation
|
|
3
|
+
|
|
4
|
+
/// Supported audio encodings accepted by `AudioDecoder.decode(data:format:)`.
|
|
5
|
+
///
|
|
6
|
+
/// `pcm16` is treated as already-decoded raw little-endian 16 kHz mono PCM16
|
|
7
|
+
/// and returned unchanged. All other formats are decoded via
|
|
8
|
+
/// `AVAudioFile` + `AVAudioConverter` to that same target format.
|
|
9
|
+
enum AudioFormat: String {
|
|
10
|
+
case pcm16, wav, mp3, m4a, aac, flac
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/// Decodes supported audio formats to 16 kHz mono PCM16 little-endian samples
|
|
14
|
+
/// suitable for feeding into a multimodal projector.
|
|
15
|
+
struct AudioDecoder {
|
|
16
|
+
/// Decode `data` (encoded in `format`) to 16 kHz mono PCM16 LE samples.
|
|
17
|
+
/// Pass-through for `.pcm16`.
|
|
18
|
+
static func decode(data: Data, format: AudioFormat) async throws -> Data {
|
|
19
|
+
switch format {
|
|
20
|
+
case .pcm16:
|
|
21
|
+
return data
|
|
22
|
+
case .wav, .mp3, .m4a, .aac, .flac:
|
|
23
|
+
return try await decodeViaAVAudioFile(data: data)
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
private static func decodeViaAVAudioFile(data: Data) async throws -> Data {
|
|
28
|
+
// AVAudioFile requires a file URL, so write to a temp file first.
|
|
29
|
+
let tmpURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
|
30
|
+
try data.write(to: tmpURL)
|
|
31
|
+
defer { try? FileManager.default.removeItem(at: tmpURL) }
|
|
32
|
+
|
|
33
|
+
let inputFile = try AVAudioFile(forReading: tmpURL)
|
|
34
|
+
guard let outputFormat = AVAudioFormat(
|
|
35
|
+
commonFormat: .pcmFormatInt16,
|
|
36
|
+
sampleRate: 16000,
|
|
37
|
+
channels: 1,
|
|
38
|
+
interleaved: true
|
|
39
|
+
) else {
|
|
40
|
+
throw NSError(
|
|
41
|
+
domain: "AudioDecoder",
|
|
42
|
+
code: 1,
|
|
43
|
+
userInfo: [NSLocalizedDescriptionKey: "Unable to create output format"]
|
|
44
|
+
)
|
|
45
|
+
}
|
|
46
|
+
guard let converter = AVAudioConverter(from: inputFile.processingFormat, to: outputFormat) else {
|
|
47
|
+
throw NSError(
|
|
48
|
+
domain: "AudioDecoder",
|
|
49
|
+
code: 2,
|
|
50
|
+
userInfo: [NSLocalizedDescriptionKey: "Unable to create converter"]
|
|
51
|
+
)
|
|
52
|
+
}
|
|
53
|
+
guard
|
|
54
|
+
let inputBuf = AVAudioPCMBuffer(pcmFormat: inputFile.processingFormat, frameCapacity: 4096),
|
|
55
|
+
let outputBuf = AVAudioPCMBuffer(pcmFormat: outputFormat, frameCapacity: 4096)
|
|
56
|
+
else {
|
|
57
|
+
throw NSError(
|
|
58
|
+
domain: "AudioDecoder",
|
|
59
|
+
code: 3,
|
|
60
|
+
userInfo: [NSLocalizedDescriptionKey: "Unable to allocate buffers"]
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
var result = Data()
|
|
65
|
+
while inputFile.framePosition < inputFile.length {
|
|
66
|
+
try inputFile.read(into: inputBuf)
|
|
67
|
+
|
|
68
|
+
// The input callback may be invoked multiple times per
|
|
69
|
+
// `convert(to:error:withInputFrom:)` call. Without this guard the
|
|
70
|
+
// same `inputBuf` would be re-emitted to the converter and we'd
|
|
71
|
+
// double-count samples / corrupt output.
|
|
72
|
+
var consumed = false
|
|
73
|
+
var error: NSError?
|
|
74
|
+
converter.convert(to: outputBuf, error: &error) { _, status in
|
|
75
|
+
if consumed {
|
|
76
|
+
status.pointee = .endOfStream
|
|
77
|
+
return nil
|
|
78
|
+
}
|
|
79
|
+
consumed = true
|
|
80
|
+
status.pointee = .haveData
|
|
81
|
+
return inputBuf
|
|
82
|
+
}
|
|
83
|
+
if let error = error { throw error }
|
|
84
|
+
|
|
85
|
+
if let int16Data = outputBuf.int16ChannelData {
|
|
86
|
+
let frameLength = Int(outputBuf.frameLength)
|
|
87
|
+
if frameLength > 0 {
|
|
88
|
+
let bytes = UnsafeRawPointer(int16Data[0])
|
|
89
|
+
result.append(Data(bytes: bytes, count: frameLength * 2))
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Drain: tell the converter we're done so any buffered tail samples
|
|
95
|
+
// (e.g. AAC priming / codec lookahead) are flushed to outputBuf.
|
|
96
|
+
var drainError: NSError?
|
|
97
|
+
let drainStatus = converter.convert(to: outputBuf, error: &drainError) { _, status in
|
|
98
|
+
status.pointee = .endOfStream
|
|
99
|
+
return nil
|
|
100
|
+
}
|
|
101
|
+
if drainStatus == .haveData, drainError == nil, let int16Data = outputBuf.int16ChannelData {
|
|
102
|
+
let frameLength = Int(outputBuf.frameLength)
|
|
103
|
+
if frameLength > 0 {
|
|
104
|
+
let bytes = UnsafeRawPointer(int16Data[0])
|
|
105
|
+
result.append(Data(bytes: bytes, count: frameLength * 2))
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// drainError on flush is acceptable — some codecs return an error when there's nothing left.
|
|
109
|
+
|
|
110
|
+
return result
|
|
111
|
+
}
|
|
112
|
+
}
|