@codex-app-sdk/backend 0.14.4 → 0.15.0

package/dist/node/apple-speech-session.d.ts

@@ -0,0 +1,13 @@
+import { type ChildProcessWithoutNullStreams } from 'node:child_process';
+import type { CodexSpeechSessionEvent, CodexSpeechSessionOptions, CodexSpeechTranscriptionResult } from '@codex-app-sdk/core/native';
+export type AppleSpeechSession = {
+    append(audio: ArrayBuffer): Promise<void>;
+    stop(): Promise<CodexSpeechTranscriptionResult>;
+    cancel(): Promise<void>;
+};
+export declare function startAppleSpeechSession(options: CodexSpeechSessionOptions, onEvent: (event: CodexSpeechSessionEvent) => void, dependencies?: {
+    assetsPath?: string;
+    signal?: AbortSignal;
+    spawn?: (command: string, args: string[]) => ChildProcessWithoutNullStreams;
+}): Promise<AppleSpeechSession>;
+//# sourceMappingURL=apple-speech-session.d.ts.map

package/dist/node/apple-speech-session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"apple-speech-session.d.ts","sourceRoot":"","sources":["../../src/node/apple-speech-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,8BAA8B,EAAE,MAAM,oBAAoB,CAAC;AAKhF,OAAO,KAAK,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,8BAA8B,EAAE,MAAM,4BAA4B,CAAC;AAGrI,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,IAAI,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAChD,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB,CAAC;AAEF,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,yBAAyB,EAClC,OAAO,EAAE,CAAC,KAAK,EAAE,uBAAuB,KAAK,IAAI,EACjD,YAAY,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,8BAA8B,CAAA;CAAO,GAC5I,OAAO,CAAC,kBAAkB,CAAC,CA8G7B"}

package/dist/node/index.d.ts

@@ -7,4 +7,5 @@ export { CodexAppBackend, createCodexAppBackend, type CodexAppBackendModule, typ
 export { CodexAppBackendTtlCache, type CodexAppBackendTtlCacheEvictionContext, type CodexAppBackendTtlCacheOptions, type CodexAppBackendTtlCacheRecord, type CodexAppBackendTtlCacheScheduler, type CodexAppBackendTtlTimer, } from './codex-app-backend-cache';
 export { codexItemToMediaPart, codexItemToSurfaceMessage, codexItemToToolPart, codexThreadToSurfaceMessages, codexTurnToSurfaceMessages, preserveHistoricalAttachmentPreviews, } from './codex-conversation-history';
 export { resolveAppleSpeechAnalyzerPath, transcribeWithAppleSpeechAnalyzer, type AppleSpeechAssetDiscoveryDependencies, type AppleSpeechTranscriptionOptions, type AppleSpeechTranscriptionResult, } from './apple-speech-transcription';
+export { startAppleSpeechSession, type AppleSpeechSession } from './apple-speech-session';
 //# sourceMappingURL=index.d.ts.map

package/dist/node/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/node/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,4BAA4B,EAC5B,KAAK,kBAAkB,EACvB,KAAK,mCAAmC,GACzC,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,iCAAiC,EACjC,KAAK,wCAAwC,GAC9C,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACpB,KAAK,oCAAoC,EACzC,KAAK,YAAY,EACjB,KAAK,0BAA0B,GAChC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,EAC9B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,4BAA4B,EACjC,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,EACzB,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EACnC,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC1B,KAAK,8BAA8B,EACnC,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,+BAA+B,EACpC,KAAK,mCAAmC,EACxC,KAAK,gCAAgC,EACrC,KAAK,sCAAsC,EAC3C,KAAK,+BAA+B,EACpC,KAAK,yBAAyB,EAC9B,KAAK,kBAAkB,EACvB,KAAK,4BAA4B,EACjC,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,4BAA4B,EAC5B,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,4BAA4B,EACjC,KAAK,sBAAsB,GAC5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,uBAAuB,EACvB,KAAK,sCAAsC,EAC3C,KAAK,8BAA8B,EACnC,KAAK,6BAA6B,EAClC,KAAK,gCAAgC,EACrC,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,mBAAmB,EACnB,4BAA4B,EAC5B,0BAA0B,EAC1B,oCAAoC,GACrC,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,8BAA8B,EAC9B,iCAAiC,EACjC,KAAK,qCAAqC,EAC1C,KAAK,+BAA+B,EACpC,KAAK,8BAA8B,GACpC,MAAM,8BAA8B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/node/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,4BAA4B,EAC5B,KAAK,kBAAkB,EACvB,KAAK,mCAAmC,GACzC,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,iCAAiC,EACjC,KAAK,wCAAwC,GAC9C,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACpB,KAAK,oCAAoC,EACzC,KAAK,YAAY,EACjB,KAAK,0BAA0B,GAChC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,EAC9B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,4BAA4B,EACjC,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,EACzB,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EACnC,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC1B,KAAK,8BAA8B,EACnC,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,+BAA+B,EACpC,KAAK,mCAAmC,EACxC,KAAK,gCAAgC,EACrC,KAAK,sCAAsC,EAC3C,KAAK,+BAA+B,EACpC,KAAK,yBAAyB,EAC9B,KAAK,kBAAkB,EACvB,KAAK,4BAA4B,EACjC,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,4BAA4B,EAC5B,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,4BAA4B,EACjC,KAAK,sBAAsB,GAC5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,uBAAuB,EACvB,KAAK,sCAAsC,EAC3C,KAAK,8BAA8B,EACnC,KAAK,6BAA6B,EAClC,KAAK,gCAAgC,EACrC,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,mBAAmB,EACnB,4BAA4B,EAC5B,0BAA0B,EAC1B,oCAAoC,GACrC,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACL,8BAA8B,EAC9B,iCAAiC,EACjC,KAAK,qCAAqC,EAC1C,KAAK,+BAA+B,EACpC,KAAK,8BAA8B,GACpC,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,uBAAuB,EAAE,KAAK,kBAAkB,EAAE,MAAM,wBAAwB,CAAC"}

package/native/apple-speech.swift

@@ -0,0 +1,128 @@
+import AVFoundation
+import Foundation
+import Speech
+
+// Owned SDK helper. Speech stays on device; stdout is a newline-delimited JSON protocol.
+func emit(_ event: [String: Any]) {
+    if let data = try? JSONSerialization.data(withJSONObject: event) {
+        FileHandle.standardOutput.write(data + Data([10]))
+    }
+}
+
+@main struct AppleSpeechCLI {
+    static func main() async {
+        do {
+            if #available(macOS 26.0, *) { try await run() }
+            else { throw failure("Apple speech recognition requires macOS 26 or later.") }
+        } catch {
+            emit(["type": "error", "message": error.localizedDescription])
+            FileHandle.standardError.write(Data((error.localizedDescription + "\n").utf8))
+            exit(1)
+        }
+    }
+
+    static func failure(_ message: String) -> NSError {
+        NSError(domain: "CodexAppSDK.Speech", code: 1, userInfo: [NSLocalizedDescriptionKey: message])
+    }
+
+    @available(macOS 26.0, *) static func run() async throws {
+        let args = CommandLine.arguments
+        func option(_ key: String) -> String? {
+            guard let i = args.firstIndex(of: key), i + 1 < args.count else { return nil }
+            return args[i + 1]
+        }
+        guard SpeechTranscriber.isAvailable else { throw failure("On-device speech recognition is unavailable on this Mac.") }
+        let requested = Locale(identifier: option("--locale") ?? Locale.current.identifier)
+        guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: requested) else {
+            throw failure("Speech recognition does not support \(requested.identifier).")
+        }
+        let streaming = args.contains("--stream")
+        let transcriber = SpeechTranscriber(locale: locale, transcriptionOptions: [],
+                                           reportingOptions: streaming ? [.volatileResults, .fastResults] : [], attributeOptions: [])
+        if let installation = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
+            try await installation.downloadAndInstall()
+        }
+        let analyzer = SpeechAnalyzer(modules: [transcriber])
+        let results = Task { () throws -> String in
+            var finalText = ""
+            for try await result in transcriber.results {
+                let text = String(result.text.characters)
+                if result.isFinal { finalText += text }
+                if streaming {
+                    emit(["type": "transcript", "finalText": finalText, "partialText": result.isFinal ? "" : text])
+                }
+            }
+            return finalText.trimmingCharacters(in: .whitespacesAndNewlines)
+        }
+        do {
+            if streaming {
+                guard let rate = Double(option("--sample-rate") ?? "16000"), (8000...96000).contains(rate),
+                      let inputFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: rate, channels: 1, interleaved: false),
+                      let format = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]),
+                      let converter = AVAudioConverter(from: inputFormat, to: format) else {
+                    throw failure("Unsupported speech audio format.")
+                }
+                try await analyzer.prepareToAnalyze(in: format)
+                let (sequence, continuation) = AsyncThrowingStream<AnalyzerInput, Error>.makeStream()
+                let reader = Task.detached {
+                    do {
+                        var pending = Data()
+                        while let bytes = try FileHandle.standardInput.read(upToCount: 8192), !bytes.isEmpty {
+                            pending.append(bytes)
+                            let count = pending.count / 4
+                            if count == 0 { continue }
+                            let input = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: AVAudioFrameCount(count))!
+                            input.frameLength = AVAudioFrameCount(count)
+                            pending.withUnsafeBytes { raw in
+                                input.floatChannelData![0].update(from: raw.bindMemory(to: Float.self).baseAddress!, count: count)
+                            }
+                            pending.removeFirst(count * 4)
+                            let capacity = AVAudioFrameCount(ceil(Double(count) * format.sampleRate / rate) + 32)
+                            let output = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity)!
+                            var supplied = false
+                            var conversionError: NSError?
+                            converter.convert(to: output, error: &conversionError) { _, status in
+                                if supplied { status.pointee = .noDataNow; return nil }
+                                supplied = true
+                                status.pointee = .haveData
+                                return input
+                            }
+                            if let conversionError { throw conversionError }
+                            if output.frameLength > 0 { continuation.yield(AnalyzerInput(buffer: output)) }
+                        }
+                        if !pending.isEmpty { throw failure("Incomplete PCM audio frame.") }
+                        // Drain the converter's resampling tail before ending recognition.
+                        let tail = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 4096)!
+                        var tailError: NSError?
+                        converter.convert(to: tail, error: &tailError) { _, status in
+                            status.pointee = .endOfStream
+                            return nil
+                        }
+                        if let tailError { throw tailError }
+                        if tail.frameLength > 0 { continuation.yield(AnalyzerInput(buffer: tail)) }
+                        continuation.finish()
+                    } catch { continuation.finish(throwing: error) }
+                }
+                emit(["type": "ready"])
+                try await analyzer.start(inputSequence: sequence)
+                await reader.value
+                try await analyzer.finalizeAndFinishThroughEndOfInput()
+            } else {
+                guard let input = option("--input-audio-path"), let output = option("--output-txt-path") else {
+                    throw failure("Use --stream --sample-rate <rate>, or --input-audio-path <file> --output-txt-path <file>.")
+                }
+                let file = try AVAudioFile(forReading: URL(fileURLWithPath: input))
+                if let last = try await analyzer.analyzeSequence(from: file) {
+                    try await analyzer.finalizeAndFinish(through: last)
+                } else { await analyzer.cancelAndFinishNow() }
+                try await results.value.write(toFile: output, atomically: true, encoding: .utf8)
+                return
+            }
+            emit(["type": "done", "text": try await results.value])
+        } catch {
+            await analyzer.cancelAndFinishNow()
+            results.cancel()
+            throw error
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codex-app-sdk/backend",
-  "version": "0.14.4",
+  "version": "0.15.0",
   "description": "Node.js app-server runtime and backend composition for Codex applications.",
   "type": "module",
   "license": "Apache-2.0",
@@ -15,6 +15,7 @@
   },
   "files": [
     "assets",
+    "native",
     "dist",
     "README.md"
   ],
@@ -38,7 +39,7 @@
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "@codex-app-sdk/core": "0.14.4"
+    "@codex-app-sdk/core": "0.15.0"
   },
   "engines": {
     "node": ">=22"