@swiftbrowser/native 0.18.0 → 0.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -8
- package/dist/bridge.d.ts +64 -2
- package/dist/bridge.d.ts.map +1 -1
- package/dist/bridge.js.map +1 -1
- package/dist/geolocation.d.ts +73 -0
- package/dist/geolocation.d.ts.map +1 -0
- package/dist/geolocation.js +109 -0
- package/dist/geolocation.js.map +1 -0
- package/dist/hostRequests.d.ts +3 -1
- package/dist/hostRequests.d.ts.map +1 -1
- package/dist/hostRequests.js +46 -5
- package/dist/hostRequests.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/nativeFunctions.d.ts +35 -0
- package/dist/nativeFunctions.d.ts.map +1 -0
- package/dist/nativeFunctions.js +90 -0
- package/dist/nativeFunctions.js.map +1 -0
- package/dist/reachability.d.ts +24 -0
- package/dist/reachability.d.ts.map +1 -0
- package/dist/reachability.js +41 -0
- package/dist/reachability.js.map +1 -0
- package/dist/run.d.ts.map +1 -1
- package/dist/run.js +3 -1
- package/dist/run.js.map +1 -1
- package/dist/runtime.d.ts +9 -2
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +62 -5
- package/dist/runtime.js.map +1 -1
- package/dist/speech.d.ts +148 -0
- package/dist/speech.d.ts.map +1 -0
- package/dist/speech.js +243 -0
- package/dist/speech.js.map +1 -0
- package/dist/wakeLock.d.ts +32 -0
- package/dist/wakeLock.d.ts.map +1 -0
- package/dist/wakeLock.js +60 -0
- package/dist/wakeLock.js.map +1 -0
- package/package.json +3 -3
- package/src/bridge.ts +70 -2
- package/src/geolocation.ts +160 -0
- package/src/hostRequests.ts +47 -7
- package/src/index.ts +1 -0
- package/src/nativeFunctions.ts +94 -0
- package/src/reachability.ts +49 -0
- package/src/run.ts +5 -3
- package/src/runtime.ts +58 -5
- package/src/speech.ts +312 -0
- package/src/wakeLock.ts +63 -0
- package/swift/Sources/SwiftBrowserNative/Host/AppleSignInBridge.swift +139 -0
- package/swift/Sources/SwiftBrowserNative/Host/AudioRecorderBridge.swift +5 -0
- package/swift/Sources/SwiftBrowserNative/Host/FilesBridge.swift +83 -12
- package/swift/Sources/SwiftBrowserNative/Host/GeolocationBridge.swift +150 -0
- package/swift/Sources/SwiftBrowserNative/Host/HostBridge.swift +131 -2
- package/swift/Sources/SwiftBrowserNative/Host/HostRequestBridge.swift +81 -11
- package/swift/Sources/SwiftBrowserNative/Host/ICloudBridge.swift +112 -0
- package/swift/Sources/SwiftBrowserNative/Host/ReachabilityBridge.swift +47 -0
- package/swift/Sources/SwiftBrowserNative/Host/SpeechBridge.swift +106 -0
- package/swift/Sources/SwiftBrowserNative/Host/SpeechRecognitionBridge.swift +198 -0
- package/swift/Sources/SwiftBrowserNative/Host/WorkerHost.swift +14 -5
- package/swift/Sources/SwiftBrowserNative/Runtime/NativeFunctions.swift +134 -0
- package/swift/Sources/SwiftBrowserNative/Views/ElementView.swift +1 -0
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/Controls.swift +2 -0
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/TextInputHints.swift +105 -0
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/Touches.swift +102 -0
- package/swift/Sources/SwiftBrowserProtocol/Props.swift +13 -0
- package/swift/Sources/SwiftBrowserProtocol/RendererEvent.swift +9 -0
- package/wasm/SwiftBrowserNative.wasm +2 -2
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
#if canImport(SwiftUI) && !canImport(FoundationBridge)
|
|
2
|
+
import Foundation
|
|
3
|
+
|
|
4
|
+
/// The app's iCloud Drive container, for `fileManager.iCloudDirectory()`
|
|
5
|
+
/// (packages/react/src/fileManager.ts) and the files bridge's roots. The
|
|
6
|
+
/// container is the first one the app's entitlements name
|
|
7
|
+
/// (`com.apple.developer.ubiquity-container-identifiers`); its `Documents`
|
|
8
|
+
/// folder is what the Files app shows under iCloud Drive when the Info.plist
|
|
9
|
+
/// declares it (`NSUbiquitousContainers`). Resolving it asks the daemon, so
|
|
10
|
+
/// it happens off the main thread, once; nil where the app has no
|
|
11
|
+
/// entitlement, the user no iCloud Drive, or the device no account.
|
|
12
|
+
final class ICloudContainer: @unchecked Sendable {
|
|
13
|
+
static let shared = ICloudContainer()
|
|
14
|
+
|
|
15
|
+
private let lock = NSLock()
|
|
16
|
+
/// nil: not asked yet; `.some(nil)`: asked, unavailable.
|
|
17
|
+
private var resolved: URL??
|
|
18
|
+
|
|
19
|
+
/// The container's URL once resolved; nil before (`resolve()` first) or when unavailable.
|
|
20
|
+
var cached: URL? {
|
|
21
|
+
lock.lock()
|
|
22
|
+
defer { lock.unlock() }
|
|
23
|
+
return resolved ?? nil
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/// What was resolved so far (a synchronous read: the lock is not for an async context).
|
|
27
|
+
private func known() -> URL?? {
|
|
28
|
+
lock.lock()
|
|
29
|
+
defer { lock.unlock() }
|
|
30
|
+
return resolved
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
private func store(_ container: URL?) {
|
|
34
|
+
lock.lock()
|
|
35
|
+
defer { lock.unlock() }
|
|
36
|
+
resolved = .some(container)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/// The container's `Documents` folder, made if need be; nil when iCloud Drive is not there for the app.
|
|
40
|
+
func resolve() async -> URL? {
|
|
41
|
+
if let known = known() {
|
|
42
|
+
return known.map { $0.appendingPathComponent("Documents", isDirectory: true) }
|
|
43
|
+
}
|
|
44
|
+
let container = await Task.detached(priority: .userInitiated) { () -> URL? in
|
|
45
|
+
let manager = FileManager.default
|
|
46
|
+
guard manager.ubiquityIdentityToken != nil, let url = manager.url(forUbiquityContainerIdentifier: nil) else { return nil }
|
|
47
|
+
try? manager.createDirectory(at: url.appendingPathComponent("Documents", isDirectory: true), withIntermediateDirectories: true)
|
|
48
|
+
return url.standardizedFileURL
|
|
49
|
+
}.value
|
|
50
|
+
store(container)
|
|
51
|
+
return container.map { $0.appendingPathComponent("Documents", isDirectory: true) }
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/// Whether `url` is in the container (and so coordinated and possibly not downloaded).
|
|
55
|
+
func contains(_ url: URL) -> Bool {
|
|
56
|
+
guard let root = cached else { return false }
|
|
57
|
+
return url.path == root.path || url.path.hasPrefix(root.path + "/")
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/// Watches the container's documents (`NSMetadataQuery`) and reports the
|
|
62
|
+
/// URLs that changed, so `fileManager.watch` and `useFileStorage` see what
|
|
63
|
+
/// another device wrote; started when the app first asks for iCloud Drive.
|
|
64
|
+
@MainActor
|
|
65
|
+
final class ICloudWatcher {
|
|
66
|
+
private let query = NSMetadataQuery()
|
|
67
|
+
private let send: ([String]) -> Void
|
|
68
|
+
private var observers: [NSObjectProtocol] = []
|
|
69
|
+
|
|
70
|
+
/// `send` hands the changed URLs to the app (`__sbFilesChanged`).
|
|
71
|
+
init(send: @escaping ([String]) -> Void) {
|
|
72
|
+
self.send = send
|
|
73
|
+
query.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope]
|
|
74
|
+
query.predicate = NSPredicate(format: "%K LIKE '*'", NSMetadataItemFSNameKey)
|
|
75
|
+
query.operationQueue = .main
|
|
76
|
+
let center = NotificationCenter.default
|
|
77
|
+
observers.append(center.addObserver(forName: .NSMetadataQueryDidFinishGathering, object: query, queue: .main) { [weak self] _ in
|
|
78
|
+
MainActor.assumeIsolated { self?.gathered() }
|
|
79
|
+
})
|
|
80
|
+
observers.append(center.addObserver(forName: .NSMetadataQueryDidUpdate, object: query, queue: .main) { [weak self] notification in
|
|
81
|
+
// The items are read on the main queue only, here; the compiler cannot see that.
|
|
82
|
+
nonisolated(unsafe) let info = notification.userInfo
|
|
83
|
+
MainActor.assumeIsolated { self?.updated(info) }
|
|
84
|
+
})
|
|
85
|
+
query.start()
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
func stop() {
|
|
89
|
+
query.stop()
|
|
90
|
+
for observer in observers { NotificationCenter.default.removeObserver(observer) }
|
|
91
|
+
observers.removeAll()
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/// The first gathering: every document, so a list made before it refreshes once.
|
|
95
|
+
private func gathered() {
|
|
96
|
+
query.disableUpdates()
|
|
97
|
+
defer { query.enableUpdates() }
|
|
98
|
+
let urls = (0..<query.resultCount).compactMap { (query.result(at: $0) as? NSMetadataItem)?.value(forAttribute: NSMetadataItemURLKey) as? URL }
|
|
99
|
+
if !urls.isEmpty { send(urls.map(\.absoluteString)) }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
private func updated(_ info: [AnyHashable: Any]?) {
|
|
103
|
+
var urls: [String] = []
|
|
104
|
+
for key in [NSMetadataQueryUpdateAddedItemsKey, NSMetadataQueryUpdateChangedItemsKey, NSMetadataQueryUpdateRemovedItemsKey] {
|
|
105
|
+
for item in info?[key] as? [NSMetadataItem] ?? [] {
|
|
106
|
+
if let url = item.value(forAttribute: NSMetadataItemURLKey) as? URL { urls.append(url.absoluteString) }
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
if !urls.isEmpty { send(urls) }
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
#endif
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#if canImport(Network)
|
|
2
|
+
import Foundation
|
|
3
|
+
import Network
|
|
4
|
+
import SwiftBrowserProtocol
|
|
5
|
+
|
|
6
|
+
/// `navigator.onLine` (packages/native/src/reachability.ts) over
|
|
7
|
+
/// `NWPathMonitor`: whether the network path is satisfied. The monitor
|
|
8
|
+
/// starts when the app first asks; each change goes to the app as
|
|
9
|
+
/// `__sbOnline(bool)`, which fires its `online` and `offline` events.
|
|
10
|
+
@MainActor
|
|
11
|
+
final class ReachabilityBridge {
|
|
12
|
+
private let monitor = NWPathMonitor()
|
|
13
|
+
private let send: (Bool) -> Void
|
|
14
|
+
private var started = false
|
|
15
|
+
/// The last status the monitor reported; online until it reports.
|
|
16
|
+
private(set) var isOnline = true
|
|
17
|
+
|
|
18
|
+
/// `send` calls the app's `__sbOnline`.
|
|
19
|
+
init(send: @escaping (Bool) -> Void) {
|
|
20
|
+
self.send = send
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/// Starts the monitor (once); until its first report the app is online.
|
|
24
|
+
func start() {
|
|
25
|
+
guard !started else { return }
|
|
26
|
+
started = true
|
|
27
|
+
monitor.pathUpdateHandler = { [weak self] path in
|
|
28
|
+
let online = path.status == .satisfied
|
|
29
|
+
DispatchQueue.main.async {
|
|
30
|
+
MainActor.assumeIsolated { self?.update(online) }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
monitor.start(queue: DispatchQueue(label: "swiftbrowser.reachability"))
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
func stop() {
|
|
37
|
+
guard started else { return }
|
|
38
|
+
monitor.cancel()
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
private func update(_ online: Bool) {
|
|
42
|
+
guard online != isOnline else { return }
|
|
43
|
+
isOnline = online
|
|
44
|
+
send(online)
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
#endif
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
#if canImport(SwiftUI) && canImport(AVFoundation) && !canImport(FoundationBridge)
|
|
2
|
+
import AVFoundation
|
|
3
|
+
import Foundation
|
|
4
|
+
import SwiftBrowserProtocol
|
|
5
|
+
|
|
6
|
+
/// `speechSynthesis` (packages/native/src/speech.ts) on `AVSpeechSynthesizer`:
|
|
7
|
+
/// `speak` queues an utterance (`text`, `lang` as the voice's language, `voice`
|
|
8
|
+
/// by name, `rate` with 1 the default rate, `pitch`, `volume`), `cancel`,
|
|
9
|
+
/// `pause` and `resume` act on the queue. Each utterance's events go to the
|
|
10
|
+
/// app as `__sbSpeechEvent(id, json)`:
|
|
11
|
+
///
|
|
12
|
+
/// ```
|
|
13
|
+
/// { "type": "start" | "end" | "cancel" | "pause" | "resume" | "boundary", "charIndex"?: n, "charLength"?: n }
|
|
14
|
+
/// ```
|
|
15
|
+
@MainActor
|
|
16
|
+
final class SpeechBridge: NSObject, AVSpeechSynthesizerDelegate {
|
|
17
|
+
private let send: (Int, JSONValue) -> Void
|
|
18
|
+
private let synthesizer = AVSpeechSynthesizer()
|
|
19
|
+
/// The utterances under way, by the synthesizer's object.
|
|
20
|
+
private var ids: [ObjectIdentifier: Int] = [:]
|
|
21
|
+
|
|
22
|
+
/// `send` hands an event for utterance `id` to the app.
|
|
23
|
+
init(send: @escaping (Int, JSONValue) -> Void) {
|
|
24
|
+
self.send = send
|
|
25
|
+
super.init()
|
|
26
|
+
synthesizer.delegate = self
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
func speak(id: Int, options: JSONObject) {
|
|
30
|
+
let utterance = AVSpeechUtterance(string: options.string("text") ?? "")
|
|
31
|
+
if let name = options.string("voice"), let voice = AVSpeechSynthesisVoice.speechVoices().first(where: { $0.name == name }) {
|
|
32
|
+
utterance.voice = voice
|
|
33
|
+
} else if let lang = options.string("lang"), let voice = AVSpeechSynthesisVoice(language: lang.replacingOccurrences(of: "_", with: "-")) {
|
|
34
|
+
utterance.voice = voice
|
|
35
|
+
}
|
|
36
|
+
// The API's rate is a multiple of the default; the synthesizer's runs from its minimum to its maximum around it.
|
|
37
|
+
let rate = Float(options["rate"]?.double ?? 1)
|
|
38
|
+
let base = AVSpeechUtteranceDefaultSpeechRate
|
|
39
|
+
utterance.rate = rate >= 1
|
|
40
|
+
? min(AVSpeechUtteranceMaximumSpeechRate, base + (AVSpeechUtteranceMaximumSpeechRate - base) * min(rate - 1, 9) / 9)
|
|
41
|
+
: max(AVSpeechUtteranceMinimumSpeechRate, AVSpeechUtteranceMinimumSpeechRate + (base - AVSpeechUtteranceMinimumSpeechRate) * max(rate, 0.1))
|
|
42
|
+
utterance.pitchMultiplier = Float(min(max(options["pitch"]?.double ?? 1, 0.5), 2))
|
|
43
|
+
utterance.volume = Float(min(max(options["volume"]?.double ?? 1, 0), 1))
|
|
44
|
+
ids[ObjectIdentifier(utterance)] = id
|
|
45
|
+
synthesizer.speak(utterance)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
func control(_ action: String) {
|
|
49
|
+
switch action {
|
|
50
|
+
case "cancel": synthesizer.stopSpeaking(at: .immediate)
|
|
51
|
+
case "pause": synthesizer.pauseSpeaking(at: .word)
|
|
52
|
+
case "resume": synthesizer.continueSpeaking()
|
|
53
|
+
default: break
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/// The app is going away.
|
|
58
|
+
func stop() {
|
|
59
|
+
synthesizer.stopSpeaking(at: .immediate)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
private func emit(_ key: ObjectIdentifier, _ type: String, _ range: NSRange? = nil, done: Bool = false) {
|
|
63
|
+
guard let id = ids[key] else { return }
|
|
64
|
+
var event: JSONObject = ["type": .string(type)]
|
|
65
|
+
if let range {
|
|
66
|
+
event["charIndex"] = .number(Double(range.location))
|
|
67
|
+
event["charLength"] = .number(Double(range.length))
|
|
68
|
+
}
|
|
69
|
+
if done { ids[key] = nil }
|
|
70
|
+
send(id, .object(event))
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// MARK: AVSpeechSynthesizerDelegate (hopped to the main actor; the utterance is named by identity)
|
|
74
|
+
|
|
75
|
+
private nonisolated func report(_ utterance: AVSpeechUtterance, _ type: String, _ range: NSRange? = nil, done: Bool = false) {
|
|
76
|
+
let key = ObjectIdentifier(utterance)
|
|
77
|
+
DispatchQueue.main.async { [weak self] in
|
|
78
|
+
MainActor.assumeIsolated { self?.emit(key, type, range, done: done) }
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didStart utterance: AVSpeechUtterance) {
|
|
83
|
+
report(utterance, "start")
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) {
|
|
87
|
+
report(utterance, "end", done: true)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) {
|
|
91
|
+
report(utterance, "cancel", done: true)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didPause utterance: AVSpeechUtterance) {
|
|
95
|
+
report(utterance, "pause")
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didContinue utterance: AVSpeechUtterance) {
|
|
99
|
+
report(utterance, "resume")
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance: AVSpeechUtterance) {
|
|
103
|
+
report(utterance, "boundary", characterRange)
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
#endif
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
#if canImport(SwiftUI) && canImport(Speech) && !canImport(FoundationBridge)
|
|
2
|
+
import AVFoundation
|
|
3
|
+
import Foundation
|
|
4
|
+
import Speech
|
|
5
|
+
import SwiftBrowserProtocol
|
|
6
|
+
|
|
7
|
+
/// `SpeechRecognition` (packages/native/src/speech.ts) on `SFSpeechRecognizer`
|
|
8
|
+
/// over the microphone (`AVAudioEngine`): `start` asks for speech recognition
|
|
9
|
+
/// and the microphone the first time (`NSSpeechRecognitionUsageDescription`,
|
|
10
|
+
/// `NSMicrophoneUsageDescription` in the app's Info.plist) and listens, with
|
|
11
|
+
/// partial results when `interimResults` or `continuous`; `stop` ends the
|
|
12
|
+
/// audio and lets the recognizer finish; `abort` ends it all. One session at
|
|
13
|
+
/// a time: a second `start` ends with `error: aborted`. The events go to the
|
|
14
|
+
/// app as `__sbRecognition(id, json)`:
|
|
15
|
+
///
|
|
16
|
+
/// ```
|
|
17
|
+
/// { "type": "start" }
|
|
18
|
+
/// { "type": "result", "transcript": string, "isFinal": bool, "confidence": n }
|
|
19
|
+
/// { "type": "error", "error": "not-allowed" | "audio-capture" | "network" | "no-speech" | "aborted", "message": string }
|
|
20
|
+
/// { "type": "end" }
|
|
21
|
+
/// ```
|
|
22
|
+
@MainActor
|
|
23
|
+
final class SpeechRecognitionBridge {
|
|
24
|
+
private let send: (Int, JSONValue) -> Void
|
|
25
|
+
private var engine: AVAudioEngine?
|
|
26
|
+
private var request: SFSpeechAudioBufferRecognitionRequest?
|
|
27
|
+
private var task: SFSpeechRecognitionTask?
|
|
28
|
+
private var recognizer: SFSpeechRecognizer?
|
|
29
|
+
/// The session under way: its id and whether it goes on after a final result.
|
|
30
|
+
private var current: (id: Int, continuous: Bool)?
|
|
31
|
+
|
|
32
|
+
/// `send` hands an event for recognition `id` to the app.
|
|
33
|
+
init(send: @escaping (Int, JSONValue) -> Void) {
|
|
34
|
+
self.send = send
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
func handle(id: Int, action: String, options: JSONObject) {
|
|
38
|
+
switch action {
|
|
39
|
+
case "start": start(id: id, options: options)
|
|
40
|
+
case "stop": if current?.id == id { request?.endAudio(); stopAudio() }
|
|
41
|
+
case "abort": if current?.id == id { finish(error: nil) }
|
|
42
|
+
default: break
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/// The app is going away.
|
|
47
|
+
func stop() {
|
|
48
|
+
if current != nil { finish(error: nil) }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
private func start(id: Int, options: JSONObject) {
|
|
52
|
+
if current != nil {
|
|
53
|
+
send(id, ["type": .string("error"), "error": .string("aborted"), "message": .string("another recognition is listening")])
|
|
54
|
+
send(id, ["type": .string("end")])
|
|
55
|
+
return
|
|
56
|
+
}
|
|
57
|
+
current = (id, options["continuous"]?.bool ?? false)
|
|
58
|
+
// Asking for speech or the microphone without the usage string in the
|
|
59
|
+
// app's Info.plist ends the app (iOS's privacy check): an error instead.
|
|
60
|
+
if let key = Self.missingUsageDescription() {
|
|
61
|
+
finish(error: ("not-allowed", "the app's Info.plist has no \(key)"))
|
|
62
|
+
return
|
|
63
|
+
}
|
|
64
|
+
let partial = (options["interimResults"]?.bool ?? false) || (options["continuous"]?.bool ?? false)
|
|
65
|
+
let locale = options.string("lang").map { Locale(identifier: $0.replacingOccurrences(of: "_", with: "-")) }
|
|
66
|
+
Task { @MainActor [weak self] in
|
|
67
|
+
guard let self else { return }
|
|
68
|
+
let authorized = await Self.authorize()
|
|
69
|
+
guard self.current?.id == id else { return }
|
|
70
|
+
guard authorized else {
|
|
71
|
+
self.finish(error: ("not-allowed", "speech recognition was not allowed"))
|
|
72
|
+
return
|
|
73
|
+
}
|
|
74
|
+
let found = locale.flatMap { SFSpeechRecognizer(locale: $0) } ?? SFSpeechRecognizer()
|
|
75
|
+
guard let recognizer = found, recognizer.isAvailable else {
|
|
76
|
+
self.finish(error: ("network", "the speech recognizer is not available"))
|
|
77
|
+
return
|
|
78
|
+
}
|
|
79
|
+
self.recognizer = recognizer
|
|
80
|
+
do {
|
|
81
|
+
try self.listen(recognizer, partial: partial, id: id)
|
|
82
|
+
self.send(id, ["type": .string("start")])
|
|
83
|
+
} catch {
|
|
84
|
+
self.finish(error: ("audio-capture", "the microphone could not be started: \(error.localizedDescription)"))
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/// The usage description the app's Info.plist lacks, if any: iOS ends an
|
|
90
|
+
/// app that asks for speech recognition or the microphone without one.
|
|
91
|
+
private static func missingUsageDescription() -> String? {
|
|
92
|
+
var keys = ["NSSpeechRecognitionUsageDescription"]
|
|
93
|
+
#if os(iOS)
|
|
94
|
+
keys.append("NSMicrophoneUsageDescription")
|
|
95
|
+
#endif
|
|
96
|
+
return keys.first { Bundle.main.object(forInfoDictionaryKey: $0) == nil }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
private nonisolated static func authorize() async -> Bool {
|
|
100
|
+
let speech = await withCheckedContinuation { (continuation: CheckedContinuation<SFSpeechRecognizerAuthorizationStatus, Never>) in
|
|
101
|
+
SFSpeechRecognizer.requestAuthorization { continuation.resume(returning: $0) }
|
|
102
|
+
}
|
|
103
|
+
guard speech == .authorized else { return false }
|
|
104
|
+
#if os(iOS)
|
|
105
|
+
return await AVAudioApplication.requestRecordPermission()
|
|
106
|
+
#else
|
|
107
|
+
return true
|
|
108
|
+
#endif
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private func listen(_ recognizer: SFSpeechRecognizer, partial: Bool, id: Int) throws {
|
|
112
|
+
#if os(iOS)
|
|
113
|
+
let session = AVAudioSession.sharedInstance()
|
|
114
|
+
try session.setCategory(.record, mode: .measurement, options: .duckOthers)
|
|
115
|
+
try session.setActive(true, options: .notifyOthersOnDeactivation)
|
|
116
|
+
#endif
|
|
117
|
+
let engine = AVAudioEngine()
|
|
118
|
+
let request = SFSpeechAudioBufferRecognitionRequest()
|
|
119
|
+
request.shouldReportPartialResults = partial
|
|
120
|
+
let input = engine.inputNode
|
|
121
|
+
let format = input.outputFormat(forBus: 0)
|
|
122
|
+
nonisolated(unsafe) let sink = request
|
|
123
|
+
input.installTap(onBus: 0, bufferSize: 1024, format: format) { buffer, _ in sink.append(buffer) }
|
|
124
|
+
engine.prepare()
|
|
125
|
+
try engine.start()
|
|
126
|
+
self.engine = engine
|
|
127
|
+
self.request = request
|
|
128
|
+
task = recognizer.recognitionTask(with: request, resultHandler: handler(id: id, partial: partial))
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/// What the recognizer reports, hopped to the main actor: a result to the
|
|
132
|
+
/// app (a final one ends a one-shot session or starts a continuous one's
|
|
133
|
+
/// next request), a failure as the end.
|
|
134
|
+
private func handler(id: Int, partial: Bool) -> @Sendable (SFSpeechRecognitionResult?, (any Error)?) -> Void {
|
|
135
|
+
{ result, error in
|
|
136
|
+
let heard = result.map { ($0.bestTranscription.formattedString, $0.isFinal, Double($0.bestTranscription.segments.last?.confidence ?? 0)) }
|
|
137
|
+
let failure = error?.localizedDescription
|
|
138
|
+
let code = (error as NSError?).map { ($0.domain, $0.code) }
|
|
139
|
+
DispatchQueue.main.async { [weak self] in
|
|
140
|
+
MainActor.assumeIsolated {
|
|
141
|
+
guard let self, let current = self.current, current.id == id else { return }
|
|
142
|
+
if let heard {
|
|
143
|
+
self.send(id, ["type": .string("result"), "transcript": .string(heard.0), "isFinal": .bool(heard.1), "confidence": .number(heard.2)])
|
|
144
|
+
if heard.1, !current.continuous { self.finish(error: nil) }
|
|
145
|
+
else if heard.1 { self.restart(partial: partial, id: id) }
|
|
146
|
+
}
|
|
147
|
+
if let failure {
|
|
148
|
+
// The recognizer ending after `stop` (no more audio) is the end, not an error; a session without a word is `no-speech`.
|
|
149
|
+
let assistant = code?.0 == "kAFAssistantErrorDomain"
|
|
150
|
+
let silent = assistant && code?.1 == 1110
|
|
151
|
+
let ended = assistant && code?.1 == 216
|
|
152
|
+
self.finish(error: ended ? nil : (silent ? "no-speech" : "network", failure))
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/// A continuous session goes on after a final result with a fresh request on the same engine.
|
|
160
|
+
private func restart(partial: Bool, id: Int) {
|
|
161
|
+
guard let engine, let recognizer else { return }
|
|
162
|
+
task?.cancel()
|
|
163
|
+
let request = SFSpeechAudioBufferRecognitionRequest()
|
|
164
|
+
request.shouldReportPartialResults = partial
|
|
165
|
+
let input = engine.inputNode
|
|
166
|
+
input.removeTap(onBus: 0)
|
|
167
|
+
nonisolated(unsafe) let sink = request
|
|
168
|
+
input.installTap(onBus: 0, bufferSize: 1024, format: input.outputFormat(forBus: 0)) { buffer, _ in sink.append(buffer) }
|
|
169
|
+
self.request = request
|
|
170
|
+
task = recognizer.recognitionTask(with: request, resultHandler: handler(id: id, partial: partial))
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
private func stopAudio() {
|
|
174
|
+
engine?.inputNode.removeTap(onBus: 0)
|
|
175
|
+
engine?.stop()
|
|
176
|
+
engine = nil
|
|
177
|
+
#if os(iOS)
|
|
178
|
+
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
|
179
|
+
#endif
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/// Ends the session: the audio, the task, and `error` (if any) then `end` to the app.
|
|
183
|
+
private func finish(error: (String, String)?) {
|
|
184
|
+
guard let current else { return }
|
|
185
|
+
let id = current.id
|
|
186
|
+
self.current = nil
|
|
187
|
+
stopAudio()
|
|
188
|
+
task?.cancel()
|
|
189
|
+
task = nil
|
|
190
|
+
request = nil
|
|
191
|
+
recognizer = nil
|
|
192
|
+
if let error {
|
|
193
|
+
send(id, ["type": .string("error"), "error": .string(error.0), "message": .string(error.1)])
|
|
194
|
+
}
|
|
195
|
+
send(id, ["type": .string("end")])
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
#endif
|
|
@@ -79,6 +79,8 @@ final class WorkerHost: @unchecked Sendable {
|
|
|
79
79
|
context.evaluateScript(script, withSourceURL: URL(string: "sb://\(name).worker.js"))
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
/// On the thread, in a turn's pool: the context, and with it the virtual
|
|
83
|
+
/// machine, go at the turn's end, while the thread and its run loop live.
|
|
82
84
|
private func tearDown() {
|
|
83
85
|
for timer in timers.values { timer.invalidate() }
|
|
84
86
|
timers.removeAll()
|
|
@@ -234,7 +236,8 @@ protocol WorkerEngine: AnyObject {
|
|
|
234
236
|
extension WorkerHost: WorkerEngine {}
|
|
235
237
|
|
|
236
238
|
/// A thread with a run loop that runs blocks sent to it, in order, after
|
|
237
|
-
/// the block it started with.
|
|
239
|
+
/// the block it started with. Each block, and each turn of the loop, runs in
|
|
240
|
+
/// an autorelease pool of its own.
|
|
238
241
|
final class WorkerThread: @unchecked Sendable {
|
|
239
242
|
private let label: String
|
|
240
243
|
private let lock = NSLock()
|
|
@@ -251,15 +254,21 @@ final class WorkerThread: @unchecked Sendable {
|
|
|
251
254
|
let loop = RunLoop.current
|
|
252
255
|
// A port keeps the loop running while nothing else is scheduled.
|
|
253
256
|
loop.add(Port(), forMode: .default)
|
|
254
|
-
|
|
257
|
+
// Every turn runs in an autorelease pool of its own, so what a turn
|
|
258
|
+
// autoreleases (each `JSValue` JavaScriptCore hands Swift) is gone
|
|
259
|
+
// at its end, on this thread, while the loop still runs: the
|
|
260
|
+
// worker's context dies when `tearDown` drops it, not at the
|
|
261
|
+
// thread's exit, where the engine's own per-thread state is going
|
|
262
|
+
// too (an exit-time release of the context has crashed the engine).
|
|
263
|
+
autoreleasepool { setUp() }
|
|
255
264
|
lock.lock()
|
|
256
265
|
runLoop = CFRunLoopGetCurrent()
|
|
257
266
|
let queued = pending
|
|
258
267
|
pending = []
|
|
259
268
|
lock.unlock()
|
|
260
|
-
for block in queued { block
|
|
269
|
+
for block in queued { autoreleasepool(invoking: block) }
|
|
261
270
|
while !isStopped {
|
|
262
|
-
loop.run(mode: .default, before: .distantFuture)
|
|
271
|
+
autoreleasepool { loop.run(mode: .default, before: .distantFuture) }
|
|
263
272
|
}
|
|
264
273
|
}
|
|
265
274
|
thread.name = label
|
|
@@ -276,7 +285,7 @@ final class WorkerThread: @unchecked Sendable {
|
|
|
276
285
|
return
|
|
277
286
|
}
|
|
278
287
|
lock.unlock()
|
|
279
|
-
CFRunLoopPerformBlock(runLoop, CFRunLoopMode.defaultMode.rawValue
|
|
288
|
+
CFRunLoopPerformBlock(runLoop, CFRunLoopMode.defaultMode.rawValue) { autoreleasepool(invoking: block) }
|
|
280
289
|
CFRunLoopWakeUp(runLoop)
|
|
281
290
|
}
|
|
282
291
|
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#if canImport(SwiftUI)
|
|
2
|
+
import Foundation
|
|
3
|
+
import SwiftBrowserProtocol
|
|
4
|
+
|
|
5
|
+
/// What a registered function is called with: the params JavaScript passed,
|
|
6
|
+
/// as JSON. `call["region"]` reads a member; `call.decode()` the whole object
|
|
7
|
+
/// as a `Decodable`.
|
|
8
|
+
public struct NativeFunctionCall: Sendable {
|
|
9
|
+
/// The name the function was registered under.
|
|
10
|
+
public let name: String
|
|
11
|
+
/// The params as JavaScript passed them (an object; `[:]` when none).
|
|
12
|
+
public let params: JSONValue
|
|
13
|
+
|
|
14
|
+
public subscript(key: String) -> JSONValue { params[key] }
|
|
15
|
+
|
|
16
|
+
/// The params decoded as a type of the app's.
|
|
17
|
+
public func decode<T: Decodable>(_ type: T.Type = T.self) throws -> T {
|
|
18
|
+
let data = try JSONEncoder().encode(params)
|
|
19
|
+
return try JSONDecoder().decode(type, from: data)
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/// A function throws this to say the viewer cancelled (closed a sheet, said
|
|
24
|
+
/// no): the app's promise rejects with an `AbortError`, as the system
|
|
25
|
+
/// pickers' cancellations do.
|
|
26
|
+
public struct NativeFunctionCancelled: Error, CustomStringConvertible {
|
|
27
|
+
public let description: String
|
|
28
|
+
public init(_ reason: String = "the viewer cancelled") { description = "cancelled: \(reason)" }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/// A failure with a message for the app: the promise rejects with it.
|
|
32
|
+
public struct NativeFunctionError: Error, CustomStringConvertible {
|
|
33
|
+
public let description: String
|
|
34
|
+
public init(_ description: String) { self.description = description }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/// Swift the app's JavaScript can call: the function half of the escape hatch
|
|
38
|
+
/// (docs/native.md; `NativeRegistry` is the view half). The Swift app
|
|
39
|
+
/// registers functions once, before the app loads, and `@swiftbrowser/native`
|
|
40
|
+
/// calls them: `callNative(name, params)` resolves with what an async function
|
|
41
|
+
/// returns, through the host request `app:<name>` (HostRequestBridge.swift);
|
|
42
|
+
/// `callNativeSync(name, params)` runs a sync one inline; `nativeFunctions()`
|
|
43
|
+
/// lists them. The other way, `send(name, value)` reaches
|
|
44
|
+
/// `onNativeEvent(name, listener)` in JavaScript, for what Swift reports
|
|
45
|
+
/// unasked: a StoreKit transaction, a motion update, a Bluetooth peripheral.
|
|
46
|
+
///
|
|
47
|
+
/// ```swift
|
|
48
|
+
/// NativeFunctions.register("purchase") { call in
|
|
49
|
+
/// let product = try await Product.products(for: [call["id"].string ?? ""]).first
|
|
50
|
+
/// ...
|
|
51
|
+
/// return ["state": "purchased"]
|
|
52
|
+
/// }
|
|
53
|
+
/// NativeFunctions.registerSync("deviceName") { _ in .string(UIDevice.current.name) }
|
|
54
|
+
/// ```
|
|
55
|
+
///
|
|
56
|
+
/// ```ts
|
|
57
|
+
/// const { state } = await callNative<{ state: string }>('purchase', { id: 'pro' });
|
|
58
|
+
/// const name = callNativeSync<string>('deviceName');
|
|
59
|
+
/// const stop = onNativeEvent('transaction', (value) => { ... });
|
|
60
|
+
/// ```
|
|
61
|
+
///
|
|
62
|
+
/// Everything runs on the main actor, as the rest of the bridge does; a
|
|
63
|
+
/// function doing heavy work hops off it itself. Values cross as JSON. An
|
|
64
|
+
/// error a function throws rejects the promise with its description; a
|
|
65
|
+
/// `NativeFunctionCancelled` as an `AbortError`.
|
|
66
|
+
@MainActor
|
|
67
|
+
public enum NativeFunctions {
|
|
68
|
+
public typealias Handler = (NativeFunctionCall) async throws -> JSONValue
|
|
69
|
+
public typealias SyncHandler = (NativeFunctionCall) throws -> JSONValue
|
|
70
|
+
|
|
71
|
+
private static var handlers: [String: Handler] = [:]
|
|
72
|
+
private static var syncHandlers: [String: SyncHandler] = [:]
|
|
73
|
+
/// Where `send` delivers: the running app's bridge (HostBridge.swift sets
|
|
74
|
+
/// it as it installs, clears it as it goes away). Nothing listening, the
|
|
75
|
+
/// value is dropped.
|
|
76
|
+
static var sink: ((String, JSONValue) -> Void)?
|
|
77
|
+
|
|
78
|
+
/// Registers an async function under `name`, replacing one registered before.
|
|
79
|
+
public static func register(_ name: String, _ handler: @escaping Handler) {
|
|
80
|
+
syncHandlers[name] = nil
|
|
81
|
+
handlers[name] = handler
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/// Registers a function JavaScript calls inline with `callNativeSync`,
|
|
85
|
+
/// replacing one registered before; `callNative` reaches it too.
|
|
86
|
+
public static func registerSync(_ name: String, _ handler: @escaping SyncHandler) {
|
|
87
|
+
handlers[name] = nil
|
|
88
|
+
syncHandlers[name] = handler
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
public static func unregister(_ name: String) {
|
|
92
|
+
handlers[name] = nil
|
|
93
|
+
syncHandlers[name] = nil
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
public static func isRegistered(_ name: String) -> Bool {
|
|
97
|
+
handlers[name] != nil || syncHandlers[name] != nil
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/// Every registered name, sorted: what `nativeFunctions()` answers.
|
|
101
|
+
public static var names: [String] {
|
|
102
|
+
Array(Set(handlers.keys).union(syncHandlers.keys)).sorted()
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/// Hands `value` to the app's `onNativeEvent(name)` listeners.
|
|
106
|
+
public static func send(_ name: String, _ value: JSONValue) {
|
|
107
|
+
sink?(name, value)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/// The host request `app:<name>`: the async function, or the sync one.
|
|
111
|
+
static func call(_ name: String, _ params: JSONObject) async throws -> JSONValue {
|
|
112
|
+
let call = NativeFunctionCall(name: name, params: .object(params))
|
|
113
|
+
if let handler = handlers[name] { return try await handler(call) }
|
|
114
|
+
if let handler = syncHandlers[name] { return try handler(call) }
|
|
115
|
+
throw NativeFunctionError("unsupported: no native function named \(name)")
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/// `callNativeSync(name, json)`: `{ value }` or `{ error }`, as a host
|
|
119
|
+
/// request answers, for a sync function; an async one is an error here.
|
|
120
|
+
static func callSync(_ name: String, _ json: String) -> JSONValue {
|
|
121
|
+
let params = (try? JSONValue.parse(json))?.object ?? [:]
|
|
122
|
+
guard let handler = syncHandlers[name] else {
|
|
123
|
+
let reason = handlers[name] == nil ? "no native function named \(name)" : "\(name) is async: call it with callNative"
|
|
124
|
+
return .object(["error": .string("unsupported: \(reason)")])
|
|
125
|
+
}
|
|
126
|
+
do {
|
|
127
|
+
let value = try handler(NativeFunctionCall(name: name, params: .object(params)))
|
|
128
|
+
return .object(["value": value])
|
|
129
|
+
} catch {
|
|
130
|
+
return .object(["error": .string("\(error)")])
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
#endif
|
|
@@ -53,6 +53,7 @@ struct ElementView: View {
|
|
|
53
53
|
case "stepper": return AnyView(StepperElement(node: node))
|
|
54
54
|
case "tapgesture": return AnyView(TapGestureElement(node: node))
|
|
55
55
|
case "gesture": return AnyView(GestureElement(node: node))
|
|
56
|
+
case "touches": return AnyView(TouchesElement(node: node))
|
|
56
57
|
case "gauge": return AnyView(GaugeElement(node: node))
|
|
57
58
|
case "datepicker": return AnyView(DatePickerElement(node: node))
|
|
58
59
|
case "menu": return AnyView(MenuElement(node: node))
|
|
@@ -127,6 +127,8 @@ struct TextFieldElement: View {
|
|
|
127
127
|
})
|
|
128
128
|
}
|
|
129
129
|
if let label = props.submitLabel { styled = AnyView(styled.submitLabel(SubmitLabel(name: label))) }
|
|
130
|
+
// The keyboard, what autofill offers, capitalization and correction (TextInputHints.swift).
|
|
131
|
+
styled = AnyView(styled.textInputHints(props))
|
|
130
132
|
// Backspace at the start of the text (DeleteBackward.swift).
|
|
131
133
|
if props.deleteBackward { styled = AnyView(styled.onDeleteBackward(true) { app?.dispatch(.deleteBackward(node.id)) }) }
|
|
132
134
|
if props.focused != nil {
|