@extentos/mcp-server 0.0.56 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -62
- package/dist/generated/events.d.ts +3 -3
- package/dist/generated/events.d.ts.map +1 -1
- package/dist/generated/events.js +11 -6
- package/dist/generated/events.js.map +1 -1
- package/dist/generated/schemas.d.ts.map +1 -1
- package/dist/generated/schemas.js +41 -124
- package/dist/generated/schemas.js.map +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/tools/data/capabilityPatterns.d.ts +11 -0
- package/dist/tools/data/capabilityPatterns.d.ts.map +1 -0
- package/dist/tools/data/capabilityPatterns.js +289 -0
- package/dist/tools/data/capabilityPatterns.js.map +1 -0
- package/dist/tools/data/codeExamples.d.ts +15 -0
- package/dist/tools/data/codeExamples.d.ts.map +1 -0
- package/dist/tools/data/codeExamples.js +494 -0
- package/dist/tools/data/codeExamples.js.map +1 -0
- package/dist/tools/data/version.js +7 -7
- package/dist/tools/definitions.d.ts.map +1 -1
- package/dist/tools/definitions.js +99 -181
- package/dist/tools/definitions.js.map +1 -1
- package/dist/tools/docs/index.d.ts.map +1 -1
- package/dist/tools/docs/index.js +321 -803
- package/dist/tools/docs/index.js.map +1 -1
- package/dist/tools/handlers/createSimulatorSession.d.ts.map +1 -1
- package/dist/tools/handlers/createSimulatorSession.js +36 -51
- package/dist/tools/handlers/createSimulatorSession.js.map +1 -1
- package/dist/tools/handlers/generateConnectionModule.js +14 -15
- package/dist/tools/handlers/generateConnectionModule.js.map +1 -1
- package/dist/tools/handlers/getCapabilityGuide.d.ts +3 -0
- package/dist/tools/handlers/getCapabilityGuide.d.ts.map +1 -0
- package/dist/tools/handlers/getCapabilityGuide.js +47 -0
- package/dist/tools/handlers/getCapabilityGuide.js.map +1 -0
- package/dist/tools/handlers/getCodeExample.d.ts +3 -0
- package/dist/tools/handlers/getCodeExample.d.ts.map +1 -0
- package/dist/tools/handlers/getCodeExample.js +50 -0
- package/dist/tools/handlers/getCodeExample.js.map +1 -0
- package/dist/tools/handlers/getCredentialGuide.d.ts.map +1 -1
- package/dist/tools/handlers/getCredentialGuide.js +44 -84
- package/dist/tools/handlers/getCredentialGuide.js.map +1 -1
- package/dist/tools/handlers/getEventLog.d.ts.map +1 -1
- package/dist/tools/handlers/getEventLog.js +7 -8
- package/dist/tools/handlers/getEventLog.js.map +1 -1
- package/dist/tools/handlers/getPermissions.d.ts.map +1 -1
- package/dist/tools/handlers/getPermissions.js +27 -12
- package/dist/tools/handlers/getPermissions.js.map +1 -1
- package/dist/tools/handlers/getPlatformInfo.d.ts.map +1 -1
- package/dist/tools/handlers/getPlatformInfo.js +112 -46
- package/dist/tools/handlers/getPlatformInfo.js.map +1 -1
- package/dist/tools/handlers/getProductionChecklist.d.ts.map +1 -1
- package/dist/tools/handlers/getProductionChecklist.js +86 -120
- package/dist/tools/handlers/getProductionChecklist.js.map +1 -1
- package/dist/tools/handlers/getSimulatorStatus.d.ts.map +1 -1
- package/dist/tools/handlers/getSimulatorStatus.js +1 -3
- package/dist/tools/handlers/getSimulatorStatus.js.map +1 -1
- package/dist/tools/handlers/getVoiceCommandGuidance.d.ts.map +1 -1
- package/dist/tools/handlers/getVoiceCommandGuidance.js +18 -45
- package/dist/tools/handlers/getVoiceCommandGuidance.js.map +1 -1
- package/dist/tools/handlers/inspectIntegration.d.ts.map +1 -1
- package/dist/tools/handlers/inspectIntegration.js +14 -52
- package/dist/tools/handlers/inspectIntegration.js.map +1 -1
- package/dist/tools/handlers/validateIntegration.d.ts.map +1 -1
- package/dist/tools/handlers/validateIntegration.js +83 -167
- package/dist/tools/handlers/validateIntegration.js.map +1 -1
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +6 -11
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/templates/androidBootstrap.d.ts.map +1 -1
- package/dist/tools/templates/androidBootstrap.js +20 -47
- package/dist/tools/templates/androidBootstrap.js.map +1 -1
- package/dist/tools/templates/iosBootstrap.d.ts.map +1 -1
- package/dist/tools/templates/iosBootstrap.js +24 -34
- package/dist/tools/templates/iosBootstrap.js.map +1 -1
- package/dist/tools/util/manifest.d.ts +0 -61
- package/dist/tools/util/manifest.d.ts.map +1 -1
- package/dist/tools/util/manifest.js +14 -112
- package/dist/tools/util/manifest.js.map +1 -1
- package/dist/tools/util/permissions.d.ts +4 -7
- package/dist/tools/util/permissions.d.ts.map +1 -1
- package/dist/tools/util/permissions.js +151 -161
- package/dist/tools/util/permissions.js.map +1 -1
- package/package.json +1 -6
|
@@ -0,0 +1,494 @@
|
|
|
1
|
+
// Code-example catalog for getCodeExample. Each entry is a complete
|
|
2
|
+
// SDK pattern in both Kotlin (Android) and Swift (iOS). These are the
|
|
3
|
+
// canonical compositions a fresh agent should peel from when building
|
|
4
|
+
// the most common voice-glasses use cases. Names match
|
|
5
|
+
// `getCodeExample({pattern: "..."})`.
|
|
6
|
+
//
|
|
7
|
+
// Authorship contract: every example COMPILES against the post-pivot
|
|
8
|
+
// library APIs (`glasses.audio.*`, `glasses.camera.*`,
|
|
9
|
+
// `glasses.connection.state`, `glasses.toggles.*`). No spec runtime, no
|
|
10
|
+
// app_callback, no triggers/blocks/actions DSL. Customer-side LLM
|
|
11
|
+
// clients are referenced as `anthropic` / `vision` / `repo` without
|
|
12
|
+
// importing — agents wire those to their existing backend.
|
|
13
|
+
const VOICE_QA_ASSISTANT = {
|
|
14
|
+
pattern: "voice_qa_assistant",
|
|
15
|
+
title: "Voice Q&A assistant (wake + question + LLM + speak)",
|
|
16
|
+
description: "The canonical Strava-style flow: user says a wake phrase, AI greets, captures the user's free-form question with silence-VAD, calls the LLM, speaks the answer. Multi-turn loop optional. This is the F21 / F23 friction-log resolution — the single use case that motivated the pure-SDK pivot.",
|
|
17
|
+
code: {
|
|
18
|
+
kotlin: `class CoachHandler(
|
|
19
|
+
private val glasses: ExtentosGlasses,
|
|
20
|
+
private val anthropic: AnthropicClient,
|
|
21
|
+
private val workouts: WorkoutRepository,
|
|
22
|
+
) {
|
|
23
|
+
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
|
24
|
+
|
|
25
|
+
fun start() = scope.launch {
|
|
26
|
+
glasses.audio.transcriptions().collect { t ->
|
|
27
|
+
if (!t.isFinal) return@collect
|
|
28
|
+
if ("ask my coach" !in t.text.lowercase()) return@collect
|
|
29
|
+
|
|
30
|
+
glasses.audio.speak("What would you like to know?")
|
|
31
|
+
|
|
32
|
+
// Multi-turn loop: keep listening until user goes silent or
|
|
33
|
+
// says "stop". Each turn captures one utterance via the
|
|
34
|
+
// built-in silence-VAD (silence_timeout_seconds = 3).
|
|
35
|
+
while (currentCoroutineContext().isActive) {
|
|
36
|
+
val recording = glasses.audio.recordDiscrete(
|
|
37
|
+
AudioRecordConfig(silenceTimeoutSeconds = 3.0)
|
|
38
|
+
).getOrNull() ?: break
|
|
39
|
+
|
|
40
|
+
val question = recording.transcript.trim()
|
|
41
|
+
if (question.isEmpty() || "stop" in question.lowercase()) break
|
|
42
|
+
|
|
43
|
+
val answer = anthropic.ask(
|
|
44
|
+
question = question,
|
|
45
|
+
workoutHistory = workouts.recent(),
|
|
46
|
+
)
|
|
47
|
+
glasses.audio.speak(answer)
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
fun stop() = scope.cancel()
|
|
53
|
+
}`,
|
|
54
|
+
swift: `final class CoachHandler: @unchecked Sendable {
|
|
55
|
+
private let glasses: any ExtentosGlasses
|
|
56
|
+
private let anthropic: AnthropicClient
|
|
57
|
+
private let workouts: WorkoutRepository
|
|
58
|
+
private var task: Task<Void, Never>?
|
|
59
|
+
|
|
60
|
+
init(glasses: any ExtentosGlasses, anthropic: AnthropicClient, workouts: WorkoutRepository) {
|
|
61
|
+
self.glasses = glasses
|
|
62
|
+
self.anthropic = anthropic
|
|
63
|
+
self.workouts = workouts
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
func start() {
|
|
67
|
+
task = Task { [glasses, anthropic, workouts] in
|
|
68
|
+
for await t in glasses.audio.transcriptions() {
|
|
69
|
+
guard case .final(let text, _, _, _) = t,
|
|
70
|
+
text.lowercased().contains("ask my coach")
|
|
71
|
+
else { continue }
|
|
72
|
+
|
|
73
|
+
_ = await glasses.audio.speak("What would you like to know?")
|
|
74
|
+
|
|
75
|
+
// Multi-turn loop — silence-VAD per turn.
|
|
76
|
+
while !Task.isCancelled {
|
|
77
|
+
let result = await glasses.audio.recordDiscrete(
|
|
78
|
+
config: AudioRecordConfig(silenceTimeoutSeconds: 3)
|
|
79
|
+
)
|
|
80
|
+
guard case .success(let recording) = result else { break }
|
|
81
|
+
|
|
82
|
+
let question = recording.transcript.trimmingCharacters(in: .whitespaces)
|
|
83
|
+
if question.isEmpty || question.lowercased().contains("stop") { break }
|
|
84
|
+
|
|
85
|
+
if let answer = try? await anthropic.ask(
|
|
86
|
+
question: question,
|
|
87
|
+
workoutHistory: workouts.recent()
|
|
88
|
+
) {
|
|
89
|
+
_ = await glasses.audio.speak(answer)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
func stop() { task?.cancel() }
|
|
97
|
+
}`,
|
|
98
|
+
},
|
|
99
|
+
explanation: "Three SDK primitives wired together: (1) glasses.audio.transcriptions() — continuous Flow of partial+final transcripts; customer matches the wake phrase via plain string contains. (2) glasses.audio.recordDiscrete(silenceTimeoutSeconds) — one-shot capture that auto-stops on N seconds of silence and returns the transcribed text. (3) glasses.audio.speak() — TTS via the platform engine. The while-loop is multi-turn voice mode (the F23 friction-log finding); break-condition is silence (recordDiscrete returns null), an empty utterance, or the user saying 'stop'.",
|
|
100
|
+
gotchas: [
|
|
101
|
+
"transcriptions() yields BOTH partial and final transcripts — always guard on t.isFinal before matching, or you'll fire the wake on a half-heard phrase.",
|
|
102
|
+
"recordDiscrete blocks until silence — call it from a coroutine/Task that's safe to suspend. Inside the while-loop is fine because it's already in a launched scope.",
|
|
103
|
+
"Don't subscribe to transcriptions() inside the while-loop — that creates a new recognizer per turn. Subscribe once outside the loop.",
|
|
104
|
+
"speak() blocks until done by default. If you want speak + listen-for-barge-in simultaneously, see the barge_in_speak pattern.",
|
|
105
|
+
"BuildConfig API-key fields don't reflect rotations because kotlinc inlines them at compile time. Use resValue / R.string.X for Android secrets. See getCredentialGuide.",
|
|
106
|
+
],
|
|
107
|
+
relatedFeatures: ["voice_command", "transcription_incremental", "record_audio"],
|
|
108
|
+
};
|
|
109
|
+
const BARGE_IN_SPEAK = {
|
|
110
|
+
pattern: "barge_in_speak",
|
|
111
|
+
title: "Barge-in: cancel TTS when user starts speaking",
|
|
112
|
+
description: "The user interrupts the AI mid-response by talking. The AI shuts up immediately and listens. This is the F24 friction-log resolution; needs glasses.audio.cancelSpeak() which lands with this commit.",
|
|
113
|
+
code: {
|
|
114
|
+
kotlin: `suspend fun speakInterruptible(
|
|
115
|
+
glasses: ExtentosGlasses,
|
|
116
|
+
text: String,
|
|
117
|
+
) = coroutineScope {
|
|
118
|
+
val speakJob = launch { glasses.audio.speak(text) }
|
|
119
|
+
val bargeIn = async {
|
|
120
|
+
glasses.audio.transcriptions()
|
|
121
|
+
.filter { it.isFinal && it.text.isNotBlank() }
|
|
122
|
+
.first()
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Whichever finishes first wins. If TTS completes naturally,
|
|
126
|
+
// bargeIn keeps waiting — cancel it to clean up.
|
|
127
|
+
speakJob.join()
|
|
128
|
+
if (bargeIn.isCompleted) {
|
|
129
|
+
// User barged in. Stop TTS, then handle the interrupting
|
|
130
|
+
// utterance (e.g. feed it back into your conversation loop).
|
|
131
|
+
glasses.audio.cancelSpeak()
|
|
132
|
+
val utterance = bargeIn.await()
|
|
133
|
+
handleInterrupt(utterance.text)
|
|
134
|
+
} else {
|
|
135
|
+
bargeIn.cancel()
|
|
136
|
+
}
|
|
137
|
+
}`,
|
|
138
|
+
swift: `func speakInterruptible(
|
|
139
|
+
glasses: any ExtentosGlasses,
|
|
140
|
+
text: String
|
|
141
|
+
) async {
|
|
142
|
+
await withTaskGroup(of: BargeOutcome.self) { group in
|
|
143
|
+
group.addTask {
|
|
144
|
+
_ = await glasses.audio.speak(text)
|
|
145
|
+
return .speakFinished
|
|
146
|
+
}
|
|
147
|
+
group.addTask {
|
|
148
|
+
for await t in glasses.audio.transcriptions() {
|
|
149
|
+
if case .final(let utterance, _, _, _) = t, !utterance.isEmpty {
|
|
150
|
+
return .bargedIn(utterance)
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return .speakFinished
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Take whichever completes first.
|
|
157
|
+
if let first = await group.next() {
|
|
158
|
+
group.cancelAll()
|
|
159
|
+
if case .bargedIn(let utterance) = first {
|
|
160
|
+
await glasses.audio.cancelSpeak()
|
|
161
|
+
await handleInterrupt(utterance)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
enum BargeOutcome {
|
|
168
|
+
case speakFinished
|
|
169
|
+
case bargedIn(String)
|
|
170
|
+
}`,
|
|
171
|
+
},
|
|
172
|
+
explanation: "Run speak() and a transcription-listener concurrently. The first to finish wins. If the user starts speaking before TTS finishes, cancelSpeak() shuts the engine off mid-utterance (Android tts.stop() / iOS synthesizer.stopSpeaking(at: .immediate)) and the customer handles the interrupting text. The race-and-cancel pattern is standard coroutine / TaskGroup territory; the SDK piece is glasses.audio.cancelSpeak().",
|
|
173
|
+
gotchas: [
|
|
174
|
+
"cancelSpeak() is fire-and-forget — it doesn't await a 'really stopped' confirmation. The platform TTS engines have their own state machines.",
|
|
175
|
+
"If transcriptions() captures the AI's own voice bleeding through the mic, the loop fires on its own speech. Meta DAT's HFP profile generally suppresses this via hardware echo cancellation, but verify on real hardware — BrowserSim doesn't replicate it.",
|
|
176
|
+
"Don't filter Partial transcripts unless you really mean it; the user's first half-syllable shouldn't trigger barge-in. Wait for isFinal.",
|
|
177
|
+
],
|
|
178
|
+
relatedFeatures: ["transcription_incremental"],
|
|
179
|
+
};
|
|
180
|
+
const PHOTO_DESCRIBE_VOICE = {
|
|
181
|
+
pattern: "photo_describe_voice",
|
|
182
|
+
title: "Voice-activated photo + vision-LLM description",
|
|
183
|
+
description: "User says 'describe what you see', glasses capture a photo, customer's vision LLM (Anthropic Claude Vision / OpenAI GPT-4V / Gemini) describes it, AI speaks the description. The canonical 'AI vision on glasses' demo.",
|
|
184
|
+
code: {
|
|
185
|
+
kotlin: `class VisionHandler(
|
|
186
|
+
private val glasses: ExtentosGlasses,
|
|
187
|
+
private val vision: VisionClient,
|
|
188
|
+
) {
|
|
189
|
+
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
|
190
|
+
|
|
191
|
+
fun start() = scope.launch {
|
|
192
|
+
glasses.audio.transcriptions().collect { t ->
|
|
193
|
+
if (!t.isFinal) return@collect
|
|
194
|
+
if ("describe what you see" !in t.text.lowercase()) return@collect
|
|
195
|
+
|
|
196
|
+
val photo = glasses.camera.capturePhoto(
|
|
197
|
+
PhotoConfig(resolution = "MEDIUM", format = "jpeg")
|
|
198
|
+
).getOrNull() ?: run {
|
|
199
|
+
glasses.audio.speak("I couldn't take a photo.")
|
|
200
|
+
return@collect
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
val description = vision.describe(
|
|
204
|
+
imageUri = photo.uri,
|
|
205
|
+
prompt = "Describe this scene briefly, conversationally.",
|
|
206
|
+
)
|
|
207
|
+
glasses.audio.speak(description.ifBlank { "I couldn't describe that one." })
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
fun stop() = scope.cancel()
|
|
212
|
+
}`,
|
|
213
|
+
swift: `final class VisionHandler: @unchecked Sendable {
|
|
214
|
+
private let glasses: any ExtentosGlasses
|
|
215
|
+
private let vision: VisionClient
|
|
216
|
+
private var task: Task<Void, Never>?
|
|
217
|
+
|
|
218
|
+
init(glasses: any ExtentosGlasses, vision: VisionClient) {
|
|
219
|
+
self.glasses = glasses
|
|
220
|
+
self.vision = vision
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
func start() {
|
|
224
|
+
task = Task { [glasses, vision] in
|
|
225
|
+
for await t in glasses.audio.transcriptions() {
|
|
226
|
+
guard case .final(let text, _, _, _) = t,
|
|
227
|
+
text.lowercased().contains("describe what you see")
|
|
228
|
+
else { continue }
|
|
229
|
+
|
|
230
|
+
let captureResult = await glasses.camera.capturePhoto(
|
|
231
|
+
config: PhotoConfig(resolution: .medium, format: .jpeg)
|
|
232
|
+
)
|
|
233
|
+
guard case .success(let photo) = captureResult else {
|
|
234
|
+
_ = await glasses.audio.speak("I couldn't take a photo.")
|
|
235
|
+
continue
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// Photo.url is a URL whose scheme varies by transport
|
|
239
|
+
// (file:// on RealMeta / LocalSim, data: on BrowserSim).
|
|
240
|
+
// Decode via the bundled extension; pass the bytes to
|
|
241
|
+
// your vision LLM however it expects them (base64, Data).
|
|
242
|
+
let image = await photo.loadImage()
|
|
243
|
+
let description = (try? await vision.describe(
|
|
244
|
+
image: image,
|
|
245
|
+
prompt: "Describe this scene briefly, conversationally."
|
|
246
|
+
)) ?? ""
|
|
247
|
+
let answer = description.isEmpty
|
|
248
|
+
? "I couldn't describe that one."
|
|
249
|
+
: description
|
|
250
|
+
_ = await glasses.audio.speak(answer)
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
func stop() { task?.cancel() }
|
|
256
|
+
}`,
|
|
257
|
+
},
|
|
258
|
+
explanation: "Voice wake → photo capture → vision LLM → spoken response. The photo URL shape varies by transport (data: in BrowserSim, file: in RealMeta + LocalSim). Android consumers use Photos.loadBase64(uri) / Photos.mediaTypeFromUri(uri) helpers; iOS consumers use the bundled `photo.loadImage()` extension on Photo to materialize a UIImage from either scheme.",
|
|
259
|
+
gotchas: [
|
|
260
|
+
"capturePhoto blocks until the photo is captured (~300-500ms on Meta DAT). Don't call it on a UI thread.",
|
|
261
|
+
"Android: Photos.loadBase64(uri) bridges both data: and file: URIs into bytes ready for Claude Vision / GPT-4V request bodies. iOS: `photo.loadImage()` (extension on Photo) returns a UIImage; for base64 you encode the JPEG/PNG data yourself via Data.base64EncodedString().",
|
|
262
|
+
"Vision LLM response time is 2-8 seconds p95; the user is staring at the glasses waiting. Consider an earcon at capture time so they know it worked.",
|
|
263
|
+
],
|
|
264
|
+
relatedFeatures: ["capture_photo", "voice_command", "transcription_incremental"],
|
|
265
|
+
};
|
|
266
|
+
const LIVE_TRANSCRIPTION_UI = {
|
|
267
|
+
pattern: "live_transcription_ui",
|
|
268
|
+
title: "Live transcription feeding into your app's UI",
|
|
269
|
+
description: "Continuous transcripts from the glasses mic, surfaced as live captions in your existing Compose / SwiftUI state. The canonical 'live captions' / meeting-notes / translation-app shape.",
|
|
270
|
+
code: {
|
|
271
|
+
kotlin: `class TranscriptionViewModel(
|
|
272
|
+
private val glasses: ExtentosGlasses,
|
|
273
|
+
) : ViewModel() {
|
|
274
|
+
private val _transcript = MutableStateFlow<String>("")
|
|
275
|
+
val transcript: StateFlow<String> = _transcript.asStateFlow()
|
|
276
|
+
|
|
277
|
+
init {
|
|
278
|
+
viewModelScope.launch {
|
|
279
|
+
glasses.audio.transcriptions(
|
|
280
|
+
TranscriptionConfig(language = "en-US")
|
|
281
|
+
).collect { t ->
|
|
282
|
+
if (t.isFinal) {
|
|
283
|
+
// Append finalized segments; partials replace the
|
|
284
|
+
// tentative tail. Simple naive concatenation here —
|
|
285
|
+
// production apps usually segment by speaker / time.
|
|
286
|
+
_transcript.value = _transcript.value + " " + t.text
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// In your Compose UI:
|
|
294
|
+
// val transcript by viewModel.transcript.collectAsState()
|
|
295
|
+
// Text(transcript)`,
|
|
296
|
+
swift: `@MainActor
|
|
297
|
+
final class TranscriptionViewModel: ObservableObject {
|
|
298
|
+
@Published var transcript: String = ""
|
|
299
|
+
private var task: Task<Void, Never>?
|
|
300
|
+
|
|
301
|
+
init(glasses: any ExtentosGlasses) {
|
|
302
|
+
// The class is @MainActor-isolated, so the Task created here
|
|
303
|
+
// inherits MainActor isolation — mutating @Published transcript
|
|
304
|
+
// is synchronous-on-main, no MainActor.run hop required.
|
|
305
|
+
let stream = glasses.audio.transcriptions(
|
|
306
|
+
config: TranscriptionConfig(language: "en-US")
|
|
307
|
+
)
|
|
308
|
+
task = Task { [weak self] in
|
|
309
|
+
for await t in stream {
|
|
310
|
+
guard case .final(let text, _, _, _) = t else { continue }
|
|
311
|
+
self?.transcript += " " + text
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
deinit { task?.cancel() }
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// In your SwiftUI view:
|
|
320
|
+
// @StateObject var vm = TranscriptionViewModel(glasses: glasses)
|
|
321
|
+
// Text(vm.transcript)`,
|
|
322
|
+
},
|
|
323
|
+
explanation: "Subscribe to transcriptions() once at view-model init, append final segments to your state. Compose / SwiftUI re-renders on the StateFlow / @Published update. The library auto-restarts the recognizer between utterances to keep the stream continuous (~100-300ms gap).",
|
|
324
|
+
gotchas: [
|
|
325
|
+
"transcriptions() is gated by listening_mode + audio_capture_enabled + privacy_mode toggles. If transcripts stop arriving unexpectedly, check those — likely listening_mode flipped to 'off' from a UI control.",
|
|
326
|
+
"Mic audio routes over HFP while transcriptions are active; A2DP music playback drops to mono 8 kHz for the duration. Show a 'listening…' indicator so the user knows why.",
|
|
327
|
+
"Partial transcripts arrive 5-20 per second on a normal-speed utterance. Throttle / debounce in your UI if you render every emission — rendering 200 Compose updates per second is wasteful.",
|
|
328
|
+
],
|
|
329
|
+
relatedFeatures: ["transcription_incremental"],
|
|
330
|
+
};
|
|
331
|
+
const VOICE_NOTES = {
|
|
332
|
+
pattern: "voice_notes",
|
|
333
|
+
title: "Voice note (wake + record + persist)",
|
|
334
|
+
description: "'Take a note' style flow: wake phrase → record_audio with silence-VAD → save the transcript to the customer's notes repository (Notion, Supabase, filesystem, etc.).",
|
|
335
|
+
code: {
|
|
336
|
+
kotlin: `class NotesHandler(
|
|
337
|
+
private val glasses: ExtentosGlasses,
|
|
338
|
+
private val notes: NotesRepository,
|
|
339
|
+
) {
|
|
340
|
+
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
|
341
|
+
|
|
342
|
+
fun start() = scope.launch {
|
|
343
|
+
glasses.audio.transcriptions().collect { t ->
|
|
344
|
+
if (!t.isFinal) return@collect
|
|
345
|
+
if ("take a note" !in t.text.lowercase()) return@collect
|
|
346
|
+
|
|
347
|
+
glasses.audio.speak("Recording.")
|
|
348
|
+
val recording = glasses.audio.recordDiscrete(
|
|
349
|
+
AudioRecordConfig(
|
|
350
|
+
maxDurationSeconds = 30,
|
|
351
|
+
silenceTimeoutSeconds = 3.0,
|
|
352
|
+
)
|
|
353
|
+
).getOrNull() ?: run {
|
|
354
|
+
glasses.audio.speak("I didn't catch that.")
|
|
355
|
+
return@collect
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
if (recording.transcript.isBlank()) {
|
|
359
|
+
glasses.audio.speak("I didn't catch that.")
|
|
360
|
+
return@collect
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
notes.save(recording.transcript)
|
|
364
|
+
glasses.audio.speak("Saved.")
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
fun stop() = scope.cancel()
|
|
369
|
+
}`,
|
|
370
|
+
swift: `final class NotesHandler: @unchecked Sendable {
|
|
371
|
+
private let glasses: any ExtentosGlasses
|
|
372
|
+
private let notes: NotesRepository
|
|
373
|
+
private var task: Task<Void, Never>?
|
|
374
|
+
|
|
375
|
+
init(glasses: any ExtentosGlasses, notes: NotesRepository) {
|
|
376
|
+
self.glasses = glasses
|
|
377
|
+
self.notes = notes
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
func start() {
|
|
381
|
+
task = Task { [glasses, notes] in
|
|
382
|
+
for await t in glasses.audio.transcriptions() {
|
|
383
|
+
guard case .final(let text, _, _, _) = t,
|
|
384
|
+
text.lowercased().contains("take a note")
|
|
385
|
+
else { continue }
|
|
386
|
+
|
|
387
|
+
_ = await glasses.audio.speak("Recording.")
|
|
388
|
+
let result = await glasses.audio.recordDiscrete(
|
|
389
|
+
config: AudioRecordConfig(
|
|
390
|
+
maxDurationSeconds: 30,
|
|
391
|
+
silenceTimeoutSeconds: 3
|
|
392
|
+
)
|
|
393
|
+
)
|
|
394
|
+
guard case .success(let recording) = result,
|
|
395
|
+
!recording.transcript.isEmpty
|
|
396
|
+
else {
|
|
397
|
+
_ = await glasses.audio.speak("I didn't catch that.")
|
|
398
|
+
continue
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
await notes.save(recording.transcript)
|
|
402
|
+
_ = await glasses.audio.speak("Saved.")
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
func stop() { task?.cancel() }
|
|
408
|
+
}`,
|
|
409
|
+
},
|
|
410
|
+
explanation: "Same recordDiscrete primitive as voice_qa_assistant, just terminating in a persistence call instead of an LLM round-trip. The 'Recording.' speech is optional — give the user audible feedback that the capture started, since the silence-VAD doesn't beep on its own.",
|
|
411
|
+
gotchas: [
|
|
412
|
+
"Pre-warm the mic with the 'Recording.' speech_text — the SCO mic profile takes ~150-300ms to switch from A2DP. Calling recordDiscrete immediately after the wake-phrase match can drop the first half-syllable.",
|
|
413
|
+
"maxDurationSeconds caps at 60 in the SDK; if you need longer notes use audio_chunks + your own buffering.",
|
|
414
|
+
],
|
|
415
|
+
relatedFeatures: ["voice_command", "record_audio", "transcription_incremental"],
|
|
416
|
+
};
|
|
417
|
+
const CONNECTION_PAGE_SETUP = {
|
|
418
|
+
pattern: "connection_page_setup",
|
|
419
|
+
title: "Bootstrap + ExtentosConnectionPage placement",
|
|
420
|
+
description: "The minimum wiring for any Extentos integration: bootstrap the library in Application onCreate / AppDelegate, and drop ExtentosConnectionPage into the customer's existing navigation. From there every other SDK call works.",
|
|
421
|
+
code: {
|
|
422
|
+
kotlin: `// In your Application subclass (e.g. MyApp.kt):
|
|
423
|
+
class MyApp : Application(), ExtentosBootstrap {
|
|
424
|
+
override lateinit var glasses: ExtentosGlasses
|
|
425
|
+
|
|
426
|
+
override fun onCreate() {
|
|
427
|
+
super.onCreate()
|
|
428
|
+
glasses = ExtentosGlasses.create(
|
|
429
|
+
ExtentosConfig(applicationContext = this)
|
|
430
|
+
)
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// In your AndroidManifest.xml:
|
|
435
|
+
// <application android:name=".MyApp" ...>
|
|
436
|
+
//
|
|
437
|
+
// In your NavHost (or wherever your tabs/screens live):
|
|
438
|
+
//
|
|
439
|
+
// val bootstrap = LocalContext.current.applicationContext as ExtentosBootstrap
|
|
440
|
+
//
|
|
441
|
+
// NavHost(navController = nav, startDestination = "home") {
|
|
442
|
+
// composable("home") { HomeScreen() }
|
|
443
|
+
// composable("glasses") { ExtentosConnectionPage(bootstrap.glasses) }
|
|
444
|
+
// }
|
|
445
|
+
//
|
|
446
|
+
// Wire a tab or button to navigate to "glasses".`,
|
|
447
|
+
swift: `// In your App / AppDelegate:
|
|
448
|
+
@main
|
|
449
|
+
struct MyApp: App {
|
|
450
|
+
let glasses: any ExtentosGlasses
|
|
451
|
+
|
|
452
|
+
init() {
|
|
453
|
+
glasses = Extentos.create(config: ExtentosConfig())
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
var body: some Scene {
|
|
457
|
+
WindowGroup {
|
|
458
|
+
ContentView(glasses: glasses)
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// In your top-level view:
|
|
464
|
+
struct ContentView: View {
|
|
465
|
+
let glasses: any ExtentosGlasses
|
|
466
|
+
|
|
467
|
+
var body: some View {
|
|
468
|
+
TabView {
|
|
469
|
+
HomeView()
|
|
470
|
+
.tabItem { Label("Home", systemImage: "house") }
|
|
471
|
+
ExtentosConnectionPage(glasses: glasses)
|
|
472
|
+
.tabItem { Label("Glasses", systemImage: "eyeglasses") }
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
}`,
|
|
476
|
+
},
|
|
477
|
+
explanation: "Two pieces of wiring: (1) construct the ExtentosGlasses instance once at app init — that starts the auto-bind probe, install registration, and telemetry hooks; (2) drop the connection page into navigation. Everything else (audio.X, camera.X, etc.) just uses the glasses reference passed through your DI / view models.",
|
|
478
|
+
gotchas: [
|
|
479
|
+
"Don't construct multiple ExtentosGlasses instances — there's no caching and each one fires its own auto-bind probe + telemetry registration. Hold one at the Application level and pass it down.",
|
|
480
|
+
"Android: applicationContext must be non-null in ExtentosConfig if any handler needs Context (e.g., to use filesDir). The canonical bootstrap class implements ExtentosBootstrap so handlers can pull it via `LocalContext.current.applicationContext as ExtentosBootstrap`.",
|
|
481
|
+
"The connection page renders the pairing flow / sim status / capability indicators automatically — don't try to build your own. Customize via ConnectionPageConfig if you need to hide sections.",
|
|
482
|
+
],
|
|
483
|
+
relatedFeatures: [],
|
|
484
|
+
};
|
|
485
|
+
export const CODE_EXAMPLES = {
|
|
486
|
+
voice_qa_assistant: VOICE_QA_ASSISTANT,
|
|
487
|
+
barge_in_speak: BARGE_IN_SPEAK,
|
|
488
|
+
photo_describe_voice: PHOTO_DESCRIBE_VOICE,
|
|
489
|
+
live_transcription_ui: LIVE_TRANSCRIPTION_UI,
|
|
490
|
+
voice_notes: VOICE_NOTES,
|
|
491
|
+
connection_page_setup: CONNECTION_PAGE_SETUP,
|
|
492
|
+
};
|
|
493
|
+
export const CODE_EXAMPLE_PATTERNS = Object.keys(CODE_EXAMPLES).sort();
|
|
494
|
+
//# sourceMappingURL=codeExamples.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codeExamples.js","sourceRoot":"","sources":["../../../src/tools/data/codeExamples.ts"],"names":[],"mappings":"AAAA,oEAAoE;AACpE,sEAAsE;AACtE,sEAAsE;AACtE,uDAAuD;AACvD,sCAAsC;AACtC,EAAE;AACF,qEAAqE;AACrE,uDAAuD;AACvD,wEAAwE;AACxE,kEAAkE;AAClE,oEAAoE;AACpE,2DAA2D;AAe3D,MAAM,kBAAkB,GAAgB;IACtC,OAAO,EAAE,oBAAoB;IAC7B,KAAK,EAAE,qDAAqD;IAC5D,WAAW,EACT,kSAAkS;IACpS,IAAI,EAAE;QACJ,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmCV;QACE,KAAK,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2CT;KACC;IACD,WAAW,EACT,ojBAAojB;IACtjB,OAAO,EAAE;QACP,yJAAyJ;QACzJ,qKAAqK;QACrK,sIAAsI;QACtI,+HAA+H;QAC/H,yKAAyK;KAC1K;IACD,eAAe,EAAE,CAAC,eAAe,EAAE,2BAA2B,EAAE,cAAc,CAAC;CAChF,CAAC;AAEF,MAAM,cAAc,GAAgB;IAClC,OAAO,EAAE,gBAAgB;IACzB,KAAK,EAAE,gDAAgD;IACvD,WAAW,EACT,uMAAuM;IACzM,IAAI,EAAE;QACJ,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;EAuBV;QACE,KAAK,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgCT;KACC;IACD,WAAW,EACT,+ZAA+Z;IACja,OAAO,EAAE;QACP,8IAA8I;QAC9I,6PAA6P;QAC7P,0IAA0I;KAC3I;IACD,eAAe,EAAE,CAAC,2BAA2B,CAAC;CAC/C,CAAC;AAEF,MAAM,oBAAoB,GAAgB;IACxC,OAAO,EAAE,sBAAsB;IAC/B,KAAK,EAAE,gDAAgD;IACvD,WAAW,EACT,0NAA0N;IAC5N,IAAI,EAAE;QACJ,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2BV;QACE,KAAK,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2CT;KACC;IACD,WAAW,EACT,gWAAgW;IAClW,OAAO,EAAE;QACP,yGAAyG;QACzG,iRAAiR;QACjR,qJAAqJ;KACtJ;IACD,eAAe,EAAE,CAAC,eAAe,EAAE,eAAe,EAAE,2BAA2B,CAAC;CACjF,CAAC;AAEF,MAAM,qBAAqB,GAAgB;IACzC,OAAO,EAAE,uBAAuB;IAChC,KAAK,EAAE,+CAA+C;IACtD,WAAW,EACT,yLAAyL;IAC3L,IAAI,EAAE;QACJ,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;;sBAwBU;QAClB,KAAK,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;yBAyBc;KACtB;IACD,WAAW,EACT,4QAA4Q;IAC9Q,OAAO,EAAE;QACP,gNAAgN;QAChN,2KAA2K;QAC3K,6LAA6L;KAC9L;IACD,eAAe,EAAE,CAAC,2BAA2B,CAAC;CAC/C,CAAC;AAEF,MAAM,WAAW,GAAgB;IAC/B,OAAO,EAAE,aAAa;IACtB,KAAK,EAAE,sCAAsC;IAC7C,WAAW,EACT,sKAAsK;IACxK,IAAI,EAAE;QACJ,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiCV;QACE,KAAK,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsCT;KACC;IACD,WAAW,EACT,yQAAyQ;IAC3Q,OAAO,EAAE;QACP,iNAAiN;QACjN,2GAA2G;KAC5G;IACD,eAAe,EAAE,CAAC,eAAe,EAAE,cAAc,EAAE,2BAA2B,CAAC;CAChF,CAAC;AAEF,MAAM,qBAAqB,GAAgB;IACzC,OAAO,EAAE,uBAAuB;IAChC,KAAK,EAAE,8CAA8C;IACrD,WAAW,EACT,+NAA+N;IACjO,IAAI,EAAE;QACJ,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;;kDAwBsC;QAC9C,KAAK,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4BT;KACC;IACD,WAAW,EACT,+TAA+T;IACjU,OAAO,EAAE;QACP,kMAAkM;QAClM,6QAA6Q;QAC7Q,iMAAiM;KAClM;IACD,eAAe,EAAE,EAAE;CACpB,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAgC;IACxD,kBAAkB,EAAE,kBAAkB;IACtC,cAAc,EAAE,cAAc;IAC9B,oBAAoB,EAAE,oBAAoB;IAC1C,qBAAqB,EAAE,qBAAqB;IAC5C,WAAW,EAAE,WAAW;IACxB,qBAAqB,EAAE,qBAAqB;CAC7C,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC"}
|
|
@@ -6,13 +6,13 @@
|
|
|
6
6
|
// scaffold a project pointing at 1.0.0 (no probe code) while running
|
|
7
7
|
// 0.0.7+ server-side smarts that needed 1.1.x to function.
|
|
8
8
|
//
|
|
9
|
-
// Pre-public-Maven note: 1.1.
|
|
9
|
+
// Pre-public-Maven note: 1.1.28-pair currently lives only on the dev's
|
|
10
10
|
// mavenLocal (./gradlew :glasses-core:publishToMavenLocal). Until a
|
|
11
11
|
// public Maven Central / JitPack publish lands, scaffolded projects
|
|
12
12
|
// require the dev to run mavenLocal-publish first. Document next to
|
|
13
13
|
// the dependency notation in generateConnectionModule.
|
|
14
14
|
export const VERSION_INFO = {
|
|
15
|
-
latestStable: "1.1.
|
|
15
|
+
latestStable: "1.1.28-pair",
|
|
16
16
|
specVersion: "1.0",
|
|
17
17
|
android: {
|
|
18
18
|
minimumSdk: 31,
|
|
@@ -26,7 +26,7 @@ export const VERSION_INFO = {
|
|
|
26
26
|
kotlinVersion: "2.0.0",
|
|
27
27
|
// Toolchain minimums (F-R4-8 from DOGFOOD_R4). The library's transitive
|
|
28
28
|
// Compose deps determine the floor:
|
|
29
|
-
// - com.extentos:glasses-ui:1.1.
|
|
29
|
+
// - com.extentos:glasses-ui:1.1.28-pair brings in androidx.compose.* 1.9.0
|
|
30
30
|
// (via compose-bom 2024.10.00 and direct deps)
|
|
31
31
|
// - Compose 1.9.0 AAR-metadata declares "requires AGP >= 8.6.0"
|
|
32
32
|
// - AGP 8.6+ requires Gradle 8.7+ (Android Gradle Plugin compatibility table)
|
|
@@ -50,13 +50,13 @@ export const VERSION_INFO = {
|
|
|
50
50
|
},
|
|
51
51
|
artifacts: {
|
|
52
52
|
android: {
|
|
53
|
-
core: "com.extentos:glasses:1.1.
|
|
54
|
-
ui: "com.extentos:glasses-ui:1.1.
|
|
55
|
-
debug: "com.extentos:glasses-debug:1.1.
|
|
53
|
+
core: "com.extentos:glasses:1.1.28-pair",
|
|
54
|
+
ui: "com.extentos:glasses-ui:1.1.28-pair",
|
|
55
|
+
debug: "com.extentos:glasses-debug:1.1.28-pair",
|
|
56
56
|
},
|
|
57
57
|
ios: {
|
|
58
58
|
package: "https://github.com/extentos/swift-glasses",
|
|
59
|
-
version: "1.1.
|
|
59
|
+
version: "1.1.28-pair",
|
|
60
60
|
products: ["GlassesCore", "GlassesUI", "GlassesDebug", "GlassesLifecycle", "GlassesTesting"],
|
|
61
61
|
},
|
|
62
62
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"definitions.d.ts","sourceRoot":"","sources":["../../src/tools/definitions.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"definitions.d.ts","sourceRoot":"","sources":["../../src/tools/definitions.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IAMpB,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ,CAAC;QACf,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACpC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,oBAAoB,EAAE,KAAK,CAAC;KAC7B,CAAC;CACH;AAMD,eAAO,MAAM,eAAe,EAAE,OAAO,EAiXpC,CAAC;AAEF,eAAO,MAAM,SAAS,QAAyB,CAAC;AAShD,wBAAgB,iCAAiC,CAC/C,IAAI,GAAE,SAAS,OAAO,EAAoB,GACzC,IAAI,CAiBN"}
|