@glassly/miniapp-cli 0.1.18-dev.0 → 0.1.19-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/permission-hints.ts +1 -1
- package/src/sim/host.test.ts +54 -0
- package/src/sim/host.ts +26 -1
- package/src/sim/server.ts +4 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@glassly/miniapp-cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.19-dev.0",
|
|
4
4
|
"description": "Build tooling for Glassly miniapps: dev server, packer, manifest validation, and JSON Schema. The `glassly` command that drives it ships in @glassly/cli.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"publishConfig": {
|
package/src/permission-hints.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import {AllowedHardwareType, AllowedPermission} from "./manifest.js"
|
|
5
5
|
|
|
6
6
|
const PERMISSION_HINTS: Record<AllowedPermission, string> = {
|
|
7
|
-
MICROPHONE: "Required for transcription, audio chunks, voice activity detection",
|
|
7
|
+
MICROPHONE: "Required for transcription, the wake word, audio chunks, voice activity detection",
|
|
8
8
|
CAMERA: "Required for taking photos via session.camera.takePhoto()",
|
|
9
9
|
CALENDAR: "Required to receive calendar event subscriptions",
|
|
10
10
|
LOCATION: "Required for foreground location updates",
|
package/src/sim/host.test.ts
CHANGED
|
@@ -254,6 +254,60 @@ describe("speech", () => {
|
|
|
254
254
|
])
|
|
255
255
|
})
|
|
256
256
|
|
|
257
|
+
test("the wake word reaches a miniapp that listens for it", async () => {
|
|
258
|
+
const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
|
|
259
|
+
const heard: string[] = []
|
|
260
|
+
session.wakeWord.on((data) => heard.push(data.phrase))
|
|
261
|
+
await settle()
|
|
262
|
+
host.wakeWord()
|
|
263
|
+
await settle()
|
|
264
|
+
expect(heard).toEqual(["hey glassly"])
|
|
265
|
+
})
|
|
266
|
+
|
|
267
|
+
test("speech that opens with the wake phrase fires the wake word, then transcribes what follows", async () => {
|
|
268
|
+
const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
|
|
269
|
+
const order: string[] = []
|
|
270
|
+
session.wakeWord.on(() => {
|
|
271
|
+
order.push("wake")
|
|
272
|
+
session.transcription.on((data) => order.push(data.text))
|
|
273
|
+
})
|
|
274
|
+
await settle()
|
|
275
|
+
await host.say("Hey Glassly, what time is it?")
|
|
276
|
+
await settle()
|
|
277
|
+
expect(order).toEqual(["wake", "what time is it?"])
|
|
278
|
+
})
|
|
279
|
+
|
|
280
|
+
test("a miniapp already transcribing hears the whole line after the wake word", async () => {
|
|
281
|
+
const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
|
|
282
|
+
const order: string[] = []
|
|
283
|
+
session.wakeWord.on(() => order.push("wake"))
|
|
284
|
+
session.transcription.on((data) => order.push(data.text))
|
|
285
|
+
await settle()
|
|
286
|
+
await host.say("Hey Glassly, what time is it?")
|
|
287
|
+
await settle()
|
|
288
|
+
expect(order).toEqual(["wake", "Hey Glassly, what time is it?"])
|
|
289
|
+
})
|
|
290
|
+
|
|
291
|
+
test("speech without the wake phrase fires no wake word", async () => {
|
|
292
|
+
const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
|
|
293
|
+
const wakes: string[] = []
|
|
294
|
+
const heard: string[] = []
|
|
295
|
+
session.wakeWord.on((data) => wakes.push(data.phrase))
|
|
296
|
+
session.transcription.on((data) => heard.push(data.text))
|
|
297
|
+
await settle()
|
|
298
|
+
await host.say("what a nice day")
|
|
299
|
+
await settle()
|
|
300
|
+
expect(wakes).toEqual([])
|
|
301
|
+
expect(heard).toEqual(["what a nice day"])
|
|
302
|
+
})
|
|
303
|
+
|
|
304
|
+
test("listening for the wake word without the manifest permission fails", async () => {
|
|
305
|
+
const {session, errors} = await connect()
|
|
306
|
+
session.wakeWord.on(() => {})
|
|
307
|
+
await settle()
|
|
308
|
+
expect((errors[0] as {code?: string}).code).toBe("PERMISSION_NOT_DECLARED")
|
|
309
|
+
})
|
|
310
|
+
|
|
257
311
|
test("a language listener hears only its language", async () => {
|
|
258
312
|
const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
|
|
259
313
|
const spanish: string[] = []
|
package/src/sim/host.ts
CHANGED
|
@@ -129,9 +129,12 @@ const PERMISSION_TYPE_TO_CANONICAL: Record<string, string> = {
|
|
|
129
129
|
HOST_WIDGETS: "hostWidgets",
|
|
130
130
|
}
|
|
131
131
|
|
|
132
|
+
const WAKE_PHRASE = "hey glassly"
|
|
133
|
+
const OPENS_WITH_WAKE_PHRASE = /^\s*hey[\s,]+glassly\b[\s,.!?]*/i
|
|
134
|
+
|
|
132
135
|
function permissionForStream(stream: string): string | null {
|
|
133
136
|
if (stream === MiniappStreamType.MIRROR_UPDATE) return "DISPLAY_MIRROR"
|
|
134
|
-
if (stream === "audio_chunk" || stream === "vad") return "MICROPHONE"
|
|
137
|
+
if (stream === "audio_chunk" || stream === "vad" || stream === MiniappStreamType.WAKE_WORD) return "MICROPHONE"
|
|
135
138
|
if (stream.startsWith("transcription") || stream.startsWith("translation")) return "MICROPHONE"
|
|
136
139
|
if (stream === "location_update" || stream === "heading_update") return "LOCATION"
|
|
137
140
|
if (stream === "phone_notification" || stream === "phone_notification_dismissed") return "READ_NOTIFICATIONS"
|
|
@@ -853,6 +856,7 @@ export class SimHost {
|
|
|
853
856
|
this.send(
|
|
854
857
|
{
|
|
855
858
|
type: MiniappResponseType.CONNECT_ACK,
|
|
859
|
+
wakeWordAvailable: true,
|
|
856
860
|
userId: "sim-user",
|
|
857
861
|
packageName: this.manifest.packageName,
|
|
858
862
|
capabilities: this.capabilities(),
|
|
@@ -1283,6 +1287,27 @@ export class SimHost {
|
|
|
1283
1287
|
this.inject([MiniappStreamType.BUTTON_PRESS], {buttonId, pressType, timestamp: Date.now()})
|
|
1284
1288
|
}
|
|
1285
1289
|
|
|
1290
|
+
wakeWord(): void {
|
|
1291
|
+
this.inject([MiniappStreamType.WAKE_WORD], {phrase: WAKE_PHRASE, confidence: 1, timestamp: Date.now()})
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
async say(text: string, options: {isFinal?: boolean; language?: string; speakerId?: string} = {}): Promise<void> {
|
|
1295
|
+
const wake = OPENS_WITH_WAKE_PHRASE.exec(text)
|
|
1296
|
+
if (!wake || !this.subscriptions.has(MiniappStreamType.WAKE_WORD)) return this.transcription(text, options)
|
|
1297
|
+
const transcribing = this.transcribing()
|
|
1298
|
+
this.wakeWord()
|
|
1299
|
+
if (transcribing) return this.transcription(text, options)
|
|
1300
|
+
for (let waited = 0; waited < 1_000 && !this.transcribing(); waited += 10) {
|
|
1301
|
+
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
1302
|
+
}
|
|
1303
|
+
const rest = text.slice(wake[0].length)
|
|
1304
|
+
if (rest.trim()) this.transcription(rest, options)
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
private transcribing(): boolean {
|
|
1308
|
+
return [...this.subscriptions.keys()].some((stream) => stream.startsWith("transcription:"))
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1286
1311
|
/**
|
|
1287
1312
|
* One transcription result, as the phone sends it: interims keep the open
|
|
1288
1313
|
* utterance's id and a final closes it.
|
package/src/sim/server.ts
CHANGED
|
@@ -366,8 +366,11 @@ export function startSimServer(options: SimServerOptions): SimServer {
|
|
|
366
366
|
case "button":
|
|
367
367
|
host.button(body.pressType === "long" ? "long" : "short", text("buttonId"))
|
|
368
368
|
break
|
|
369
|
+
case "wake":
|
|
370
|
+
host.wakeWord()
|
|
371
|
+
break
|
|
369
372
|
case "say":
|
|
370
|
-
host.
|
|
373
|
+
await host.say(text("text") ?? "", {
|
|
371
374
|
isFinal: body.isFinal !== false,
|
|
372
375
|
language: text("language"),
|
|
373
376
|
speakerId: text("speakerId"),
|