@glassly/miniapp-cli 0.1.18-dev.0 → 0.1.19-dev.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@glassly/miniapp-cli",
3
- "version": "0.1.18-dev.0",
3
+ "version": "0.1.19-dev.0",
4
4
  "description": "Build tooling for Glassly miniapps: dev server, packer, manifest validation, and JSON Schema. The `glassly` command that drives it ships in @glassly/cli.",
5
5
  "type": "module",
6
6
  "publishConfig": {
@@ -4,7 +4,7 @@
4
4
  import {AllowedHardwareType, AllowedPermission} from "./manifest.js"
5
5
 
6
6
  const PERMISSION_HINTS: Record<AllowedPermission, string> = {
7
- MICROPHONE: "Required for transcription, audio chunks, voice activity detection",
7
+ MICROPHONE: "Required for transcription, the wake word, audio chunks, voice activity detection",
8
8
  CAMERA: "Required for taking photos via session.camera.takePhoto()",
9
9
  CALENDAR: "Required to receive calendar event subscriptions",
10
10
  LOCATION: "Required for foreground location updates",
@@ -254,6 +254,60 @@ describe("speech", () => {
254
254
  ])
255
255
  })
256
256
 
257
+ test("the wake word reaches a miniapp that listens for it", async () => {
258
+ const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
259
+ const heard: string[] = []
260
+ session.wakeWord.on((data) => heard.push(data.phrase))
261
+ await settle()
262
+ host.wakeWord()
263
+ await settle()
264
+ expect(heard).toEqual(["hey glassly"])
265
+ })
266
+
267
+ test("speech that opens with the wake phrase fires the wake word, then transcribes what follows", async () => {
268
+ const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
269
+ const order: string[] = []
270
+ session.wakeWord.on(() => {
271
+ order.push("wake")
272
+ session.transcription.on((data) => order.push(data.text))
273
+ })
274
+ await settle()
275
+ await host.say("Hey Glassly, what time is it?")
276
+ await settle()
277
+ expect(order).toEqual(["wake", "what time is it?"])
278
+ })
279
+
280
+ test("a miniapp already transcribing hears the whole line after the wake word", async () => {
281
+ const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
282
+ const order: string[] = []
283
+ session.wakeWord.on(() => order.push("wake"))
284
+ session.transcription.on((data) => order.push(data.text))
285
+ await settle()
286
+ await host.say("Hey Glassly, what time is it?")
287
+ await settle()
288
+ expect(order).toEqual(["wake", "Hey Glassly, what time is it?"])
289
+ })
290
+
291
+ test("speech without the wake phrase fires no wake word", async () => {
292
+ const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
293
+ const wakes: string[] = []
294
+ const heard: string[] = []
295
+ session.wakeWord.on((data) => wakes.push(data.phrase))
296
+ session.transcription.on((data) => heard.push(data.text))
297
+ await settle()
298
+ await host.say("what a nice day")
299
+ await settle()
300
+ expect(wakes).toEqual([])
301
+ expect(heard).toEqual(["what a nice day"])
302
+ })
303
+
304
+ test("listening for the wake word without the manifest permission fails", async () => {
305
+ const {session, errors} = await connect()
306
+ session.wakeWord.on(() => {})
307
+ await settle()
308
+ expect((errors[0] as {code?: string}).code).toBe("PERMISSION_NOT_DECLARED")
309
+ })
310
+
257
311
  test("a language listener hears only its language", async () => {
258
312
  const {session, host} = await connect({permissions: [{type: "MICROPHONE"}]})
259
313
  const spanish: string[] = []
package/src/sim/host.ts CHANGED
@@ -129,9 +129,12 @@ const PERMISSION_TYPE_TO_CANONICAL: Record<string, string> = {
129
129
  HOST_WIDGETS: "hostWidgets",
130
130
  }
131
131
 
132
+ const WAKE_PHRASE = "hey glassly"
133
+ const OPENS_WITH_WAKE_PHRASE = /^\s*hey[\s,]+glassly\b[\s,.!?]*/i
134
+
132
135
  function permissionForStream(stream: string): string | null {
133
136
  if (stream === MiniappStreamType.MIRROR_UPDATE) return "DISPLAY_MIRROR"
134
- if (stream === "audio_chunk" || stream === "vad") return "MICROPHONE"
137
+ if (stream === "audio_chunk" || stream === "vad" || stream === MiniappStreamType.WAKE_WORD) return "MICROPHONE"
135
138
  if (stream.startsWith("transcription") || stream.startsWith("translation")) return "MICROPHONE"
136
139
  if (stream === "location_update" || stream === "heading_update") return "LOCATION"
137
140
  if (stream === "phone_notification" || stream === "phone_notification_dismissed") return "READ_NOTIFICATIONS"
@@ -853,6 +856,7 @@ export class SimHost {
853
856
  this.send(
854
857
  {
855
858
  type: MiniappResponseType.CONNECT_ACK,
859
+ wakeWordAvailable: true,
856
860
  userId: "sim-user",
857
861
  packageName: this.manifest.packageName,
858
862
  capabilities: this.capabilities(),
@@ -1283,6 +1287,27 @@ export class SimHost {
1283
1287
  this.inject([MiniappStreamType.BUTTON_PRESS], {buttonId, pressType, timestamp: Date.now()})
1284
1288
  }
1285
1289
 
1290
+ wakeWord(): void {
1291
+ this.inject([MiniappStreamType.WAKE_WORD], {phrase: WAKE_PHRASE, confidence: 1, timestamp: Date.now()})
1292
+ }
1293
+
1294
+ async say(text: string, options: {isFinal?: boolean; language?: string; speakerId?: string} = {}): Promise<void> {
1295
+ const wake = OPENS_WITH_WAKE_PHRASE.exec(text)
1296
+ if (!wake || !this.subscriptions.has(MiniappStreamType.WAKE_WORD)) return this.transcription(text, options)
1297
+ const transcribing = this.transcribing()
1298
+ this.wakeWord()
1299
+ if (transcribing) return this.transcription(text, options)
1300
+ for (let waited = 0; waited < 1_000 && !this.transcribing(); waited += 10) {
1301
+ await new Promise((resolve) => setTimeout(resolve, 10))
1302
+ }
1303
+ const rest = text.slice(wake[0].length)
1304
+ if (rest.trim()) this.transcription(rest, options)
1305
+ }
1306
+
1307
+ private transcribing(): boolean {
1308
+ return [...this.subscriptions.keys()].some((stream) => stream.startsWith("transcription:"))
1309
+ }
1310
+
1286
1311
  /**
1287
1312
  * One transcription result, as the phone sends it: interims keep the open
1288
1313
  * utterance's id and a final closes it.
package/src/sim/server.ts CHANGED
@@ -366,8 +366,11 @@ export function startSimServer(options: SimServerOptions): SimServer {
366
366
  case "button":
367
367
  host.button(body.pressType === "long" ? "long" : "short", text("buttonId"))
368
368
  break
369
+ case "wake":
370
+ host.wakeWord()
371
+ break
369
372
  case "say":
370
- host.transcription(text("text") ?? "", {
373
+ await host.say(text("text") ?? "", {
371
374
  isFinal: body.isFinal !== false,
372
375
  language: text("language"),
373
376
  speakerId: text("speakerId"),