@memberjunction/ai-realtime-client 0.0.1 → 5.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +148 -28
  2. package/dist/audio/audioMeter.d.ts +101 -0
  3. package/dist/audio/audioMeter.d.ts.map +1 -0
  4. package/dist/audio/audioMeter.js +193 -0
  5. package/dist/audio/audioMeter.js.map +1 -0
  6. package/dist/audio/micCapture.d.ts +26 -0
  7. package/dist/audio/micCapture.d.ts.map +1 -0
  8. package/dist/audio/micCapture.js +69 -0
  9. package/dist/audio/micCapture.js.map +1 -0
  10. package/dist/audio/pcmPlayback.d.ts +73 -0
  11. package/dist/audio/pcmPlayback.d.ts.map +1 -0
  12. package/dist/audio/pcmPlayback.js +78 -0
  13. package/dist/audio/pcmPlayback.js.map +1 -0
  14. package/dist/audio/pcmUtils.d.ts +17 -0
  15. package/dist/audio/pcmUtils.d.ts.map +1 -0
  16. package/dist/audio/pcmUtils.js +46 -0
  17. package/dist/audio/pcmUtils.js.map +1 -0
  18. package/dist/drivers/assemblyAIRealtimeClient.d.ts +384 -0
  19. package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -0
  20. package/dist/drivers/assemblyAIRealtimeClient.js +732 -0
  21. package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -0
  22. package/dist/drivers/elevenLabsRealtimeClient.d.ts +362 -0
  23. package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -0
  24. package/dist/drivers/elevenLabsRealtimeClient.js +686 -0
  25. package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -0
  26. package/dist/drivers/geminiRealtimeClient.d.ts +406 -0
  27. package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -0
  28. package/dist/drivers/geminiRealtimeClient.js +675 -0
  29. package/dist/drivers/geminiRealtimeClient.js.map +1 -0
  30. package/dist/drivers/openAIRealtimeClient.d.ts +381 -0
  31. package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -0
  32. package/dist/drivers/openAIRealtimeClient.js +602 -0
  33. package/dist/drivers/openAIRealtimeClient.js.map +1 -0
  34. package/dist/drivers/xaiRealtimeClient.d.ts +430 -0
  35. package/dist/drivers/xaiRealtimeClient.d.ts.map +1 -0
  36. package/dist/drivers/xaiRealtimeClient.js +676 -0
  37. package/dist/drivers/xaiRealtimeClient.js.map +1 -0
  38. package/dist/generic/baseRealtimeClient.d.ts +401 -0
  39. package/dist/generic/baseRealtimeClient.d.ts.map +1 -0
  40. package/dist/generic/baseRealtimeClient.js +226 -0
  41. package/dist/generic/baseRealtimeClient.js.map +1 -0
  42. package/dist/index.d.ts +11 -0
  43. package/dist/index.d.ts.map +1 -0
  44. package/dist/index.js +11 -0
  45. package/dist/index.js.map +1 -0
  46. package/package.json +28 -7
@@ -0,0 +1 @@
1
+ {"version":3,"file":"elevenLabsRealtimeClient.js","sourceRoot":"","sources":["../../src/drivers/elevenLabsRealtimeClient.ts"],"names":[],"mappings":";;;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAuB,MAAM,+BAA+B,CAAC;AACxF,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAwB,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAE1E,kFAAkF;AAElF;;;GAGG;AACH,MAAM,8BAA8B,GAAG,KAAK,CAAC;AA2D7C,kFAAkF;AAElF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEI,IAAM,wBAAwB,GAA9B,MAAM,wBAAyB,SAAQ,kBAAkB;IAAzD;;QACH,8EAA8E;QACtE,WAAM,GAAmC,IAAI,CAAC;QAC9C,cAAS,GAAuB,IAAI,CAAC;QACrC,eAAU,GAA0B,IAAI,CAAC;QACzC,aAAQ,GAAgC,IAAI,CAAC;QAErD,8EAA8E;QAC9E,wFAAwF;QAChF,mBAAc,GAAG,KAAK,CAAC;QAC/B,6FAA6F;QACrF,uBAAkB,GAA2B,QAAQ,CAAC;QAC9D,2FAA2F;QACnF,gBAAW,GAAsB,EAAE,CAAC;QAC5C;;;;;;WAMG;QACK,yBAAoB,GAAyC,IAAI,CAAC;QAC1E;;;;WAIG;QACK,uBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/C,qFAAqF;QAC7E,qBAAgB,GAAG,KAAK,CAAC;QACjC;;;;WAIG;QACK,iBAAY,GAAwB,QAAQ,CAAC;QAkRrD,8EAA8E;QAE9E,iFAAiF;QACzE,yBAAoB,GAAoD,IAAI,CAAC;IA+VzF,CAAC;;IAlnBG,8EAA8E;IAE9E;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,MAAmC,EAAE,SAAsB;QAC5E,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,UAAU,GAAwB,IAAI,CAAC;QAC3C,IAAI,QAAQ,GAAoC,IAAI,CAAC;QACrD,IAAI,eAAe,GAAoD,IAAI,CAAC;QAC5E,IAAI,YAAY,GAAoC,IAAI,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACjD,UAAU,GAAG,OAAO,CAAC;YACrB,QAAQ,GAAG,MAAM,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,OAAO,CAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACpE,eAAe,GAAG,OAAO,CAAC;YAC1B,YAAY,GAAG,MAAM,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,sFAAsF;QACtF,wFAAwF;QACxF,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,oBAAoB,GAAG,eAAe,CAAC;QAC5C,MAAM,WAAW,GAAG,CAAC,KAAY,EAAE,EAAE;YACjC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC;YAClB,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC;QACrC,MAAM,CAAC,SAAS,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,CAAC,OAAO,GAAG,CAAC,OAAO,EAAE,EAAE;YACzB,WAAW,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAChC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC,CAAC;QACF,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;YAClB,WAAW,CAAC,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7B,CAAC,CAAC;QAEF,MAAM,MAAM,CAAC;QACb,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC3B,kFAAkF;QAClF,sFAAsF;QACtF,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,qCAAqC,EAAE,4BAA4B,EAAE,SAAS,EAAE,CAAC,CAAC;QAEzG,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC;QACrC,MAAM,OAAO,GAAG,aAAa,CAAC,sCAAsC,CAAC;QACrE,MAAM,UAAU,GAAG,0BAAwB,CAAC,YAAY,CAAC,OAAO,EAAE,yBAAyB,EAAE,QAAQ,CAAC,CAAC;QACvG,MAAM,SAAS,GAAG,0BAAwB,CAAC,YAAY,CAAC,OAAO,EAAE,uBAAuB,EAAE,OAAO,CAAC,CAAC;QACnG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,WAAW,EAAE,EAAE,CAChF,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CACjC,CAAC;QACF,8EAA8E;QAC9E,kFAAkF;QAClF,gEAAgE;QAChE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,IAAI,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,UAAU;QACnB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACL,qBAAqB;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,6EAA6E;IAE7E;;;;;;;;;;OAUG;IACI,QAAQ,CAAC,IAAY;QACxB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;IACxE,CAAC;IAED;;;;;;;;;OASG;IACI,oBAAoB;QACvB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,OAAO,CAAC,qCAAqC;QACjD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,eAAe,CAAC,IAAY;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;OAMG;IACI,mBAAmB,CAAC,YAAoB;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;OAOG;IACI,cAAc,CAAC,MAAc,EAAE,UAAkB;QACpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,IAAI,CACR,kEAAkE,MAAM,0CAA0C,CACrH,CAAC;YACF,OAAO;QACX,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED;;;;OAIG;IACI,QAAQ,CAAC,KAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;QACtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACH,IAAW,cAAc;QACrB,OAAO,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,KAAK,CAAC;IAC7C,CAAC;IAED,6EAA6E;IAE7E;;;;;OAKG;IACO,YAAY,CAAC,SAAiB;QACpC,MAAM,EAAE,GAAI,UAAkF,CAAC,SAAS,CAAC;QACzG,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;QACnG,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7B,MAAM,IAAI,GAA4B;YAClC,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;YAC7B,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE;SAC1B,CAAC;QACF,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,EAAE,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,yCAAyC,CAAC,CAAC;QAC7E,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAC5B,SAAsB,EACtB,UAAkB,EAClB,UAAyC;QAEzC,OAAO,mBAAmB,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAClE,CAAC;IAED;;;OAGG;IACO,cAAc,CAAC,UAAkB;QACvC,OAAO,IAAI,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC;IAOD;;;;;OAKG;IACK,cAAc,CAAC,MAAmC;QACtD,MAAM,aAAa,GAAe,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;QAC7D,MAAM,GAAG,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;QACvC,OAAO,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,GAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;IACrG,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,YAAY,CAAC,MAA0B,EAAE,SAA6B;QAChF,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC/C,IAAI,KAAK,EAAE,CAAC;YACR,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,MAAM,EAAE,CAAC;YACT,OAAO,CAAC,IAAI,CACR,yCAAyC,SAAS,kBAAkB,MAAM,yBAAyB;gBACnG,kCAAkC,8BAA8B,sCAAsC,CACzG,CAAC;QACN,CAAC;QACD,OAAO,8BAA8B,CAAC;IAC1C,CAAC;IAED,iFAAiF;IACzE,YAAY,CAAC,WAAmB;QACpC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC,CAAC;QACtD,CAAC;IACL,CAAC;IAED,oFAAoF;IAC5E,iBAAiB,CAAC,OAAe;QACrC,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAClE,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,wCAAwC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5F,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACK,iBAAiB;QACrB,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAC3F,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,6CAA6C,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACxF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,8EAA8E;IAE9E,kEAAkE;IAC1D,mBAAmB,CAAC,IAAY;QACpC,IAAI,KAA4B,CAAC;QACjC,IAAI,CAAC;YACD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA0B,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO;QACX,CAAC;QACD,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,yEAAyE;IACjE,iBAAiB,CAAC,KAA4B;QAClD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,kCAAkC;gBACnC,IAAI,CAAC,oBAAoB,EAAE,CAAC,KAAK,CAAC,CAAC;gBACnC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;gBACjC,MAAM;YACV,KAAK,OAAO;gBACR,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;gBACnD,MAAM;YACV,KAAK,iBAAiB;gBAClB,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,wBAAwB,EAAE,eAAe,CAAC,CAAC;gBAC3E,MAAM;YACV,KAAK,gBAAgB;gBACjB,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;gBACrE,MAAM;YACV,KAAK,2BAA2B;gBAC5B,IAAI,CAAC,6BAA6B,CAAC,KAAK,CAAC,+BAA+B,EAAE,wBAAwB,CAAC,CAAC;gBACpG,MAAM;YACV,KAAK,yBAAyB;gBAC1B,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC9B,MAAM;YACV,KAAK,kBAAkB;gBACnB,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBAClD,MAAM;YACV,KAAK,cAAc;gBACf,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC1B,MAAM;YACV,KAAK,MAAM;gBACP,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC5E,MAAM;YACV,KAAK,WAAW;gBACZ,MAAM,CAAC,6DAA6D;YACxE,KAAK,qBAAqB;gBACtB,OAAO,CAAC,IAAI,CAAC,qDAAqD,EAAE,KAAK,CAAC,CAAC;gBAC3E,MAAM;YACV;gBACI,MAAM,CAAC,2CAA2C;QAC1D,CAAC;IACL,CAAC;IAED,6FAA6F;IACrF,WAAW,CAAC,WAA+B;QAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,sFAAsF;IAC9E,oBAAoB,CAAC,IAAwB;QACjD,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,mBAAmB,CAAC,IAAwB;QAChD,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACzG,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACK,6BAA6B,CAAC,aAAiC;QACnE,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3H,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,sBAAsB;QAC1B,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACK,oBAAoB,CAAC,IAAwF;QACjH,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QACpC,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IAChI,CAAC;IAED;;;;;;;OAOG;IACK,kBAAkB;QACtB,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACK,qBAAqB;QACzB,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,8EAA8E;IAE9E;;;;OAIG;IACK,YAAY,CAAC,IAAgB;QACjC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QACD,IAAI,EAAE,CAAC;IACX,CAAC;IAED;;;OAGG;IACK,gBAAgB;QACpB,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YACtC,IAAI,EAAE,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,IAAY,EAAE,IAA4B,EAAE,YAAqB;QACrF,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI,YAAY,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,mBAAmB,CAAC,MAAc,EAAE,UAAkB;QAC1D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC;YACX,IAAI,EAAE,oBAAoB;YAC1B,YAAY,EAAE,MAAM;YACpB,MAAM,EAAE,0BAAwB,CAAC,eAAe,CAAC,UAAU,CAAC;YAC5D,QAAQ,EAAE,KAAK;SAClB,CAAC,CAAC;QACH,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC1B,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC9B,CAAC;IAED,iFAAiF;aACzD,sBAAiB,GAAG,IAAI,AAAP,CAAQ;IAEjD,8EAA8E;IACtE,kBAAkB;QACtB,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,UAAU,CAAC,GAAG,EAAE;YACxC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;YACjC,6EAA6E;YAC7E,8EAA8E;YAC9E,sCAAsC;YACtC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;YAC5B,IAAI,CAAC,eAAe,CAChB,mFAAmF;gBACnF,kFAAkF,EAClF,QAAQ,EACR,KAAK,CACR,CAAC;QACN,CAAC,EAAE,0BAAwB,CAAC,iBAAiB,CAAC,CAAC;IACnD,CAAC;IAED,mFAAmF;IAC3E,qBAAqB;QACzB,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC5B,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACxC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACrC,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,eAAe,CAAC,MAAc;QACxC,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAc,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,MAAM,CAAC;QAClB,CAAC;IACL,CAAC;IAED,8EAA8E;IAE9E,kFAAkF;IAC1E,SAAS,CAAC,KAAiB;QAC/B,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,0EAA0E;IAClE,kBAAkB;QACtB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;IACrC,CAAC;IAED,4EAA4E;IACpE,QAAQ,CAAC,KAA0B;QACvC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;;AAtpBQ,wBAAwB;IADpC,aAAa,CAAC,kBAAkB,EAAE,YAAY,CAAC;GACnC,wBAAwB,CAupBpC;;AAED;;;;GAIG;AACH,MAAM,UAAU,4BAA4B;IACxC,oEAAoE;AACxE,CAAC"}
@@ -0,0 +1,406 @@
1
+ import { ClientRealtimeSessionConfig } from '@memberjunction/ai';
2
+ import { type Blob as GeminiBlob, type Content, type FunctionResponse, type LiveConnectConfig, type LiveServerMessage } from '@google/genai';
3
+ import { BaseRealtimeClient } from '../generic/baseRealtimeClient.js';
4
+ import { IRealtimePcmPlayback, RealtimePcmPlayback } from '../audio/pcmPlayback.js';
5
+ import { IPcmMicCapture } from '../audio/micCapture.js';
6
+ /**
7
+ * The minimal subset of `@google/genai`'s `Session` this client depends on. Declaring the seam
8
+ * as an interface (rather than the concrete SDK `Session`) lets unit tests inject a fully
9
+ * in-memory fake that captures outbound sends and drives the registered message callback with
10
+ * Gemini-shaped frames — no websocket, no network. (Mirrors the server driver's
11
+ * `GeminiLiveSession` seam.)
12
+ */
13
+ export interface GeminiLiveClientSession {
14
+ /**
15
+ * Streams realtime user input to the model — mic audio frames AND mid-session text.
16
+ * Text via realtime input is the Live API's "respond now" path for in-conversation
17
+ * messages: native-audio models treat {@link sendClientContent} as history seeding
18
+ * only and will NOT generate from it mid-call, but realtime text triggers immediately.
19
+ */
20
+ sendRealtimeInput(params: {
21
+ audio?: GeminiBlob;
22
+ text?: string;
23
+ }): void;
24
+ /** Appends client content WITHOUT triggering generation (context notes, history seeding). */
25
+ sendClientContent(params: {
26
+ turns?: Content[];
27
+ turnComplete?: boolean;
28
+ }): void;
29
+ /** Replies to a server tool call with one or more function responses. */
30
+ sendToolResponse(params: {
31
+ functionResponses: FunctionResponse[];
32
+ }): void;
33
+ /** Terminates the underlying connection. */
34
+ close(): void;
35
+ }
36
+ /**
37
+ * Arguments handed to {@link GeminiRealtimeClient.connectLiveSession}. Bundles the resolved
38
+ * model + connect config (from the server-minted `SessionConfig`), the ephemeral token, and the
39
+ * lifecycle callbacks so the seam owns the entire `live.connect` call and tests can substitute
40
+ * it wholesale.
41
+ */
42
+ export interface GeminiClientConnectArgs {
43
+ /** The Gemini Live model id to open the session against. */
44
+ Model: string;
45
+ /** The server-built connect config (system instruction, tools, modalities, transcription). */
46
+ Config: LiveConnectConfig;
47
+ /** The server-minted ephemeral auth token (used as the API key on a `v1alpha` client). */
48
+ EphemeralToken: string;
49
+ /** Invoked for every {@link LiveServerMessage} the server emits over the session. */
50
+ OnMessage: (message: LiveServerMessage) => void;
51
+ /** Invoked on a websocket-level error (fatal). */
52
+ OnError: (event: ErrorEvent) => void;
53
+ /** Invoked when the websocket closes. */
54
+ OnClose: (event: CloseEvent) => void;
55
+ }
56
+ /**
57
+ * Handle returned by the mic-capture seam: the only operation the driver needs is teardown.
58
+ * Production wraps an `AudioContext` + `AudioWorkletNode` pipeline; tests return a no-op fake.
59
+ *
60
+ * Back-compat alias for the shared {@link IPcmMicCapture} (the capture pipeline now lives in
61
+ * `src/audio/micCapture.ts`, shared with the ElevenLabs driver).
62
+ */
63
+ export type IGeminiMicCapture = IPcmMicCapture;
64
+ /**
65
+ * The playback seam: schedules raw PCM16 chunks for gapless playout and reports whether audio
66
+ * is AUDIBLY playing. Production is {@link GeminiPcmPlayback} (Web Audio, playhead-clock
67
+ * scheduling); tests inject a fake with a controllable `IsPlaying`.
68
+ *
69
+ * Back-compat alias for the shared {@link IRealtimePcmPlayback} (the playout engine now lives
70
+ * in `src/audio/pcmPlayback.ts`, shared with the ElevenLabs driver).
71
+ */
72
+ export type IGeminiAudioPlayback = IRealtimePcmPlayback;
73
+ /**
74
+ * Web Audio playout scheduler for Gemini's 24 kHz PCM16 model audio.
75
+ *
76
+ * A thin specialization of the shared {@link RealtimePcmPlayback} (playhead-clock scheduling,
77
+ * gapless playout, honest `IsPlaying`) fixed at Gemini Live's 24 kHz output rate. Kept as a
78
+ * named export for back-compat with existing consumers.
79
+ */
80
+ export declare class GeminiPcmPlayback extends RealtimePcmPlayback {
81
+ constructor();
82
+ }
83
+ /**
84
+ * Google Gemini implementation of {@link BaseRealtimeClient}: a **browser-direct** Gemini Live
85
+ * websocket session authenticated with the server-minted ephemeral auth token (`v1alpha`
86
+ * `auth_tokens` mechanism — the token is passed as the SDK `apiKey`).
87
+ *
88
+ * Registered with the ClassFactory under the key `'gemini'` — the `Provider` string the
89
+ * server's `GeminiRealtime` driver stamps on its `ClientRealtimeSessionConfig` — so hosts
90
+ * resolve it without referencing this class directly.
91
+ *
92
+ * Owns ALL Gemini wire concerns (the behavioral twin of {@link OpenAIRealtimeClient}, adapted
93
+ * to Gemini's client-owned audio plane — there is no WebRTC here):
94
+ * - **Audio in**: mic PCM16 @ 16 kHz captured via an inline-Blob `AudioWorklet`
95
+ * ({@link createMicCapture} seam) and streamed with `sendRealtimeInput`.
96
+ * - **Audio out**: model PCM16 @ 24 kHz chunks scheduled through {@link GeminiPcmPlayback}
97
+ * ({@link createPlayback} seam); {@link IsAudioPlaying} is computed from the playout clock.
98
+ * - **Event translation** (Gemini → contract): `inputTranscription` → User deltas/finals,
99
+ * `outputTranscription` → Assistant deltas/finals (accumulated like the OpenAI driver),
100
+ * `toolCall.functionCalls` → {@link OnToolCall} (callID→name cached for
101
+ * {@link SendToolResult}), `interrupted` → playback flush + `'listening'`,
102
+ * `turnComplete` → busy cleared + queued sends flushed, `usageMetadata` → {@link OnUsage}
103
+ * (per-turn prompt/response token deltas — see {@link handleUsageMetadata}).
104
+ * - **Busy mapping**: Gemini has no `response.created` frame, so `IsBusy` is set EAGERLY when
105
+ * this client triggers a response (text / narration / tool result) and on the first model
106
+ * output of a turn (audio part or output-transcription delta); cleared on `turnComplete`,
107
+ * and on a `toolCall` frame (the model has yielded the floor pending the tool result — so a
108
+ * slow `turnComplete` can never deadlock the queued result).
109
+ * - **Collision safety**: ANY `sendClientContent` interrupts in-flight Gemini generation (per
110
+ * the Live API contract), so text / narration / context-note / tool-result sends issued
111
+ * while a turn is in flight are queued and flushed in order on `turnComplete` (the flush
112
+ * stops at the first send that starts a new response).
113
+ * - **Open-turn commit**: context notes ride as `turnComplete: false` client content, which
114
+ * tells the Live API MORE INPUT IS COMING — the server holds ALL generation (including the
115
+ * normally-automatic continuation after a tool response) until a `turnComplete: true`
116
+ * commit. {@link SendToolResult} therefore follows `sendToolResponse` with an empty-turn
117
+ * commit whenever a note left the turn open, so the model speaks the result immediately
118
+ * (the behavioral equivalent of the OpenAI driver's explicit `response.create`).
119
+ - **Triggering turns ride realtime text**: typed text ({@link SendText}) and narration
120
+ * triggers ({@link RequestSpokenUpdate}) are sent via `sendRealtimeInput({ text })` — the
121
+ * Live API's documented in-conversation text path. Native-audio Live models treat
122
+ * `sendClientContent` as initial-history seeding only: a mid-call `turnComplete: true`
123
+ * client turn appends to history WITHOUT starting generation (the model stays silent until
124
+ * the user's next spoken turn), while realtime text triggers an immediate response on every
125
+ * model generation. Context notes stay on `sendClientContent` (`turnComplete: false`) — the
126
+ * silent history-append is exactly the contract they want.
127
+ * - **Narration tagging**: {@link RequestSpokenUpdate} has no per-response-instructions
128
+ * equivalent on Gemini, so it is emulated as a realtime-text user turn carrying the
129
+ * instructions; the response kind is stamped `'narration'` at send time (sends ARE
130
+ * the turn triggers on Gemini, unlike OpenAI where `response.created` confirms) and reset to
131
+ * `'normal'` when the turn completes.
132
+ */
133
+ export declare class GeminiRealtimeClient extends BaseRealtimeClient {
134
+ private session;
135
+ private micStream;
136
+ private micCapture;
137
+ private playback;
138
+ /** Accumulates the in-flight assistant transcript across delta frames. */
139
+ private pendingAssistantText;
140
+ /** Accumulates the in-flight user transcription across delta frames. */
141
+ private pendingUserText;
142
+ /** True while a model turn is in flight; gates (queues) client-triggered sends. */
143
+ private responseActive;
144
+ /** The kind of the turn currently in flight; stamped at send time, reset on turnComplete. */
145
+ private activeResponseKind;
146
+ /** Sends deferred while a turn is in flight; drained in order on turnComplete. */
147
+ private queuedSends;
148
+ /**
149
+ * Maps each pending tool call's `CallID` to its function name: Gemini's `sendToolResponse`
150
+ * requires the function name, which the {@link SendToolResult} contract does not carry.
151
+ */
152
+ private pendingToolCallNames;
153
+ /**
154
+ * True while client content sent with `turnComplete: false` (context notes) has not yet
155
+ * been committed. Per the Live API, an open client turn tells the server MORE INPUT IS
156
+ * COMING — clientContent-driven generation (notably the normally-automatic continuation
157
+ * after a tool response) holds until a `turnComplete: true` arrives. Set by
158
+ * {@link SendContextNote}; cleared by the empty-turn commit in
159
+ * {@link sendToolResponseTurn} and on a model `turnComplete` (the generation consumed the
160
+ * open content). Realtime-text triggers ({@link sendTriggeringUserTurn}) do NOT commit
161
+ * client content and leave this flag untouched.
162
+ */
163
+ private openClientTurn;
164
+ /**
165
+ * The client's own view of the session state — mirrors what was last emitted, EXCEPT after
166
+ * a tool call: the host typically shows its own busy indicator then, so the client silently
167
+ * leaves `'speaking'` (no emission) until the result reply's first output re-asserts it.
168
+ */
169
+ private currentState;
170
+ /**
171
+ * Opens the client-direct Gemini Live session: creates the playout engine, connects with
172
+ * the ephemeral token + the server-built `SessionConfig` (`{ model, config }` — the same
173
+ * values the server LOCKED into the token, so tampering is ignored by the API), then wires
174
+ * the mic-capture worklet. Reports `'listening'` once audio is flowing.
175
+ */
176
+ Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
177
+ /**
178
+ * Tears down the session, mic capture, mic tracks, and playout engine, resets the response
179
+ * state machine, and emits a final `'closed'` (unless already `'error'`). Safe to call
180
+ * more than once.
181
+ */
182
+ Disconnect(): Promise<void>;
183
+ /**
184
+ * Injects typed text as a realtime-text user turn (`sendRealtimeInput({ text })` —
185
+ * Gemini's "respond now" trigger on every Live model generation, including native-audio
186
+ * models that ignore mid-call `sendClientContent`). No-op when the session is not open.
187
+ *
188
+ * **SendText implies barge-in** (base-contract rule): an active spoken response is
189
+ * cancelled via {@link CancelActiveResponse} first — playback flushed, turn marked
190
+ * inactive, queued sends drained — so the typed turn takes the floor immediately. If a
191
+ * drained queued send (e.g. a pending tool result) starts a new turn, the text queues
192
+ * behind it, preserving the tool-result delivery invariant. On the wire, sending the
193
+ * user turn itself interrupts any residual server-side generation (Gemini Live's
194
+ * any-client-content-interrupts contract), so no explicit cancel frame exists or is
195
+ * needed.
196
+ */
197
+ SendText(text: string): void;
198
+ /**
199
+ * @inheritdoc
200
+ *
201
+ * Gemini has no explicit cancel frame — the client OWNS the audio plane, so cancelling
202
+ * means: flush the local playout queue ({@link GeminiPcmPlayback}) so speech stops
203
+ * immediately, mark the in-flight turn inactive, and drain queued sends (a queued tool
204
+ * result or context note takes the floor next — tool-result delivery is never dropped
205
+ * by a cancel). Server-side, the next client content sent naturally interrupts any
206
+ * residual generation per the Live API contract. The interrupted turn's accumulated
207
+ * transcript is kept — the provider's trailing frames finalize what WAS spoken. No-op
208
+ * when nothing is active.
209
+ */
210
+ CancelActiveResponse(): void;
211
+ /**
212
+ * Injects background context as a user turn with `turnComplete: false` — appended to the
213
+ * conversation WITHOUT starting generation, so the model draws on it the next time it
214
+ * speaks. Gemini Live turns have no system role, so the user role carries it (the host owns
215
+ * prefixing policy). Queued while a turn is in flight because ANY client content interrupts
216
+ * in-flight generation on Gemini (a divergence from the OpenAI driver, which can inject
217
+ * items mid-response safely).
218
+ *
219
+ * SIDE EFFECT: `turnComplete: false` leaves the client content turn OPEN — the server
220
+ * holds generation until a `turnComplete: true` commit arrives. {@link openClientTurn}
221
+ * tracks this so {@link SendToolResult} can commit the turn and unblock the spoken reply.
222
+ */
223
+ SendContextNote(text: string): void;
224
+ /**
225
+ * Triggers ONE short spoken update. Gemini has no per-response instructions (OpenAI's
226
+ * `response.create.instructions`), so this is EMULATED: the instructions ride as a
227
+ * realtime-text user turn (`sendRealtimeInput({ text })` — the path that triggers
228
+ * generation on native-audio models, where mid-call `sendClientContent` is inert), and
229
+ * the resulting turn is stamped `Kind: 'narration'` at send time (reset on
230
+ * `turnComplete`) — mirroring the OpenAI driver's narration semantics. Queued behind any
231
+ * in-flight turn so it can never interrupt a pending reply.
232
+ */
233
+ RequestSpokenUpdate(instructions: string): void;
234
+ /**
235
+ * Feeds an executed tool's result back via `sendToolResponse`, supplying the function name
236
+ * cached from the originating tool call (Gemini requires it; the contract only carries the
237
+ * callID). Sent immediately when idle — Gemini speaks the result as its next turn —
238
+ * otherwise queued until the in-flight turn (e.g. a progress narration) completes.
239
+ *
240
+ * If context notes left a client content turn OPEN (`turnComplete: false`), the tool
241
+ * response is followed by an empty-turn commit (`sendClientContent({ turnComplete: true })`)
242
+ * — without it the server keeps waiting for more client input and NEVER starts the spoken
243
+ * reply (observed live as the model staying silent after a delegated agent's result).
244
+ */
245
+ SendToolResult(callID: string, outputJson: string): void;
246
+ /**
247
+ * Mutes / unmutes by toggling the mic tracks' `enabled` flag: the capture pipeline stays
248
+ * up and streams SILENCE while muted (chosen over gating the worklet send so the provider's
249
+ * VAD sees a continuous stream and the un-mute is glitch-free — same policy as the OpenAI
250
+ * client driver).
251
+ */
252
+ SetMuted(muted: boolean): void;
253
+ /** @inheritdoc */
254
+ get IsBusy(): boolean;
255
+ /**
256
+ * @inheritdoc
257
+ *
258
+ * Computed directly from the playout engine's playhead clock — this client OWNS the output
259
+ * buffer (no WebRTC playback events exist on Gemini), so "audibly playing" is precisely
260
+ * "scheduled audio extends beyond the audio context's current time".
261
+ */
262
+ get IsAudioPlaying(): boolean;
263
+ /**
264
+ * Creation seam for the Gemini Live session. Production constructs a `GoogleGenAI` client
265
+ * with the **ephemeral token as the API key** on the `v1alpha` API version (the only
266
+ * version that accepts `auth_tokens`), then opens `live.connect` with the server-built
267
+ * model + config. Unit tests override this to return an in-memory fake.
268
+ */
269
+ protected connectLiveSession(args: GeminiClientConnectArgs): Promise<GeminiLiveClientSession>;
270
+ /**
271
+ * Creation seam for the mic-capture pipeline. Production delegates to the shared
272
+ * {@link createPcmMicCapture} (a 16 kHz `AudioContext`, inline-Blob capture worklet, and a
273
+ * zero-gain tail; each worklet block is PCM16-encoded and handed to `onPcmChunk` as base64).
274
+ * Unit tests override this with a no-op fake (and may capture `onPcmChunk` to simulate mic
275
+ * frames).
276
+ */
277
+ protected createMicCapture(micStream: MediaStream, onPcmChunk: (base64Pcm16: string) => void): Promise<IGeminiMicCapture>;
278
+ /** Creation seam for the playout engine. Production returns {@link GeminiPcmPlayback}. */
279
+ protected createPlayback(): IGeminiAudioPlayback;
280
+ /**
281
+ * Extracts the model + Live connect config from the server-minted `SessionConfig`
282
+ * (shaped `{ model, config }` by the server's `GeminiRealtime.CreateClientSession`).
283
+ * Falls back to the top-level `Model` / an empty config if a field is missing — the
284
+ * server locked the real config into the token, so the session still behaves correctly.
285
+ */
286
+ private parseSessionConfig;
287
+ /** Streams one base64 PCM16 mic chunk to the model (no-op once the session is gone). */
288
+ private sendMicChunk;
289
+ /** Surfaces a fatal websocket error and marks the session unusable. */
290
+ private handleTransportError;
291
+ /** Reflects a provider-side close (unless the session already ended in error). */
292
+ private handleTransportClose;
293
+ /**
294
+ * Entry point for every inbound {@link LiveServerMessage}; fans out to per-concern
295
+ * handlers, including `usageMetadata` → {@link emitUsage}.
296
+ */
297
+ private handleServerMessage;
298
+ /**
299
+ * Emits a usage update from a server message's `usageMetadata`.
300
+ *
301
+ * **Delta-vs-cumulative (verified against the `@google/genai` SDK types):** `UsageMetadata`
302
+ * documents `promptTokenCount` as "Number of tokens in the prompt" and `responseTokenCount`
303
+ * as "Total number of tokens across all the generated response candidates" — i.e. PER-RESPONSE
304
+ * counts for the turn this message reports, NOT a session-cumulative running total. Each
305
+ * emission is therefore already a delta, matching the `OnUsage` contract — and matching how
306
+ * the server-bridged `GeminiRealtime` driver forwards the same payload to `IRealtimeSession.OnUsage`.
307
+ */
308
+ private handleUsageMetadata;
309
+ /** Translates one {@link LiveServerContent} frame in provider-documented signal order. */
310
+ private handleServerContent;
311
+ /**
312
+ * Barge-in: the provider stopped generating because the user spoke. Flush every scheduled
313
+ * playout source (per the Live API contract, `interrupted` is the signal to empty the
314
+ * client's audio queue), surface the TRUE barge-in to the host (Gemini only emits
315
+ * `interrupted` when user input actually cut off in-flight generation — no extra gating
316
+ * needed), and give the floor back. The interrupted turn's accumulated transcript is
317
+ * kept — the following `turnComplete` finalizes what WAS spoken.
318
+ */
319
+ private handleInterruption;
320
+ /** Decodes inline model-audio parts (base64 PCM16 @ 24 kHz) into the playout queue. */
321
+ private handleModelAudio;
322
+ /**
323
+ * User transcription: each frame's `text` is an incremental DELTA (emitted with
324
+ * `IsFinal: false`); the accumulated turn text is finalized on the `finished` flag — or,
325
+ * because the user's turn is over once the model starts answering, by
326
+ * {@link markGenerationStarted}.
327
+ */
328
+ private handleUserTranscription;
329
+ /**
330
+ * Assistant transcription: deltas accumulate (like the OpenAI driver) and are emitted with
331
+ * the ACTIVE response kind so narration turns are tagged correctly; the turn finalizes on
332
+ * the `finished` flag, with {@link handleTurnComplete} as the fallback.
333
+ */
334
+ private handleAssistantTranscription;
335
+ /**
336
+ * Surfaces the model's tool calls to the host and caches each callID→name for
337
+ * {@link SendToolResult}. Two deliberate behaviors mirror the OpenAI driver: (1) the client
338
+ * silently leaves `'speaking'` (no emission) so a host-rendered busy indicator isn't
339
+ * clobbered by this turn's trailing frames; (2) `responseActive` is CLEARED — the model has
340
+ * yielded the floor pending the result, so a queued tool result can never deadlock waiting
341
+ * for a `turnComplete` that may not arrive until after the result is sent.
342
+ */
343
+ private handleToolCallFrame;
344
+ /**
345
+ * Turn boundary: finalize any un-finished assistant transcript, release the busy lock,
346
+ * reset the response kind to `'normal'`, drain queued sends (stopping at the first one
347
+ * that starts a new turn), and return the floor to the user.
348
+ */
349
+ private handleTurnComplete;
350
+ /**
351
+ * First model output of a turn (audio part or transcription delta): the user's turn is
352
+ * over (finalize their pending transcript), the model is busy, and the client is audibly /
353
+ * imminently `'speaking'`.
354
+ */
355
+ private markGenerationStarted;
356
+ /** Emits the accumulated user turn as final (if non-empty) and clears the accumulator. */
357
+ private finalizeUserTranscript;
358
+ /** Emits the accumulated assistant turn as final (if non-empty), tagged with its kind. */
359
+ private finalizeAssistantTranscript;
360
+ /**
361
+ * Runs a send immediately when no turn is in flight; otherwise queues it for the next
362
+ * `turnComplete`. This is Gemini's equivalent of the OpenAI driver's
363
+ * queue-behind-active-response rule — stricter here because ANY client content interrupts
364
+ * in-flight generation on the Live API.
365
+ */
366
+ private enqueueOrRun;
367
+ /**
368
+ * Drains queued sends in order on a turn boundary, stopping as soon as one starts a new
369
+ * turn (sets {@link responseActive}) — the rest wait for that turn's completion.
370
+ */
371
+ private flushQueuedSends;
372
+ /**
373
+ * Sends a user turn that TRIGGERS generation (`turnComplete: true`), stamping the upcoming
374
+ * turn's kind at send time and eagerly marking the model busy. `emitSpeaking` mirrors the
375
+ * OpenAI driver: typed-text / tool-result replies reflect `'speaking'` immediately;
376
+ * narration waits for the first model output.
377
+ */
378
+ private sendTriggeringUserTurn;
379
+ /**
380
+ * Sends the tool response (Gemini continues the turn with it) and marks the model busy.
381
+ *
382
+ * When context notes have left a client content turn open ({@link openClientTurn}), the
383
+ * tool response alone does NOT start generation — the Live API holds for more client input
384
+ * until the turn is committed. The empty-turn commit (the SDK-documented
385
+ * `sendClientContent({ turnComplete: true })` form) releases generation so the model
386
+ * speaks the result immediately, matching the OpenAI driver's explicit `response.create`.
387
+ */
388
+ private sendToolResponseTurn;
389
+ /**
390
+ * Parses a JSON-stringified tool result into the structured object Gemini's
391
+ * function-response slot expects, wrapping non-object output as `{ result: <value> }` so a
392
+ * free-text result still round-trips (same fallback as the server driver).
393
+ */
394
+ private parseToolOutput;
395
+ /** Resets the per-session response state machine (used on Disconnect). */
396
+ private resetResponseState;
397
+ /** Updates the client's own state view and emits the change to the host. */
398
+ private setState;
399
+ }
400
+ /**
401
+ * Tree-shaking prevention: bundlers cannot see that {@link GeminiRealtimeClient} is
402
+ * instantiated dynamically through the ClassFactory, so a consumer must call this no-op
403
+ * to create a static code path that keeps the `@RegisterClass` side effect alive.
404
+ */
405
+ export declare function LoadGeminiRealtimeClient(): void;
406
+ //# sourceMappingURL=geminiRealtimeClient.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"geminiRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/geminiRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAyB,MAAM,oBAAoB,CAAC;AACxF,OAAO,EAEH,KAAK,IAAI,IAAI,UAAU,EACvB,KAAK,OAAO,EAEZ,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EAEtB,KAAK,iBAAiB,EAEzB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,kBAAkB,EAAuB,MAAM,+BAA+B,CAAC;AAExF,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAuB,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAa1E;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,iBAAiB,CAAC,MAAM,EAAE;QAAE,KAAK,CAAC,EAAE,UAAU,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACvE,6FAA6F;IAC7F,iBAAiB,CAAC,MAAM,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;QAAC,YAAY,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAC/E,yEAAyE;IACzE,gBAAgB,CAAC,MAAM,EAAE;QAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAA;KAAE,GAAG,IAAI,CAAC;IAC1E,4CAA4C;IAC5C,KAAK,IAAI,IAAI,CAAC;CACjB;AAED;;;;;GAKG;AACH,MAAM,WAAW,uBAAuB;IACpC,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd,8FAA8F;IAC9F,MAAM,EAAE,iBAAiB,CAAC;IAC1B,0FAA0F;IAC1F,cAAc,EAAE,MAAM,CAAC;IACvB,qFAAqF;IACrF,SAAS,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAChD,kDAAkD;IAClD,OAAO,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IACrC,yCAAyC;IACzC,OAAO,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;CACxC;AAED;;;;;;GAMG;AACH,MAAM,MAAM,iBAAiB,GAAG,cAAc,CAAC;AAE/C;;;;;;;GAOG;AACH,MAAM,MAAM,oBAAoB,GAAG,oBAAoB,CAAC;AAIxD;;;;;;GAMG;AACH,qBAAa,iBAAkB,SAAQ,mBAAmB;;CAIzD;AAID;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AACH,qBACa,oBAAqB,SAAQ,kBAAkB;IAExD,OAAO,CAAC,OAAO,CAAwC;IACvD,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,QAAQ,CAAqC;IAGrD,0EAA0E;IAC1E,OAAO,CAAC,oBAAoB,CAAM;IAClC,wEAAwE;IACxE,OAAO,CAAC,eAAe,CAAM;IAC7B,mFAAmF;IACnF,OAAO,CAAC,cAAc,CAAS;IAC/B,6FAA6F;IAC7F,OAAO,CAAC,kBAAkB,CAAoC;IAC9D,kFAAkF;IAClF,OAAO,CAAC,WAAW,CAAyB;IAC5C;;;OAGG;IACH,OAAO,CAAC,oBAAoB,CAA6B;IACzD;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc,CAAS;IAC/B;;;;OAIG;IACH,OAAO,CAAC,YAAY,CAAiC;IAIrD;;;;;OAKG;IACU,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBhG;;;;OAIG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBxC;;;;;;;;;;;;;OAaG;IACI,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAQnC;;;;;;;;;;;OAWG;IACI,oBAAoB,IAAI,IAAI;IAgBnC;;;;;;;;;;;OAWG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAU1C;;;;;;;;OAQG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAOtD;;;;;;;;;;OAUG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAQ/D;;;;;OAKG;IACI,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAOrC,kBAAkB;IAClB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED;;;;;;OAMG;IACH,IAAW,cAAc,IAAI,OAAO,CAEnC;IAID;;;;;OAKG;cACa,kBAAkB,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAanG;;;;;;OAMG;cACa,gBAAgB,CAC5B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GAC1C,OAAO,CAAC,iBAAiB,CAAC;IAI7B,0FAA0F;IAC1F,SAAS,CAAC,cAAc,IAAI,oBAAoB;IAMhD;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAS1B,wFAAwF;IACxF,OAAO,CAAC,YAAY;IAMpB,uEAAuE;IACvE,OAAO,CAAC,oBAAoB;IAK5B,kFAAkF;IAClF,OAAO,CAAC,oBAAoB;IAQ5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAY3B;;;;;;;;;OASG;IACH,OAAO,CAAC,mBAAmB;IAQ3B,0FAA0F;IAC1F,OAAO,CAAC,mBAAmB;IAkB3B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAM1B,uFAAuF;IACvF,OAAO,CAAC,gBAAgB;IAaxB;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;;OAIG;IACH,OAAO,CAAC,4BAA4B;IAgBpC;;;;;;;OAOG;IACH,OAAO,CAAC,mBAAmB;IAgB3B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAQ7B,0FAA0F;IAC1F,OAAO,CAAC,sBAAsB;IAQ9B,0FAA0F;IAC1F,OAAO,CAAC,2BAA2B;IAUnC;;;;;OAKG;IACH,OAAO,CAAC,YAAY;IAQpB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAOxB;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;IAoB9B;;;;;;;;OAQG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAcvB,0EAA0E;IAC1E,OAAO,CAAC,kBAAkB;IAU1B,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;CAInB;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,IAAI,IAAI,CAE/C"}