@memberjunction/ai-realtime-client 0.0.1 → 5.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +148 -28
  2. package/dist/audio/audioMeter.d.ts +101 -0
  3. package/dist/audio/audioMeter.d.ts.map +1 -0
  4. package/dist/audio/audioMeter.js +193 -0
  5. package/dist/audio/audioMeter.js.map +1 -0
  6. package/dist/audio/micCapture.d.ts +26 -0
  7. package/dist/audio/micCapture.d.ts.map +1 -0
  8. package/dist/audio/micCapture.js +69 -0
  9. package/dist/audio/micCapture.js.map +1 -0
  10. package/dist/audio/pcmPlayback.d.ts +73 -0
  11. package/dist/audio/pcmPlayback.d.ts.map +1 -0
  12. package/dist/audio/pcmPlayback.js +78 -0
  13. package/dist/audio/pcmPlayback.js.map +1 -0
  14. package/dist/audio/pcmUtils.d.ts +17 -0
  15. package/dist/audio/pcmUtils.d.ts.map +1 -0
  16. package/dist/audio/pcmUtils.js +46 -0
  17. package/dist/audio/pcmUtils.js.map +1 -0
  18. package/dist/drivers/assemblyAIRealtimeClient.d.ts +384 -0
  19. package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -0
  20. package/dist/drivers/assemblyAIRealtimeClient.js +732 -0
  21. package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -0
  22. package/dist/drivers/elevenLabsRealtimeClient.d.ts +362 -0
  23. package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -0
  24. package/dist/drivers/elevenLabsRealtimeClient.js +686 -0
  25. package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -0
  26. package/dist/drivers/geminiRealtimeClient.d.ts +406 -0
  27. package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -0
  28. package/dist/drivers/geminiRealtimeClient.js +675 -0
  29. package/dist/drivers/geminiRealtimeClient.js.map +1 -0
  30. package/dist/drivers/openAIRealtimeClient.d.ts +381 -0
  31. package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -0
  32. package/dist/drivers/openAIRealtimeClient.js +602 -0
  33. package/dist/drivers/openAIRealtimeClient.js.map +1 -0
  34. package/dist/drivers/xaiRealtimeClient.d.ts +430 -0
  35. package/dist/drivers/xaiRealtimeClient.d.ts.map +1 -0
  36. package/dist/drivers/xaiRealtimeClient.js +676 -0
  37. package/dist/drivers/xaiRealtimeClient.js.map +1 -0
  38. package/dist/generic/baseRealtimeClient.d.ts +401 -0
  39. package/dist/generic/baseRealtimeClient.d.ts.map +1 -0
  40. package/dist/generic/baseRealtimeClient.js +226 -0
  41. package/dist/generic/baseRealtimeClient.js.map +1 -0
  42. package/dist/index.d.ts +11 -0
  43. package/dist/index.d.ts.map +1 -0
  44. package/dist/index.js +11 -0
  45. package/dist/index.js.map +1 -0
  46. package/package.json +28 -7
@@ -0,0 +1 @@
1
+ {"version":3,"file":"xaiRealtimeClient.js","sourceRoot":"","sources":["../../src/drivers/xaiRealtimeClient.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAuB,MAAM,+BAA+B,CAAC;AACxF,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAwB,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAE1E,kFAAkF;AAElF;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,4BAA4B,CAAC;AAEhE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,oBAAoB,CAAC;AAEzE;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CAAC;AAuLzC,kFAAkF;AAElF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEI,IAAM,iBAAiB,GAAvB,MAAM,iBAAkB,SAAQ,kBAAkB;IAAlD;;QACH,8EAA8E;QACtE,WAAM,GAA4B,IAAI,CAAC;QACvC,cAAS,GAAuB,IAAI,CAAC;QACrC,eAAU,GAA0B,IAAI,CAAC;QACzC,aAAQ,GAAgC,IAAI,CAAC;QACrD;;;WAGG;QACO,kBAAa,GAAsB,IAAI,CAAC;QAElD,8EAA8E;QAC9E;;;;;WAKG;QACK,wBAAmB,GAAG,KAAK,CAAC;QACpC,0EAA0E;QAClE,yBAAoB,GAAG,EAAE,CAAC;QAClC,+FAA+F;QACvF,mBAAc,GAAG,KAAK,CAAC;QAC/B,kGAAkG;QAC1F,0BAAqB,GAAG,KAAK,CAAC;QACtC;;;;WAIG;QACK,yBAAoB,GAAG,KAAK,CAAC;QACrC;;;;;WAKG;QACK,uBAAkB,GAA2B,QAAQ,CAAC;QAC9D;;;;;WAKG;QACK,iBAAY,GAAwB,QAAQ,CAAC;QACrD,qFAAqF;QAC7E,qBAAgB,GAAG,KAAK,CAAC;IAolBrC,CAAC;IAllBG,8EAA8E;IAE9E;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,MAAmC,EAAE,SAAsB;QAC5E,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE5B,IAAI,UAAU,GAAwB,IAAI,CAAC;QAC3C,IAAI,QAAQ,GAAoC,IAAI,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACjD,UAAU,GAAG,OAAO,CAAC;YACrB,QAAQ,GAAG,MAAM,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,mBAAmB,UAAU,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/E,MAAM,WAAW,GAAG,GAAG,oCAAoC,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QACtF,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC;QACrC,MAAM,CAAC,SAAS,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,CAAC,OAAO,GAAG,CAAC,OAAO,EAAE,EAAE;YACzB,QAAQ,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC,CAAC;QACF,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;YAClB,QAAQ,EAAE,CAAC,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;YACrE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7B,CAAC,CAAC;QAEF,MAAM,MAAM,CAAC;QACb,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC3B,sFAAsF;QACtF,wFAAwF;QACxF,uFAAuF;QACvF,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;QACzD,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,mBAAmB,EAAE,CAAC,WAAW,EAAE,EAAE,CAC1F,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CACjC,CAAC;QACF,uFAAuF;QACvF,qFAAqF;QACrF,oDAAoD;QACpD,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,IAAI,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,UAAU;QACnB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACL,qBAAqB;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,6EAA6E;IAE7E;;;;;;;;;OASG;IACI,QAAQ,CAAC,IAAY;QACxB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,CAAC;YACX,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACF,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aAC1C;SACJ,CAAC,CAAC;QACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACjC,CAAC;IAED;;;;;;;;;OASG;IACI,oBAAoB;QACvB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,OAAO,CAAC,qCAAqC;QACjD,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;YAC5C,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;YAC5B,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;YAClC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACnC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QACnC,CAAC;QACD,wFAAwF;QACxF,kFAAkF;QAClF,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACI,eAAe,CAAC,IAAY;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC;YACX,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACF,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aAC1C;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACI,mBAAmB,CAAC,YAAoB;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,8GAA8G,CAAC,CAAC;YAC9H,OAAO;QACX,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED;;;;;;OAMG;IACI,cAAc,CAAC,MAAc,EAAE,UAAkB;QACpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC;YACX,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACF,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,UAAU;aACrB;SACJ,CAAC,CAAC;QACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACI,QAAQ,CAAC,KAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;QACtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACH,IAAW,cAAc;QACrB,OAAO,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,KAAK,CAAC;IAC7C,CAAC;IAED,6EAA6E;IAE7E;;;;;;OAMG;IACO,YAAY,CAAC,GAAW,EAAE,WAAmB;QACnD,MAAM,EAAE,GAAI,UAAiH,CAAC,SAAS,CAAC;QACxI,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;QAC5F,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACpC,MAAM,IAAI,GAAqB;YAC3B,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;YAC7B,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE;SAC1B,CAAC;QACF,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,EAAE,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,gCAAgC,CAAC,CAAC;QACpE,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAC5B,SAAsB,EACtB,UAAkB,EAClB,UAAyC;QAEzC,OAAO,mBAAmB,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAClE,CAAC;IAED;;;OAGG;IACO,cAAc,CAAC,UAAkB;QACvC,OAAO,IAAI,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC;IAED,8EAA8E;IAE9E;;;;;OAKG;IACK,kBAAkB;QACtB,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtE,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,kFAAkF;IAC1E,YAAY,CAAC,WAAmB;QACpC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QAC9E,CAAC;IACL,CAAC;IAED,oFAAoF;IAC5E,iBAAiB,CAAC,OAAe;QACrC,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAClE,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,4CAA4C,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAChG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACK,iBAAiB;QACrB,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAC3F,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,wDAAwD,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,8EAA8E;IAE9E,kEAAkE;IAC1D,mBAAmB,CAAC,IAAY;QACpC,IAAI,KAAuB,CAAC;QAC5B,IAAI,CAAC;YACD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAqB,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3F,OAAO,CAAC,0BAA0B;QACtC,CAAC;QACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9C,OAAO,CAAC,sEAAsE;QAClF,CAAC;QACD,+FAA+F;QAC/F,8FAA8F;QAC9F,8FAA8F;QAC9F,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,IAAI,EACtD,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,gFAAgF;IACxE,WAAW,CAAC,KAAuB;QACvC,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,wCAAwC,CAAC;YAC9C,KAAK,iCAAiC;gBAClC,IAAI,CAAC,gBAAgB,CAAE,KAAyC,CAAC,KAAK,CAAC,CAAC;gBACxE,MAAM;YACV,KAAK,uCAAuC,CAAC;YAC7C,KAAK,gCAAgC;gBACjC,IAAI,CAAC,eAAe,CAAE,KAAwC,CAAC,UAAU,CAAC,CAAC;gBAC3E,MAAM;YACV,KAAK,sBAAsB,CAAC;YAC5B,KAAK,6BAA6B;gBAC9B,IAAI,CAAC,YAAY,CAAE,KAA+B,CAAC,KAAK,CAAC,CAAC;gBAC1D,MAAM;YACV,KAAK,uDAAuD;gBACxD,IAAI,CAAC,gBAAgB,CAAE,KAA6C,CAAC,UAAU,CAAC,CAAC;gBACjF,MAAM;YACV,KAAK,uCAAuC;gBACxC,IAAI,CAAC,eAAe,CAAC,KAAqC,CAAC,CAAC;gBAC5D,MAAM;YACV,KAAK,mCAAmC;gBACpC,IAAI,CAAC,eAAe,EAAE,CAAC;gBACvB,MAAM;YACV,KAAK,kBAAkB;gBACnB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,6EAA6E;gBAC7E,8EAA8E;gBAC9E,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC7E,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;gBAClC,MAAM;YACV,KAAK,eAAe;gBAChB,6EAA6E;gBAC7E,qEAAqE;gBACrE,uEAAuE;gBACvE,gEAAgE;gBAChE,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;gBACnC,IAAI,CAAC,iBAAiB,CAAC,KAAwB,CAAC,CAAC;gBACjD,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAClC,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;oBACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC/B,CAAC;gBACD,MAAM;YACV,KAAK,OAAO;gBACR,IAAI,CAAC,YAAY,CAAC,KAAsB,CAAC,CAAC;gBAC1C,MAAM;YACV;gBACI,uEAAuE;gBACvE,MAAM;QACd,CAAC;IACL,CAAC;IAED,yFAAyF;IACjF,gBAAgB,CAAC,KAAa;QAClC,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,oBAAoB,IAAI,KAAK,CAAC;QACnC,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC3G,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,UAAkB;QACtC,MAAM,SAAS,GAAG,UAAU,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAC1D,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAC/B,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC9G,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,YAAY,CAAC,WAAmB;QACpC,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED;;;;;;OAMG;IACK,gBAAgB,CAAC,UAAkB;QACvC,IAAI,UAAU,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,cAAc,CAAC;gBAChB,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ;gBAC7D,gBAAgB,EAAE,IAAI,CAAC,mBAAmB;aAC7C,CAAC,CAAC;YACH,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QACpC,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACK,eAAe,CAAC,IAAkC;QACtD,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QACpC,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IACpG,CAAC;IAED;;;;;;;OAOG;IACK,eAAe;QACnB,gGAAgG;QAChG,qFAAqF;QACrF,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACK,iBAAiB,CAAC,KAAsB;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC;YACX,WAAW,EAAE,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;YACpF,YAAY,EAAE,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;YACvF,GAAG,EAAE,KAAK;SACb,CAAC,CAAC;IACP,CAAC;IAED,0EAA0E;IAClE,YAAY,CAAC,KAAoB;QACrC,IAAI,CAAC,SAAS,CAAC;YACX,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,IAAI,wBAAwB;YACzD,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI;YACvB,KAAK,EAAE,KAAK;SACf,CAAC,CAAC;IACP,CAAC;IAED,8EAA8E;IAE9E;;;;;;OAMG;IACK,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;YAClC,OAAO;QACX,CAAC;QACD,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,0FAA0F;IAClF,0BAA0B;QAC9B,IAAI,CAAC,IAAI,CAAC,qBAAqB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9C,OAAO;QACX,CAAC;QACD,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,0EAA0E;IAClE,kBAAkB;QACtB,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;QACnC,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;QAClC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAED,8EAA8E;IAE9E,4EAA4E;IACpE,QAAQ,CAAC,KAA0B;QACvC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,+EAA+E;IACvE,SAAS,CAAC,KAA6B;QAC3C,4FAA4F;QAC5F,4FAA4F;QAC5F,uCAAuC;QACvC,IAAI,KAAK,CAAC,IAAI,KAAK,2BAA2B,EAAE,CAAC;YAC7C,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7C,CAAC;CACJ,CAAA;AAnoBY,iBAAiB;IAD7B,aAAa,CAAC,kBAAkB,EAAE,KAAK,CAAC;GAC5B,iBAAiB,CAmoB7B;;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB;IACjC,oEAAoE;AACxE,CAAC"}
@@ -0,0 +1,401 @@
1
+ import { ClientRealtimeSessionConfig } from '@memberjunction/ai';
2
+ import { IRealtimeAudioMeter } from '../audio/audioMeter.js';
3
+ /**
4
+ * A point-in-time snapshot of the session's audible activity, sampled by the host
5
+ * (typically per animation frame) via {@link BaseRealtimeClient.GetAudioActivity} to drive
6
+ * audio-reactive call visuals — the orb that vibrates with the agent's voice and the
7
+ * true-spectrum EQ.
8
+ *
9
+ * CAPABILITY-SHAPED, like usage telemetry: every member is `null` when that direction has
10
+ * no meter. Hosts must degrade gracefully — levels-without-bins renders envelope breathing,
11
+ * full `null` falls back to turn-state-driven animation.
12
+ */
13
+ export interface RealtimeAudioActivity {
14
+ /** RMS level (0..1) of the USER's microphone, or `null` when un-metered. */
15
+ InputLevel: number | null;
16
+ /** RMS level (0..1) of the AGENT's audio output, or `null` when un-metered. */
17
+ OutputLevel: number | null;
18
+ /** Mic frequency spectrum as {@link REALTIME_AUDIO_BIN_COUNT} normalized bins, or `null`. */
19
+ InputBins: number[] | null;
20
+ /** Agent-output frequency spectrum as {@link REALTIME_AUDIO_BIN_COUNT} normalized bins, or `null`. */
21
+ OutputBins: number[] | null;
22
+ }
23
+ /**
24
+ * A transcript event emitted by a realtime client for either side of the conversation.
25
+ *
26
+ * `Kind` distinguishes NORMAL conversation turns from NARRATION turns — the brief,
27
+ * ephemeral spoken updates a host requests via {@link BaseRealtimeClient.RequestSpokenUpdate}
28
+ * while background (delegated) work runs. By product decision narration transcripts are
29
+ * NOT captions and are never persisted; hosts surface them as a transient "live note".
30
+ */
31
+ export interface RealtimeClientTranscript {
32
+ /** Whose turn this transcript belongs to. */
33
+ Role: 'User' | 'Assistant';
34
+ /**
35
+ * The transcribed text. For `IsFinal: false` events this is the incremental DELTA
36
+ * for the in-flight turn; for `IsFinal: true` it is the complete turn text.
37
+ */
38
+ Text: string;
39
+ /** Whether this is the final transcript for the turn (`true`) or an interim delta (`false`). */
40
+ IsFinal: boolean;
41
+ /**
42
+ * `'normal'` for a regular conversation turn; `'narration'` for an ephemeral
43
+ * spoken-progress update triggered by {@link BaseRealtimeClient.RequestSpokenUpdate}.
44
+ */
45
+ Kind: 'normal' | 'narration';
46
+ /**
47
+ * MACHINE-READABLE correction marker: `true` when this FINAL transcript SUPERSEDES the
48
+ * session's immediately-preceding final transcript of the same role — e.g. ElevenLabs'
49
+ * post-barge-in `agent_response_correction`, which re-finalizes the assistant turn with
50
+ * the text that was ACTUALLY spoken before the interruption cut it off. Hosts that
51
+ * persist finals must UPDATE the previous turn in place instead of appending a
52
+ * duplicate. Absent/`false` on ordinary turns and on interim deltas.
53
+ */
54
+ ReplacesPrevious?: boolean;
55
+ }
56
+ /**
57
+ * A tool-call request emitted by the realtime model. The host executes the tool and feeds
58
+ * the result back via {@link BaseRealtimeClient.SendToolResult}, correlated by `CallID`.
59
+ */
60
+ export interface RealtimeClientToolCall {
61
+ /** Provider-assigned identifier for this call, used to correlate the eventual tool result. */
62
+ CallID: string;
63
+ /** The name of the tool the model is requesting to invoke. */
64
+ ToolName: string;
65
+ /** The arguments for the call as a JSON string, exactly as the provider emitted them. */
66
+ ArgumentsJson: string;
67
+ }
68
+ /**
69
+ * A usage/telemetry update emitted by a realtime client.
70
+ *
71
+ * **Deltas preferred:** each emission SHOULD carry the incremental token counts for the
72
+ * response/turn that just completed (not a session-cumulative running total), matching the
73
+ * server-side `RealtimeUsage` contract — consumers accumulate. A driver whose provider only
74
+ * reports cumulative totals must convert to deltas before emitting.
75
+ *
76
+ * **Per-driver availability** (capability, not laziness — providers without usage events
77
+ * simply never emit):
78
+ * - **OpenAI** — emits per-response deltas from the GA `response.done` frame's `usage` payload.
79
+ * - **Gemini** — emits per-turn deltas from `LiveServerMessage.usageMetadata`
80
+ * (`promptTokenCount` / `responseTokenCount` are per-response counts, not cumulative).
81
+ * - **ElevenLabs** — the Conversational AI socket exposes no usage events; never emits.
82
+ * - **AssemblyAI** — the streaming STT socket exposes no token-usage events; never emits.
83
+ */
84
+ export interface RealtimeClientUsage {
85
+ /** Input tokens reported in this update (a delta for the completed response/turn). */
86
+ InputTokens?: number;
87
+ /** Output tokens reported in this update (a delta for the completed response/turn). */
88
+ OutputTokens?: number;
89
+ /** The raw provider usage payload, for hosts that want provider-specific detail. */
90
+ Raw?: unknown;
91
+ }
92
+ /**
93
+ * Connection / turn state of a realtime client session.
94
+ * - `connecting` — the provider connection is being negotiated
95
+ * - `connected` — the transport is up but the control channel is not yet open
96
+ * - `listening` — control channel open; mic live; waiting on / hearing the user
97
+ * - `speaking` — the model is producing (or about to produce) audible output
98
+ * - `closed` — the session has been torn down
99
+ * - `error` — a fatal transport error occurred; the session is no longer usable
100
+ *
101
+ * Note there is deliberately NO `thinking` state here: "the host is executing a tool" is
102
+ * host policy, not wire state. Hosts that want a busy indicator set it themselves between
103
+ * receiving a tool call and calling {@link BaseRealtimeClient.SendToolResult}.
104
+ */
105
+ export type RealtimeClientState = 'connecting' | 'connected' | 'listening' | 'speaking' | 'closed' | 'error';
106
+ /**
107
+ * An error surfaced by a realtime client. `Fatal: true` means the session is unusable
108
+ * (transport failure); `Fatal: false` is a provider-reported, recoverable error frame.
109
+ */
110
+ export interface RealtimeClientError {
111
+ /** Human-readable error message. */
112
+ Message: string;
113
+ /** Optional provider-specific error code. */
114
+ Code?: string;
115
+ /** Whether the error terminated the session. */
116
+ Fatal: boolean;
117
+ }
118
+ /**
119
+ * Abstract base class for **browser-side, provider-direct** realtime (voice) clients.
120
+ *
121
+ * This is the client-side mirror of the server's `BaseRealtimeModel` pattern: the server
122
+ * mints an ephemeral credential + session config (`ClientRealtimeSessionConfig`) via its
123
+ * provider driver, and the browser resolves the matching CLIENT driver through the
124
+ * MemberJunction `ClassFactory` using the config's `Provider` string as the registration
125
+ * key (e.g. `'openai'`):
126
+ *
127
+ * ```typescript
128
+ * const client = MJGlobal.Instance.ClassFactory.CreateInstance<BaseRealtimeClient>(
129
+ * BaseRealtimeClient, startResult.Provider);
130
+ * ```
131
+ *
132
+ * Concrete drivers (e.g. {@link OpenAIRealtimeClient}) own ALL provider wire concerns:
133
+ * transport (WebRTC/WebSocket), event-name translation, the response state machine
134
+ * (never letting a tool-result reply collide with an in-flight response), narration-kind
135
+ * tagging, and audible-playback tracking. Hosts own POLICY: when to narrate, what
136
+ * instructions to speak, transcript persistence, and UI state.
137
+ *
138
+ * All `On*` methods register a single handler (matching the server `IRealtimeSession`
139
+ * style); registering again replaces the previous handler.
140
+ *
141
+ * ## DRIVER AUTHOR OBLIGATIONS
142
+ *
143
+ * Hard-won, provider-independent rules every client driver MUST honor (the client-side
144
+ * mirror of the obligations documented on the server's `BaseRealtimeModel` in
145
+ * `@memberjunction/ai`). Most were paid for in live debugging; do not relearn them:
146
+ *
147
+ * 1. **Silent exit from `'speaking'` after a tool call.** When the model emits a tool
148
+ * call, the host typically shows its own busy ("thinking") indicator while the tool
149
+ * executes. The driver must leave the `'speaking'` state *silently* (no state
150
+ * emission) at tool-call time so the turn's trailing frames don't clobber the host's
151
+ * indicator.
152
+ * 2. **Busy-flag release on tool-call emission (deadlock guard).** A tool-call frame
153
+ * means the model has yielded the floor pending the result. Any internal "response
154
+ * active" flag MUST be cleared then — otherwise the eventual {@link SendToolResult}
155
+ * (or a queued send) deadlocks waiting for a turn boundary that will never arrive
156
+ * until after the result is sent.
157
+ * 3. **Playback flush + honest {@link IsAudioPlaying} on interruption.** On a true
158
+ * barge-in the driver must flush any locally-owned audio playback and report
159
+ * `IsAudioPlaying === false` promptly — stale queued audio after an interruption is
160
+ * a product bug, not a nicety. The same flush applies to {@link CancelActiveResponse}.
161
+ * 4. **{@link SendText} must NOT echo a user transcript — and implies barge-in.** The
162
+ * driver must not synthesize a user-role transcript event for injected text (the host
163
+ * owns the local echo and would render the message twice), and an active spoken
164
+ * response is cancelled via the driver's own cancel path before the text is injected
165
+ * (see {@link CancelActiveResponse}).
166
+ * 5. **Tool-result delivery invariant.** Every result fed back via {@link SendToolResult}
167
+ * must EVENTUALLY be voiced/processed by the model and never dropped. If the provider
168
+ * rejects overlapping generation triggers, the driver queues the result's trigger
169
+ * behind the in-flight response and flushes it at the next turn boundary. The result
170
+ * must also EXPLICITLY trigger generation on providers that don't auto-continue after
171
+ * a tool response: OpenAI requires an explicit `response.create`; Gemini auto-continues
172
+ * UNLESS prior client content (a context note's `turnComplete: false`) left the turn
173
+ * open, in which case the driver must commit the turn or the model stays silent.
174
+ * 6. **Token/credential expiry surfaces as a FATAL error.** When the ephemeral credential
175
+ * dies (expiry, revocation, unexpected socket loss), the driver must surface it through
176
+ * {@link OnError} with `Fatal: true` so the host tears down cleanly instead of idling
177
+ * forever on a dead connection.
178
+ * 7. **`'listening'` only after the session config is applied.** The driver must not
179
+ * report the session as listening until the server-built session config (system prompt
180
+ * + tools) has actually been applied to the provider socket — otherwise early turns run
181
+ * against an unconfigured model.
182
+ * 8. **`SessionConfig` is a private pact.** The `ClientRealtimeSessionConfig.SessionConfig`
183
+ * payload is authored by the same-keyed SERVER driver and consumed only by this client
184
+ * driver; its shape may change between the two driver halves without notice. Hosts and
185
+ * intermediaries treat it as an opaque blob — and so must any code outside the matching
186
+ * driver pair.
187
+ * 9. **Audio metering is a CAPABILITY, not an obligation — but wire what you own.** A
188
+ * driver that owns (or can tap) an audio plane should attach
189
+ * {@link BaseRealtimeClient.attachInputAudioMeter} / {@link attachOutputAudioMeter} so
190
+ * {@link GetAudioActivity} reports real levels: the mic stream everywhere
191
+ * (`RealtimeAudioMeter.ForStream`), the shared `RealtimePcmPlayback.CreateMeter()` on
192
+ * client-owned-audio drivers, the remote WebRTC stream on peer-connection drivers.
193
+ * Meters must be released on disconnect ({@link closeAudioMeters}). A driver with no
194
+ * tappable plane simply attaches nothing — hosts fall back to turn-state animation.
195
+ */
196
+ export declare abstract class BaseRealtimeClient {
197
+ private transcriptHandler?;
198
+ private toolCallHandler?;
199
+ private stateChangeHandler?;
200
+ private errorHandler?;
201
+ private interruptionHandler?;
202
+ private usageHandler?;
203
+ private remoteVideoHandler?;
204
+ /** Meter over the USER's microphone, when the driver attached one. */
205
+ private inputAudioMeter;
206
+ /** Meter over the AGENT's audio output, when the driver attached one. */
207
+ private outputAudioMeter;
208
+ /**
209
+ * The session's current audible activity, or `null` when this driver attached no
210
+ * meters at all (hosts then keep their turn-state-driven visuals). Sampled by hosts
211
+ * per animation frame — implementations are cheap, allocation-light reads of an
212
+ * `AnalyserNode`; no per-call provider traffic.
213
+ */
214
+ GetAudioActivity(): RealtimeAudioActivity | null;
215
+ /** Attaches (replacing + closing any previous) the USER-microphone meter. `null` detaches. */
216
+ protected attachInputAudioMeter(meter: IRealtimeAudioMeter | null): void;
217
+ /** Attaches (replacing + closing any previous) the AGENT-output meter. `null` detaches. */
218
+ protected attachOutputAudioMeter(meter: IRealtimeAudioMeter | null): void;
219
+ /** Releases both meters — every driver calls this from its disconnect/teardown path. */
220
+ protected closeAudioMeters(): void;
221
+ /**
222
+ * Opens the provider connection using the server-minted ephemeral credential and applies
223
+ * `config.SessionConfig` once the control channel is ready — so prompt and tool authority
224
+ * stay server-side even though the browser owns the socket. The payload's SHAPE is a
225
+ * private pact between this driver and the same-keyed server driver that minted it (see
226
+ * driver obligation #8); how it is applied is entirely driver-specific.
227
+ *
228
+ * @param config The server-minted client session config (provider, model, ephemeral token, session config).
229
+ * @param micStream The user's microphone capture stream. The caller acquires it (so IT owns
230
+ * the permission prompt UX); the client attaches it to the transport and stops its tracks
231
+ * on {@link Disconnect}.
232
+ * @param cameraStream OPTIONAL camera capture stream for a VIDEO session (the model "sees" the
233
+ * user). Only attached by video-capable drivers; audio-only drivers ignore it (back-compatible —
234
+ * existing callers and drivers need not pass or read it). The model/avatar's video comes BACK via
235
+ * {@link OnRemoteVideo}.
236
+ */
237
+ abstract Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream, cameraStream?: MediaStream): Promise<void>;
238
+ /**
239
+ * Injects typed text into the live session as a USER turn and asks the model to respond.
240
+ * Implementations must route the reply through the same collision-safe path as tool
241
+ * results so it never collides with an in-flight response. No-op when the control
242
+ * channel is not open.
243
+ *
244
+ * **SendText implies barge-in:** an active spoken response is cancelled (via the
245
+ * driver's own {@link CancelActiveResponse} path) before the text is injected, so the
246
+ * typed turn takes the floor immediately instead of waiting behind stale speech.
247
+ *
248
+ * Implementations must NOT synthesize a user-role transcript event for the injected
249
+ * text — the host owns the local echo (driver obligation #4).
250
+ *
251
+ * @param text The user's typed message (callers pass pre-trimmed, non-empty text).
252
+ */
253
+ abstract SendText(text: string): void;
254
+ /**
255
+ * Cancels the model's ACTIVE spoken response and flushes pending playback so a new user
256
+ * turn can take the floor immediately; no-op when nothing is active. MUST NOT affect
257
+ * server-side delegated work — cancelling speech is a floor-control action, not an abort
258
+ * of in-flight tools/agents (hosts abort delegated work from {@link OnInterruption} /
259
+ * their own policy, never from this call).
260
+ *
261
+ * Implementations must leave {@link IsBusy} and {@link IsAudioPlaying} honest after the
262
+ * cancel (response inactive, local playback flushed).
263
+ */
264
+ abstract CancelActiveResponse(): void;
265
+ /**
266
+ * Injects background context into the model's conversation WITHOUT forcing a reply —
267
+ * e.g. delegated-run progress the model can draw on the next time it speaks.
268
+ *
269
+ * @param text The context note text (the host owns any prefixing/formatting policy).
270
+ */
271
+ abstract SendContextNote(text: string): void;
272
+ /**
273
+ * Asks the model for ONE brief interim utterance (e.g. a spoken progress update while
274
+ * delegated work runs). Implementations MUST tag the resulting turn's transcripts with
275
+ * `Kind: 'narration'` and MUST NOT let this response collide with a pending tool-result
276
+ * reply — the tool result is queued and spoken as soon as the narration finishes.
277
+ *
278
+ * **Collision rule — drivers MUST queue or skip:** when a model response is already in
279
+ * flight, the driver must either defer the update to the next turn boundary or skip it
280
+ * outright (skipping is acceptable — narration is disposable by contract; a stale
281
+ * "still working…" line has no value once the real result lands). Hosts SHOULD still
282
+ * gate calls on {@link IsBusy} / {@link IsAudioPlaying} for timing quality — a
283
+ * well-timed update beats a queued-then-stale one — but the driver is the safety net.
284
+ *
285
+ * @param instructions The exact provider instructions for the utterance. The instruction
286
+ * TEXT is host policy (e.g. first-person phrasing rules) — the client only carries it.
287
+ */
288
+ abstract RequestSpokenUpdate(instructions: string): void;
289
+ /**
290
+ * Feeds an executed tool's result back to the model, correlated by `callID`, and ensures
291
+ * the model SPEAKS the result as soon as possible: immediately when idle, otherwise
292
+ * queued behind the in-flight response (e.g. a progress narration) so the trigger is
293
+ * never dropped by the provider.
294
+ *
295
+ * @param callID The `CallID` from the originating {@link RealtimeClientToolCall}.
296
+ * @param outputJson The tool's result as a JSON-stringified string.
297
+ */
298
+ abstract SendToolResult(callID: string, outputJson: string): void;
299
+ /**
300
+ * Mutes / unmutes the microphone (implementations toggle the mic tracks' `enabled` flag —
301
+ * the transport stays up; the provider just receives silence while muted).
302
+ *
303
+ * @param muted `true` to mute the mic, `false` to unmute.
304
+ */
305
+ abstract SetMuted(muted: boolean): void;
306
+ /**
307
+ * Tears down the provider connection and all client-held resources (control channel,
308
+ * transport, mic tracks, audio sink) and emits a final `'closed'` state (unless the
309
+ * session already ended in `'error'`). Safe to call more than once.
310
+ */
311
+ abstract Disconnect(): Promise<void>;
312
+ /**
313
+ * `true` while a model response is in flight (generation started and not yet done).
314
+ * Distinct from {@link IsAudioPlaying}: generation runs ahead of playback.
315
+ * Hosts use this to gate interim narration so it never interrupts a reply.
316
+ */
317
+ abstract get IsBusy(): boolean;
318
+ /**
319
+ * `true` while model audio is AUDIBLY playing in the browser. Distinct from
320
+ * {@link IsBusy} — audio plays at realtime while generation finishes early, so the
321
+ * model can be "idle" while speech is still coming out of the speaker. Hosts must
322
+ * gate narration on BOTH, or queued utterances come out late and stale.
323
+ */
324
+ abstract get IsAudioPlaying(): boolean;
325
+ /**
326
+ * Registers the (single) transcript handler. Receives interim deltas (`IsFinal: false`)
327
+ * and final turn transcripts (`IsFinal: true`) for both roles, tagged with their
328
+ * {@link RealtimeClientTranscript.Kind}.
329
+ */
330
+ OnTranscript(handler: (transcript: RealtimeClientTranscript) => void): void;
331
+ /**
332
+ * Registers the (single) tool-call handler. The host executes the tool and calls
333
+ * {@link SendToolResult} with the outcome.
334
+ */
335
+ OnToolCall(handler: (call: RealtimeClientToolCall) => void): void;
336
+ /**
337
+ * Registers the (single) state-change handler. States may be re-emitted (e.g. repeated
338
+ * `'listening'` on successive barge-ins); hosts should treat emissions as idempotent.
339
+ */
340
+ OnStateChange(handler: (state: RealtimeClientState) => void): void;
341
+ /**
342
+ * Registers the (single) error handler. Receives both fatal transport errors (which are
343
+ * also followed by an `'error'` state change) and non-fatal provider error frames.
344
+ */
345
+ OnError(handler: (error: RealtimeClientError) => void): void;
346
+ /**
347
+ * Registers the (single) interruption handler.
348
+ *
349
+ * **True barge-in only:** fires ONLY when USER INPUT CUT OFF ACTIVE MODEL OUTPUT — a
350
+ * model response in flight or audio audibly playing when the user took the floor. A
351
+ * user simply taking their normal turn while the model is idle is NOT an interruption,
352
+ * and drivers must not report it as one (e.g. a raw "speech started" frame must be
353
+ * gated on whether a response is actually active or audio is playing).
354
+ *
355
+ * On interruption, drivers must also flush locally-owned playback and report
356
+ * {@link IsAudioPlaying} `=== false` promptly (driver obligation #3). Hosts typically
357
+ * use this hook to abort in-flight delegated work so a stale result is never narrated
358
+ * into a conversation that has moved on.
359
+ */
360
+ OnInterruption(handler: () => void): void;
361
+ /**
362
+ * Registers the (single) usage handler.
363
+ *
364
+ * Emissions carry token **deltas** for the response/turn that just completed (see
365
+ * {@link RealtimeClientUsage} — deltas preferred; cumulative-only providers must convert
366
+ * in the driver). Hosts accumulate and relay/persist on their own cadence (e.g. the voice
367
+ * session service debounces a `RelayRealtimeUsage` mutation onto the co-agent prompt run).
368
+ *
369
+ * **Optional capability:** drivers whose provider exposes no usage telemetry simply never
370
+ * emit — registering a handler is always safe, but hosts must not assume emissions arrive.
371
+ * See {@link RealtimeClientUsage} for per-driver availability.
372
+ */
373
+ OnUsage(handler: (usage: RealtimeClientUsage) => void): void;
374
+ /**
375
+ * Registers the (single) remote-VIDEO handler — the model/avatar's video track for a VIDEO session
376
+ * (a talking-head the host renders, e.g. as the agent's tile). Invoked once the provider publishes
377
+ * its video track.
378
+ *
379
+ * **Optional capability:** audio-only drivers (the default) never emit — registering is always safe,
380
+ * but hosts must not assume a video track arrives. Video-capable drivers
381
+ * ({@link BaseRealtimeModel.SupportsVideo}) call {@link emitRemoteVideo} when the track is live.
382
+ *
383
+ * @param handler Invoked with the remote video `MediaStream` when it becomes available.
384
+ */
385
+ OnRemoteVideo(handler: (stream: MediaStream) => void): void;
386
+ /** Emits a transcript event to the registered handler (if any). */
387
+ protected emitTranscript(transcript: RealtimeClientTranscript): void;
388
+ /** Emits a tool-call request to the registered handler (if any). */
389
+ protected emitToolCall(call: RealtimeClientToolCall): void;
390
+ /** Emits a state change to the registered handler (if any). */
391
+ protected emitStateChange(state: RealtimeClientState): void;
392
+ /** Emits an error to the registered handler (if any). */
393
+ protected emitError(error: RealtimeClientError): void;
394
+ /** Emits a true barge-in interruption to the registered handler (if any). */
395
+ protected emitInterruption(): void;
396
+ /** Emits a usage update (token deltas for a completed response/turn) to the registered handler (if any). */
397
+ protected emitUsage(usage: RealtimeClientUsage): void;
398
+ /** Emits the model/avatar's remote video stream to the registered handler (video drivers only). */
399
+ protected emitRemoteVideo(stream: MediaStream): void;
400
+ }
401
+ //# sourceMappingURL=baseRealtimeClient.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"baseRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAA4B,MAAM,qBAAqB,CAAC;AAEpF;;;;;;;;;GASG;AACH,MAAM,WAAW,qBAAqB;IAClC,4EAA4E;IAC5E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,+EAA+E;IAC/E,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,6FAA6F;IAC7F,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC3B,sGAAsG;IACtG,UAAU,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,wBAAwB;IACrC,6CAA6C;IAC7C,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,gGAAgG;IAChG,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC7B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC;IACjB,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,mBAAmB;IAChC,sFAAsF;IACtF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oFAAoF;IACpF,GAAG,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,mBAAmB,GAAG,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE7G;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAChC,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AACH,8BAAsB,kBAAkB;IAEpC,OAAO,CAAC,iBAAiB,CAAC,CAAiD;IAC3E,OAAO,CAAC,eAAe,CAAC,CAAyC;IACjE,OAAO,CAAC,kBAAkB,CAAC,CAAuC;IAClE,OAAO,CAAC,YAAY,CAAC,CAAuC;IAC5D,OAAO,CAAC,mBAAmB,CAAC,CAAa;IACzC,OAAO,CAAC,YAAY,CAAC,CAAuC;IAC5D,OAAO,CAAC,kBAAkB,CAAC,CAAgC;IAG3D,sEAAsE;IACtE,OAAO,CAAC,eAAe,CAAoC;IAC3D,yEAAyE;IACzE,OAAO,CAAC,gBAAgB,CAAoC;IAE5D;;;;;OAKG;IACI,gBAAgB,IAAI,qBAAqB,GAAG,IAAI;IAYvD,8FAA8F;IAC9F,SAAS,CAAC,qBAAqB,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,GAAG,IAAI;IAKxE,2FAA2F;IAC3F,SAAS,CAAC,sBAAsB,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,GAAG,IAAI;IAKzE,wFAAwF;IACxF,SAAS,CAAC,gBAAgB,IAAI,IAAI;IAKlC;;;;;;;;;;;;;;;OAeG;aACa,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/H;;;;;;;;;;;;;;OAcG;aACa,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAE5C;;;;;;;;;OASG;aACa,oBAAoB,IAAI,IAAI;IAE5C;;;;;OAKG;aACa,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAEnD;;;;;;;;;;;;;;;OAeG;aACa,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAE/D;;;;;;;;OAQG;aACa,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAExE;;;;;OAKG;aACa,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAE9C;;;;OAIG;aACa,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAE3C;;;;OAIG;IACH,aAAoB,MAAM,IAAI,OAAO,CAAC;IAEtC;;;;;OAKG;IACH,aAAoB,cAAc,IAAI,OAAO,CAAC;IAI9C;;;;OAIG;IACI,YAAY,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,wBAAwB,KAAK,IAAI,GAAG,IAAI;IAIlF;;;OAGG;IACI,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,sBAAsB,KAAK,IAAI,GAAG,IAAI;IAIxE;;;OAGG;IACI,aAAa,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAIzE;;;OAGG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAInE;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIhD;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAInE;;;;;;;;;;OAUG;IACI,aAAa,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAMlE,mEAAmE;IACnE,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,wBAAwB,GAAG,IAAI;IAIpE,oEAAoE;IACpE,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,sBAAsB,GAAG,IAAI;IAI1D,+DAA+D;IAC/D,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAI3D,yDAAyD;IACzD,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAIrD,6EAA6E;IAC7E,SAAS,CAAC,gBAAgB,IAAI,IAAI;IAIlC,4GAA4G;IAC5G,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAIrD,mGAAmG;IACnG,SAAS,CAAC,eAAe,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;CAGvD"}