@memberjunction/ai-realtime-client 0.0.1 → 5.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +148 -28
  2. package/dist/audio/audioMeter.d.ts +101 -0
  3. package/dist/audio/audioMeter.d.ts.map +1 -0
  4. package/dist/audio/audioMeter.js +193 -0
  5. package/dist/audio/audioMeter.js.map +1 -0
  6. package/dist/audio/micCapture.d.ts +26 -0
  7. package/dist/audio/micCapture.d.ts.map +1 -0
  8. package/dist/audio/micCapture.js +69 -0
  9. package/dist/audio/micCapture.js.map +1 -0
  10. package/dist/audio/pcmPlayback.d.ts +73 -0
  11. package/dist/audio/pcmPlayback.d.ts.map +1 -0
  12. package/dist/audio/pcmPlayback.js +78 -0
  13. package/dist/audio/pcmPlayback.js.map +1 -0
  14. package/dist/audio/pcmUtils.d.ts +17 -0
  15. package/dist/audio/pcmUtils.d.ts.map +1 -0
  16. package/dist/audio/pcmUtils.js +46 -0
  17. package/dist/audio/pcmUtils.js.map +1 -0
  18. package/dist/drivers/assemblyAIRealtimeClient.d.ts +384 -0
  19. package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -0
  20. package/dist/drivers/assemblyAIRealtimeClient.js +732 -0
  21. package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -0
  22. package/dist/drivers/elevenLabsRealtimeClient.d.ts +362 -0
  23. package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -0
  24. package/dist/drivers/elevenLabsRealtimeClient.js +686 -0
  25. package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -0
  26. package/dist/drivers/geminiRealtimeClient.d.ts +406 -0
  27. package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -0
  28. package/dist/drivers/geminiRealtimeClient.js +675 -0
  29. package/dist/drivers/geminiRealtimeClient.js.map +1 -0
  30. package/dist/drivers/openAIRealtimeClient.d.ts +381 -0
  31. package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -0
  32. package/dist/drivers/openAIRealtimeClient.js +602 -0
  33. package/dist/drivers/openAIRealtimeClient.js.map +1 -0
  34. package/dist/drivers/xaiRealtimeClient.d.ts +430 -0
  35. package/dist/drivers/xaiRealtimeClient.d.ts.map +1 -0
  36. package/dist/drivers/xaiRealtimeClient.js +676 -0
  37. package/dist/drivers/xaiRealtimeClient.js.map +1 -0
  38. package/dist/generic/baseRealtimeClient.d.ts +382 -0
  39. package/dist/generic/baseRealtimeClient.d.ts.map +1 -0
  40. package/dist/generic/baseRealtimeClient.js +208 -0
  41. package/dist/generic/baseRealtimeClient.js.map +1 -0
  42. package/dist/index.d.ts +11 -0
  43. package/dist/index.d.ts.map +1 -0
  44. package/dist/index.js +11 -0
  45. package/dist/index.js.map +1 -0
  46. package/package.json +28 -7
@@ -0,0 +1 @@
1
+ {"version":3,"file":"xaiRealtimeClient.js","sourceRoot":"","sources":["../../src/drivers/xaiRealtimeClient.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAuB,MAAM,+BAA+B,CAAC;AACxF,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAwB,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAE1E,kFAAkF;AAElF;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,4BAA4B,CAAC;AAEhE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,oBAAoB,CAAC;AAEzE;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CAAC;AAuLzC,kFAAkF;AAElF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEI,IAAM,iBAAiB,GAAvB,MAAM,iBAAkB,SAAQ,kBAAkB;IAAlD;;QACH,8EAA8E;QACtE,WAAM,GAA4B,IAAI,CAAC;QACvC,cAAS,GAAuB,IAAI,CAAC;QACrC,eAAU,GAA0B,IAAI,CAAC;QACzC,aAAQ,GAAgC,IAAI,CAAC;QACrD;;;WAGG;QACO,kBAAa,GAAsB,IAAI,CAAC;QAElD,8EAA8E;QAC9E;;;;;WAKG;QACK,wBAAmB,GAAG,KAAK,CAAC;QACpC,0EAA0E;QAClE,yBAAoB,GAAG,EAAE,CAAC;QAClC,+FAA+F;QACvF,mBAAc,GAAG,KAAK,CAAC;QAC/B,kGAAkG;QAC1F,0BAAqB,GAAG,KAAK,CAAC;QACtC;;;;WAIG;QACK,yBAAoB,GAAG,KAAK,CAAC;QACrC;;;;;WAKG;QACK,uBAAkB,GAA2B,QAAQ,CAAC;QAC9D;;;;;WAKG;QACK,iBAAY,GAAwB,QAAQ,CAAC;QACrD,qFAAqF;QAC7E,qBAAgB,GAAG,KAAK,CAAC;IAolBrC,CAAC;IAllBG,8EAA8E;IAE9E;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,MAAmC,EAAE,SAAsB;QAC5E,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE5B,IAAI,UAAU,GAAwB,IAAI,CAAC;QAC3C,IAAI,QAAQ,GAAoC,IAAI,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACjD,UAAU,GAAG,OAAO,CAAC;YACrB,QAAQ,GAAG,MAAM,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,mBAAmB,UAAU,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/E,MAAM,WAAW,GAAG,GAAG,oCAAoC,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QACtF,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC;QACrC,MAAM,CAAC,SAAS,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,CAAC,OAAO,GAAG,CAAC,OAAO,EAAE,EAAE;YACzB,QAAQ,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC,CAAC;QACF,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;YAClB,QAAQ,EAAE,CAAC,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;YACrE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7B,CAAC,CAAC;QAEF,MAAM,MAAM,CAAC;QACb,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC3B,sFAAsF;QACtF,wFAAwF;QACxF,uFAAuF;QACvF,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;QACzD,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,mBAAmB,EAAE,CAAC,WAAW,EAAE,EAAE,CAC1F,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CACjC,CAAC;QACF,uFAAuF;QACvF,qFAAqF;QACrF,oDAAoD;QACpD,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,IAAI,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,UAAU;QACnB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACL,qBAAqB;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,6EAA6E;IAE7E;;;;;;;;;OASG;IACI,QAAQ,CAAC,IAAY;QACxB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,CAAC;YACX,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACF,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aAC1C;SACJ,CAAC,CAAC;QACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACjC,CAAC;IAED;;;;;;;;;OASG;IACI,oBAAoB;QACvB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,OAAO,CAAC,qCAAqC;QACjD,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;YAC5C,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;YAC5B,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;YAClC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACnC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QACnC,CAAC;QACD,wFAAwF;QACxF,kFAAkF;QAClF,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACI,eAAe,CAAC,IAAY;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC;YACX,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACF,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aAC1C;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACI,mBAAmB,CAAC,YAAoB;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,8GAA8G,CAAC,CAAC;YAC9H,OAAO;QACX,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED;;;;;;OAMG;IACI,cAAc,CAAC,MAAc,EAAE,UAAkB;QACpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC;YACX,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACF,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,UAAU;aACrB;SACJ,CAAC,CAAC;QACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACI,QAAQ,CAAC,KAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;QACtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACH,IAAW,cAAc;QACrB,OAAO,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,KAAK,CAAC;IAC7C,CAAC;IAED,6EAA6E;IAE7E;;;;;;OAMG;IACO,YAAY,CAAC,GAAW,EAAE,WAAmB;QACnD,MAAM,EAAE,GAAI,UAAiH,CAAC,SAAS,CAAC;QACxI,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;QAC5F,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACpC,MAAM,IAAI,GAAqB;YAC3B,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;YAC7B,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE;SAC1B,CAAC;QACF,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,EAAE,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,gCAAgC,CAAC,CAAC;QACpE,EAAE,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAC5B,SAAsB,EACtB,UAAkB,EAClB,UAAyC;QAEzC,OAAO,mBAAmB,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAClE,CAAC;IAED;;;OAGG;IACO,cAAc,CAAC,UAAkB;QACvC,OAAO,IAAI,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC;IAED,8EAA8E;IAE9E;;;;;OAKG;IACK,kBAAkB;QACtB,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtE,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,kFAAkF;IAC1E,YAAY,CAAC,WAAmB;QACpC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QAC9E,CAAC;IACL,CAAC;IAED,oFAAoF;IAC5E,iBAAiB,CAAC,OAAe;QACrC,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAClE,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,4CAA4C,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAChG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACK,iBAAiB;QACrB,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAC3F,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,wDAAwD,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,8EAA8E;IAE9E,kEAAkE;IAC1D,mBAAmB,CAAC,IAAY;QACpC,IAAI,KAAuB,CAAC;QAC5B,IAAI,CAAC;YACD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAqB,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3F,OAAO,CAAC,0BAA0B;QACtC,CAAC;QACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9C,OAAO,CAAC,sEAAsE;QAClF,CAAC;QACD,+FAA+F;QAC/F,8FAA8F;QAC9F,8FAA8F;QAC9F,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,IAAI,EACtD,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,gFAAgF;IACxE,WAAW,CAAC,KAAuB;QACvC,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,wCAAwC,CAAC;YAC9C,KAAK,iCAAiC;gBAClC,IAAI,CAAC,gBAAgB,CAAE,KAAyC,CAAC,KAAK,CAAC,CAAC;gBACxE,MAAM;YACV,KAAK,uCAAuC,CAAC;YAC7C,KAAK,gCAAgC;gBACjC,IAAI,CAAC,eAAe,CAAE,KAAwC,CAAC,UAAU,CAAC,CAAC;gBAC3E,MAAM;YACV,KAAK,sBAAsB,CAAC;YAC5B,KAAK,6BAA6B;gBAC9B,IAAI,CAAC,YAAY,CAAE,KAA+B,CAAC,KAAK,CAAC,CAAC;gBAC1D,MAAM;YACV,KAAK,uDAAuD;gBACxD,IAAI,CAAC,gBAAgB,CAAE,KAA6C,CAAC,UAAU,CAAC,CAAC;gBACjF,MAAM;YACV,KAAK,uCAAuC;gBACxC,IAAI,CAAC,eAAe,CAAC,KAAqC,CAAC,CAAC;gBAC5D,MAAM;YACV,KAAK,mCAAmC;gBACpC,IAAI,CAAC,eAAe,EAAE,CAAC;gBACvB,MAAM;YACV,KAAK,kBAAkB;gBACnB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,6EAA6E;gBAC7E,8EAA8E;gBAC9E,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC7E,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;gBAClC,MAAM;YACV,KAAK,eAAe;gBAChB,6EAA6E;gBAC7E,qEAAqE;gBACrE,uEAAuE;gBACvE,gEAAgE;gBAChE,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;gBACnC,IAAI,CAAC,iBAAiB,CAAC,KAAwB,CAAC,CAAC;gBACjD,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAClC,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;oBACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC/B,CAAC;gBACD,MAAM;YACV,KAAK,OAAO;gBACR,IAAI,CAAC,YAAY,CAAC,KAAsB,CAAC,CAAC;gBAC1C,MAAM;YACV;gBACI,uEAAuE;gBACvE,MAAM;QACd,CAAC;IACL,CAAC;IAED,yFAAyF;IACjF,gBAAgB,CAAC,KAAa;QAClC,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,oBAAoB,IAAI,KAAK,CAAC;QACnC,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC3G,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,UAAkB;QACtC,MAAM,SAAS,GAAG,UAAU,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAC1D,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAC/B,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC9G,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,YAAY,CAAC,WAAmB;QACpC,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED;;;;;;OAMG;IACK,gBAAgB,CAAC,UAAkB;QACvC,IAAI,UAAU,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,cAAc,CAAC;gBAChB,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ;gBAC7D,gBAAgB,EAAE,IAAI,CAAC,mBAAmB;aAC7C,CAAC,CAAC;YACH,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QACpC,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACK,eAAe,CAAC,IAAkC;QACtD,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QACpC,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IACpG,CAAC;IAED;;;;;;;OAOG;IACK,eAAe;QACnB,gGAAgG;QAChG,qFAAqF;QACrF,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACK,iBAAiB,CAAC,KAAsB;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC;YACX,WAAW,EAAE,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;YACpF,YAAY,EAAE,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;YACvF,GAAG,EAAE,KAAK;SACb,CAAC,CAAC;IACP,CAAC;IAED,0EAA0E;IAClE,YAAY,CAAC,KAAoB;QACrC,IAAI,CAAC,SAAS,CAAC;YACX,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,IAAI,wBAAwB;YACzD,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI;YACvB,KAAK,EAAE,KAAK;SACf,CAAC,CAAC;IACP,CAAC;IAED,8EAA8E;IAE9E;;;;;;OAMG;IACK,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;YAClC,OAAO;QACX,CAAC;QACD,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,0FAA0F;IAClF,0BAA0B;QAC9B,IAAI,CAAC,IAAI,CAAC,qBAAqB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9C,OAAO;QACX,CAAC;QACD,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,0EAA0E;IAClE,kBAAkB;QACtB,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;QACnC,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;QAClC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAED,8EAA8E;IAE9E,4EAA4E;IACpE,QAAQ,CAAC,KAA0B;QACvC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,+EAA+E;IACvE,SAAS,CAAC,KAA6B;QAC3C,4FAA4F;QAC5F,4FAA4F;QAC5F,uCAAuC;QACvC,IAAI,KAAK,CAAC,IAAI,KAAK,2BAA2B,EAAE,CAAC;YAC7C,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7C,CAAC;CACJ,CAAA;AAnoBY,iBAAiB;IAD7B,aAAa,CAAC,kBAAkB,EAAE,KAAK,CAAC;GAC5B,iBAAiB,CAmoB7B;;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB;IACjC,oEAAoE;AACxE,CAAC"}
@@ -0,0 +1,382 @@
1
+ import { ClientRealtimeSessionConfig } from '@memberjunction/ai';
2
+ import { IRealtimeAudioMeter } from '../audio/audioMeter.js';
3
+ /**
4
+ * A point-in-time snapshot of the session's audible activity, sampled by the host
5
+ * (typically per animation frame) via {@link BaseRealtimeClient.GetAudioActivity} to drive
6
+ * audio-reactive call visuals — the orb that vibrates with the agent's voice and the
7
+ * true-spectrum EQ.
8
+ *
9
+ * CAPABILITY-SHAPED, like usage telemetry: every member is `null` when that direction has
10
+ * no meter. Hosts must degrade gracefully — levels-without-bins renders envelope breathing,
11
+ * full `null` falls back to turn-state-driven animation.
12
+ */
13
+ export interface RealtimeAudioActivity {
14
+ /** RMS level (0..1) of the USER's microphone, or `null` when un-metered. */
15
+ InputLevel: number | null;
16
+ /** RMS level (0..1) of the AGENT's audio output, or `null` when un-metered. */
17
+ OutputLevel: number | null;
18
+ /** Mic frequency spectrum as {@link REALTIME_AUDIO_BIN_COUNT} normalized bins, or `null`. */
19
+ InputBins: number[] | null;
20
+ /** Agent-output frequency spectrum as {@link REALTIME_AUDIO_BIN_COUNT} normalized bins, or `null`. */
21
+ OutputBins: number[] | null;
22
+ }
23
+ /**
24
+ * A transcript event emitted by a realtime client for either side of the conversation.
25
+ *
26
+ * `Kind` distinguishes NORMAL conversation turns from NARRATION turns — the brief,
27
+ * ephemeral spoken updates a host requests via {@link BaseRealtimeClient.RequestSpokenUpdate}
28
+ * while background (delegated) work runs. By product decision narration transcripts are
29
+ * NOT captions and are never persisted; hosts surface them as a transient "live note".
30
+ */
31
+ export interface RealtimeClientTranscript {
32
+ /** Whose turn this transcript belongs to. */
33
+ Role: 'User' | 'Assistant';
34
+ /**
35
+ * The transcribed text. For `IsFinal: false` events this is the incremental DELTA
36
+ * for the in-flight turn; for `IsFinal: true` it is the complete turn text.
37
+ */
38
+ Text: string;
39
+ /** Whether this is the final transcript for the turn (`true`) or an interim delta (`false`). */
40
+ IsFinal: boolean;
41
+ /**
42
+ * `'normal'` for a regular conversation turn; `'narration'` for an ephemeral
43
+ * spoken-progress update triggered by {@link BaseRealtimeClient.RequestSpokenUpdate}.
44
+ */
45
+ Kind: 'normal' | 'narration';
46
+ /**
47
+ * MACHINE-READABLE correction marker: `true` when this FINAL transcript SUPERSEDES the
48
+ * session's immediately-preceding final transcript of the same role — e.g. ElevenLabs'
49
+ * post-barge-in `agent_response_correction`, which re-finalizes the assistant turn with
50
+ * the text that was ACTUALLY spoken before the interruption cut it off. Hosts that
51
+ * persist finals must UPDATE the previous turn in place instead of appending a
52
+ * duplicate. Absent/`false` on ordinary turns and on interim deltas.
53
+ */
54
+ ReplacesPrevious?: boolean;
55
+ }
56
+ /**
57
+ * A tool-call request emitted by the realtime model. The host executes the tool and feeds
58
+ * the result back via {@link BaseRealtimeClient.SendToolResult}, correlated by `CallID`.
59
+ */
60
+ export interface RealtimeClientToolCall {
61
+ /** Provider-assigned identifier for this call, used to correlate the eventual tool result. */
62
+ CallID: string;
63
+ /** The name of the tool the model is requesting to invoke. */
64
+ ToolName: string;
65
+ /** The arguments for the call as a JSON string, exactly as the provider emitted them. */
66
+ ArgumentsJson: string;
67
+ }
68
+ /**
69
+ * A usage/telemetry update emitted by a realtime client.
70
+ *
71
+ * **Deltas preferred:** each emission SHOULD carry the incremental token counts for the
72
+ * response/turn that just completed (not a session-cumulative running total), matching the
73
+ * server-side `RealtimeUsage` contract — consumers accumulate. A driver whose provider only
74
+ * reports cumulative totals must convert to deltas before emitting.
75
+ *
76
+ * **Per-driver availability** (capability, not laziness — providers without usage events
77
+ * simply never emit):
78
+ * - **OpenAI** — emits per-response deltas from the GA `response.done` frame's `usage` payload.
79
+ * - **Gemini** — emits per-turn deltas from `LiveServerMessage.usageMetadata`
80
+ * (`promptTokenCount` / `responseTokenCount` are per-response counts, not cumulative).
81
+ * - **ElevenLabs** — the Conversational AI socket exposes no usage events; never emits.
82
+ * - **AssemblyAI** — the streaming STT socket exposes no token-usage events; never emits.
83
+ */
84
+ export interface RealtimeClientUsage {
85
+ /** Input tokens reported in this update (a delta for the completed response/turn). */
86
+ InputTokens?: number;
87
+ /** Output tokens reported in this update (a delta for the completed response/turn). */
88
+ OutputTokens?: number;
89
+ /** The raw provider usage payload, for hosts that want provider-specific detail. */
90
+ Raw?: unknown;
91
+ }
92
+ /**
93
+ * Connection / turn state of a realtime client session.
94
+ * - `connecting` — the provider connection is being negotiated
95
+ * - `connected` — the transport is up but the control channel is not yet open
96
+ * - `listening` — control channel open; mic live; waiting on / hearing the user
97
+ * - `speaking` — the model is producing (or about to produce) audible output
98
+ * - `closed` — the session has been torn down
99
+ * - `error` — a fatal transport error occurred; the session is no longer usable
100
+ *
101
+ * Note there is deliberately NO `thinking` state here: "the host is executing a tool" is
102
+ * host policy, not wire state. Hosts that want a busy indicator set it themselves between
103
+ * receiving a tool call and calling {@link BaseRealtimeClient.SendToolResult}.
104
+ */
105
+ export type RealtimeClientState = 'connecting' | 'connected' | 'listening' | 'speaking' | 'closed' | 'error';
106
+ /**
107
+ * An error surfaced by a realtime client. `Fatal: true` means the session is unusable
108
+ * (transport failure); `Fatal: false` is a provider-reported, recoverable error frame.
109
+ */
110
+ export interface RealtimeClientError {
111
+ /** Human-readable error message. */
112
+ Message: string;
113
+ /** Optional provider-specific error code. */
114
+ Code?: string;
115
+ /** Whether the error terminated the session. */
116
+ Fatal: boolean;
117
+ }
118
+ /**
119
+ * Abstract base class for **browser-side, provider-direct** realtime (voice) clients.
120
+ *
121
+ * This is the client-side mirror of the server's `BaseRealtimeModel` pattern: the server
122
+ * mints an ephemeral credential + session config (`ClientRealtimeSessionConfig`) via its
123
+ * provider driver, and the browser resolves the matching CLIENT driver through the
124
+ * MemberJunction `ClassFactory` using the config's `Provider` string as the registration
125
+ * key (e.g. `'openai'`):
126
+ *
127
+ * ```typescript
128
+ * const client = MJGlobal.Instance.ClassFactory.CreateInstance<BaseRealtimeClient>(
129
+ * BaseRealtimeClient, startResult.Provider);
130
+ * ```
131
+ *
132
+ * Concrete drivers (e.g. {@link OpenAIRealtimeClient}) own ALL provider wire concerns:
133
+ * transport (WebRTC/WebSocket), event-name translation, the response state machine
134
+ * (never letting a tool-result reply collide with an in-flight response), narration-kind
135
+ * tagging, and audible-playback tracking. Hosts own POLICY: when to narrate, what
136
+ * instructions to speak, transcript persistence, and UI state.
137
+ *
138
+ * All `On*` methods register a single handler (matching the server `IRealtimeSession`
139
+ * style); registering again replaces the previous handler.
140
+ *
141
+ * ## DRIVER AUTHOR OBLIGATIONS
142
+ *
143
+ * Hard-won, provider-independent rules every client driver MUST honor (the client-side
144
+ * mirror of the obligations documented on the server's `BaseRealtimeModel` in
145
+ * `@memberjunction/ai`). Most were paid for in live debugging; do not relearn them:
146
+ *
147
+ * 1. **Silent exit from `'speaking'` after a tool call.** When the model emits a tool
148
+ * call, the host typically shows its own busy ("thinking") indicator while the tool
149
+ * executes. The driver must leave the `'speaking'` state *silently* (no state
150
+ * emission) at tool-call time so the turn's trailing frames don't clobber the host's
151
+ * indicator.
152
+ * 2. **Busy-flag release on tool-call emission (deadlock guard).** A tool-call frame
153
+ * means the model has yielded the floor pending the result. Any internal "response
154
+ * active" flag MUST be cleared then — otherwise the eventual {@link SendToolResult}
155
+ * (or a queued send) deadlocks waiting for a turn boundary that will never arrive
156
+ * until after the result is sent.
157
+ * 3. **Playback flush + honest {@link IsAudioPlaying} on interruption.** On a true
158
+ * barge-in the driver must flush any locally-owned audio playback and report
159
+ * `IsAudioPlaying === false` promptly — stale queued audio after an interruption is
160
+ * a product bug, not a nicety. The same flush applies to {@link CancelActiveResponse}.
161
+ * 4. **{@link SendText} must NOT echo a user transcript — and implies barge-in.** The
162
+ * driver must not synthesize a user-role transcript event for injected text (the host
163
+ * owns the local echo and would render the message twice), and an active spoken
164
+ * response is cancelled via the driver's own cancel path before the text is injected
165
+ * (see {@link CancelActiveResponse}).
166
+ * 5. **Tool-result delivery invariant.** Every result fed back via {@link SendToolResult}
167
+ * must EVENTUALLY be voiced/processed by the model and never dropped. If the provider
168
+ * rejects overlapping generation triggers, the driver queues the result's trigger
169
+ * behind the in-flight response and flushes it at the next turn boundary. The result
170
+ * must also EXPLICITLY trigger generation on providers that don't auto-continue after
171
+ * a tool response: OpenAI requires an explicit `response.create`; Gemini auto-continues
172
+ * UNLESS prior client content (a context note's `turnComplete: false`) left the turn
173
+ * open, in which case the driver must commit the turn or the model stays silent.
174
+ * 6. **Token/credential expiry surfaces as a FATAL error.** When the ephemeral credential
175
+ * dies (expiry, revocation, unexpected socket loss), the driver must surface it through
176
+ * {@link OnError} with `Fatal: true` so the host tears down cleanly instead of idling
177
+ * forever on a dead connection.
178
+ * 7. **`'listening'` only after the session config is applied.** The driver must not
179
+ * report the session as listening until the server-built session config (system prompt
180
+ * + tools) has actually been applied to the provider socket — otherwise early turns run
181
+ * against an unconfigured model.
182
+ * 8. **`SessionConfig` is a private pact.** The `ClientRealtimeSessionConfig.SessionConfig`
183
+ * payload is authored by the same-keyed SERVER driver and consumed only by this client
184
+ * driver; its shape may change between the two driver halves without notice. Hosts and
185
+ * intermediaries treat it as an opaque blob — and so must any code outside the matching
186
+ * driver pair.
187
+ * 9. **Audio metering is a CAPABILITY, not an obligation — but wire what you own.** A
188
+ * driver that owns (or can tap) an audio plane should attach
189
+ * {@link BaseRealtimeClient.attachInputAudioMeter} / {@link attachOutputAudioMeter} so
190
+ * {@link GetAudioActivity} reports real levels: the mic stream everywhere
191
+ * (`RealtimeAudioMeter.ForStream`), the shared `RealtimePcmPlayback.CreateMeter()` on
192
+ * client-owned-audio drivers, the remote WebRTC stream on peer-connection drivers.
193
+ * Meters must be released on disconnect ({@link closeAudioMeters}). A driver with no
194
+ * tappable plane simply attaches nothing — hosts fall back to turn-state animation.
195
+ */
196
+ export declare abstract class BaseRealtimeClient {
197
+ private transcriptHandler?;
198
+ private toolCallHandler?;
199
+ private stateChangeHandler?;
200
+ private errorHandler?;
201
+ private interruptionHandler?;
202
+ private usageHandler?;
203
+ /** Meter over the USER's microphone, when the driver attached one. */
204
+ private inputAudioMeter;
205
+ /** Meter over the AGENT's audio output, when the driver attached one. */
206
+ private outputAudioMeter;
207
+ /**
208
+ * The session's current audible activity, or `null` when this driver attached no
209
+ * meters at all (hosts then keep their turn-state-driven visuals). Sampled by hosts
210
+ * per animation frame — implementations are cheap, allocation-light reads of an
211
+ * `AnalyserNode`; no per-call provider traffic.
212
+ */
213
+ GetAudioActivity(): RealtimeAudioActivity | null;
214
+ /** Attaches (replacing + closing any previous) the USER-microphone meter. `null` detaches. */
215
+ protected attachInputAudioMeter(meter: IRealtimeAudioMeter | null): void;
216
+ /** Attaches (replacing + closing any previous) the AGENT-output meter. `null` detaches. */
217
+ protected attachOutputAudioMeter(meter: IRealtimeAudioMeter | null): void;
218
+ /** Releases both meters — every driver calls this from its disconnect/teardown path. */
219
+ protected closeAudioMeters(): void;
220
+ /**
221
+ * Opens the provider connection using the server-minted ephemeral credential and applies
222
+ * `config.SessionConfig` once the control channel is ready — so prompt and tool authority
223
+ * stay server-side even though the browser owns the socket. The payload's SHAPE is a
224
+ * private pact between this driver and the same-keyed server driver that minted it (see
225
+ * driver obligation #8); how it is applied is entirely driver-specific.
226
+ *
227
+ * @param config The server-minted client session config (provider, model, ephemeral token, session config).
228
+ * @param micStream The user's microphone capture stream. The caller acquires it (so IT owns
229
+ * the permission prompt UX); the client attaches it to the transport and stops its tracks
230
+ * on {@link Disconnect}.
231
+ */
232
+ abstract Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
233
+ /**
234
+ * Injects typed text into the live session as a USER turn and asks the model to respond.
235
+ * Implementations must route the reply through the same collision-safe path as tool
236
+ * results so it never collides with an in-flight response. No-op when the control
237
+ * channel is not open.
238
+ *
239
+ * **SendText implies barge-in:** an active spoken response is cancelled (via the
240
+ * driver's own {@link CancelActiveResponse} path) before the text is injected, so the
241
+ * typed turn takes the floor immediately instead of waiting behind stale speech.
242
+ *
243
+ * Implementations must NOT synthesize a user-role transcript event for the injected
244
+ * text — the host owns the local echo (driver obligation #4).
245
+ *
246
+ * @param text The user's typed message (callers pass pre-trimmed, non-empty text).
247
+ */
248
+ abstract SendText(text: string): void;
249
+ /**
250
+ * Cancels the model's ACTIVE spoken response and flushes pending playback so a new user
251
+ * turn can take the floor immediately; no-op when nothing is active. MUST NOT affect
252
+ * server-side delegated work — cancelling speech is a floor-control action, not an abort
253
+ * of in-flight tools/agents (hosts abort delegated work from {@link OnInterruption} /
254
+ * their own policy, never from this call).
255
+ *
256
+ * Implementations must leave {@link IsBusy} and {@link IsAudioPlaying} honest after the
257
+ * cancel (response inactive, local playback flushed).
258
+ */
259
+ abstract CancelActiveResponse(): void;
260
+ /**
261
+ * Injects background context into the model's conversation WITHOUT forcing a reply —
262
+ * e.g. delegated-run progress the model can draw on the next time it speaks.
263
+ *
264
+ * @param text The context note text (the host owns any prefixing/formatting policy).
265
+ */
266
+ abstract SendContextNote(text: string): void;
267
+ /**
268
+ * Asks the model for ONE brief interim utterance (e.g. a spoken progress update while
269
+ * delegated work runs). Implementations MUST tag the resulting turn's transcripts with
270
+ * `Kind: 'narration'` and MUST NOT let this response collide with a pending tool-result
271
+ * reply — the tool result is queued and spoken as soon as the narration finishes.
272
+ *
273
+ * **Collision rule — drivers MUST queue or skip:** when a model response is already in
274
+ * flight, the driver must either defer the update to the next turn boundary or skip it
275
+ * outright (skipping is acceptable — narration is disposable by contract; a stale
276
+ * "still working…" line has no value once the real result lands). Hosts SHOULD still
277
+ * gate calls on {@link IsBusy} / {@link IsAudioPlaying} for timing quality — a
278
+ * well-timed update beats a queued-then-stale one — but the driver is the safety net.
279
+ *
280
+ * @param instructions The exact provider instructions for the utterance. The instruction
281
+ * TEXT is host policy (e.g. first-person phrasing rules) — the client only carries it.
282
+ */
283
+ abstract RequestSpokenUpdate(instructions: string): void;
284
+ /**
285
+ * Feeds an executed tool's result back to the model, correlated by `callID`, and ensures
286
+ * the model SPEAKS the result as soon as possible: immediately when idle, otherwise
287
+ * queued behind the in-flight response (e.g. a progress narration) so the trigger is
288
+ * never dropped by the provider.
289
+ *
290
+ * @param callID The `CallID` from the originating {@link RealtimeClientToolCall}.
291
+ * @param outputJson The tool's result as a JSON-stringified string.
292
+ */
293
+ abstract SendToolResult(callID: string, outputJson: string): void;
294
+ /**
295
+ * Mutes / unmutes the microphone (implementations toggle the mic tracks' `enabled` flag —
296
+ * the transport stays up; the provider just receives silence while muted).
297
+ *
298
+ * @param muted `true` to mute the mic, `false` to unmute.
299
+ */
300
+ abstract SetMuted(muted: boolean): void;
301
+ /**
302
+ * Tears down the provider connection and all client-held resources (control channel,
303
+ * transport, mic tracks, audio sink) and emits a final `'closed'` state (unless the
304
+ * session already ended in `'error'`). Safe to call more than once.
305
+ */
306
+ abstract Disconnect(): Promise<void>;
307
+ /**
308
+ * `true` while a model response is in flight (generation started and not yet done).
309
+ * Distinct from {@link IsAudioPlaying}: generation runs ahead of playback.
310
+ * Hosts use this to gate interim narration so it never interrupts a reply.
311
+ */
312
+ abstract get IsBusy(): boolean;
313
+ /**
314
+ * `true` while model audio is AUDIBLY playing in the browser. Distinct from
315
+ * {@link IsBusy} — audio plays at realtime while generation finishes early, so the
316
+ * model can be "idle" while speech is still coming out of the speaker. Hosts must
317
+ * gate narration on BOTH, or queued utterances come out late and stale.
318
+ */
319
+ abstract get IsAudioPlaying(): boolean;
320
+ /**
321
+ * Registers the (single) transcript handler. Receives interim deltas (`IsFinal: false`)
322
+ * and final turn transcripts (`IsFinal: true`) for both roles, tagged with their
323
+ * {@link RealtimeClientTranscript.Kind}.
324
+ */
325
+ OnTranscript(handler: (transcript: RealtimeClientTranscript) => void): void;
326
+ /**
327
+ * Registers the (single) tool-call handler. The host executes the tool and calls
328
+ * {@link SendToolResult} with the outcome.
329
+ */
330
+ OnToolCall(handler: (call: RealtimeClientToolCall) => void): void;
331
+ /**
332
+ * Registers the (single) state-change handler. States may be re-emitted (e.g. repeated
333
+ * `'listening'` on successive barge-ins); hosts should treat emissions as idempotent.
334
+ */
335
+ OnStateChange(handler: (state: RealtimeClientState) => void): void;
336
+ /**
337
+ * Registers the (single) error handler. Receives both fatal transport errors (which are
338
+ * also followed by an `'error'` state change) and non-fatal provider error frames.
339
+ */
340
+ OnError(handler: (error: RealtimeClientError) => void): void;
341
+ /**
342
+ * Registers the (single) interruption handler.
343
+ *
344
+ * **True barge-in only:** fires ONLY when USER INPUT CUT OFF ACTIVE MODEL OUTPUT — a
345
+ * model response in flight or audio audibly playing when the user took the floor. A
346
+ * user simply taking their normal turn while the model is idle is NOT an interruption,
347
+ * and drivers must not report it as one (e.g. a raw "speech started" frame must be
348
+ * gated on whether a response is actually active or audio is playing).
349
+ *
350
+ * On interruption, drivers must also flush locally-owned playback and report
351
+ * {@link IsAudioPlaying} `=== false` promptly (driver obligation #3). Hosts typically
352
+ * use this hook to abort in-flight delegated work so a stale result is never narrated
353
+ * into a conversation that has moved on.
354
+ */
355
+ OnInterruption(handler: () => void): void;
356
+ /**
357
+ * Registers the (single) usage handler.
358
+ *
359
+ * Emissions carry token **deltas** for the response/turn that just completed (see
360
+ * {@link RealtimeClientUsage} — deltas preferred; cumulative-only providers must convert
361
+ * in the driver). Hosts accumulate and relay/persist on their own cadence (e.g. the voice
362
+ * session service debounces a `RelayRealtimeUsage` mutation onto the co-agent prompt run).
363
+ *
364
+ * **Optional capability:** drivers whose provider exposes no usage telemetry simply never
365
+ * emit — registering a handler is always safe, but hosts must not assume emissions arrive.
366
+ * See {@link RealtimeClientUsage} for per-driver availability.
367
+ */
368
+ OnUsage(handler: (usage: RealtimeClientUsage) => void): void;
369
+ /** Emits a transcript event to the registered handler (if any). */
370
+ protected emitTranscript(transcript: RealtimeClientTranscript): void;
371
+ /** Emits a tool-call request to the registered handler (if any). */
372
+ protected emitToolCall(call: RealtimeClientToolCall): void;
373
+ /** Emits a state change to the registered handler (if any). */
374
+ protected emitStateChange(state: RealtimeClientState): void;
375
+ /** Emits an error to the registered handler (if any). */
376
+ protected emitError(error: RealtimeClientError): void;
377
+ /** Emits a true barge-in interruption to the registered handler (if any). */
378
+ protected emitInterruption(): void;
379
+ /** Emits a usage update (token deltas for a completed response/turn) to the registered handler (if any). */
380
+ protected emitUsage(usage: RealtimeClientUsage): void;
381
+ }
382
+ //# sourceMappingURL=baseRealtimeClient.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"baseRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAA4B,MAAM,qBAAqB,CAAC;AAEpF;;;;;;;;;GASG;AACH,MAAM,WAAW,qBAAqB;IAClC,4EAA4E;IAC5E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,+EAA+E;IAC/E,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,6FAA6F;IAC7F,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC3B,sGAAsG;IACtG,UAAU,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,wBAAwB;IACrC,6CAA6C;IAC7C,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,gGAAgG;IAChG,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC7B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC;IACjB,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,mBAAmB;IAChC,sFAAsF;IACtF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oFAAoF;IACpF,GAAG,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,mBAAmB,GAAG,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE7G;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAChC,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AACH,8BAAsB,kBAAkB;IAEpC,OAAO,CAAC,iBAAiB,CAAC,CAAiD;IAC3E,OAAO,CAAC,eAAe,CAAC,CAAyC;IACjE,OAAO,CAAC,kBAAkB,CAAC,CAAuC;IAClE,OAAO,CAAC,YAAY,CAAC,CAAuC;IAC5D,OAAO,CAAC,mBAAmB,CAAC,CAAa;IACzC,OAAO,CAAC,YAAY,CAAC,CAAuC;IAG5D,sEAAsE;IACtE,OAAO,CAAC,eAAe,CAAoC;IAC3D,yEAAyE;IACzE,OAAO,CAAC,gBAAgB,CAAoC;IAE5D;;;;;OAKG;IACI,gBAAgB,IAAI,qBAAqB,GAAG,IAAI;IAYvD,8FAA8F;IAC9F,SAAS,CAAC,qBAAqB,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,GAAG,IAAI;IAKxE,2FAA2F;IAC3F,SAAS,CAAC,sBAAsB,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,GAAG,IAAI;IAKzE,wFAAwF;IACxF,SAAS,CAAC,gBAAgB,IAAI,IAAI;IAKlC;;;;;;;;;;;OAWG;aACa,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnG;;;;;;;;;;;;;;OAcG;aACa,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAE5C;;;;;;;;;OASG;aACa,oBAAoB,IAAI,IAAI;IAE5C;;;;;OAKG;aACa,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAEnD;;;;;;;;;;;;;;;OAeG;aACa,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAE/D;;;;;;;;OAQG;aACa,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAExE;;;;;OAKG;aACa,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAE9C;;;;OAIG;aACa,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAE3C;;;;OAIG;IACH,aAAoB,MAAM,IAAI,OAAO,CAAC;IAEtC;;;;;OAKG;IACH,aAAoB,cAAc,IAAI,OAAO,CAAC;IAI9C;;;;OAIG;IACI,YAAY,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,wBAAwB,KAAK,IAAI,GAAG,IAAI;IAIlF;;;OAGG;IACI,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,sBAAsB,KAAK,IAAI,GAAG,IAAI;IAIxE;;;OAGG;IACI,aAAa,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAIzE;;;OAGG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAInE;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIhD;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAMnE,mEAAmE;IACnE,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,wBAAwB,GAAG,IAAI;IAIpE,oEAAoE;IACpE,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,sBAAsB,GAAG,IAAI;IAI1D,+DAA+D;IAC/D,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAI3D,yDAAyD;IACzD,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAIrD,6EAA6E;IAC7E,SAAS,CAAC,gBAAgB,IAAI,IAAI;IAIlC,4GAA4G;IAC5G,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;CAGxD"}
@@ -0,0 +1,208 @@
1
+ import { REALTIME_AUDIO_BIN_COUNT } from '../audio/audioMeter.js';
2
+ /**
3
+ * Abstract base class for **browser-side, provider-direct** realtime (voice) clients.
4
+ *
5
+ * This is the client-side mirror of the server's `BaseRealtimeModel` pattern: the server
6
+ * mints an ephemeral credential + session config (`ClientRealtimeSessionConfig`) via its
7
+ * provider driver, and the browser resolves the matching CLIENT driver through the
8
+ * MemberJunction `ClassFactory` using the config's `Provider` string as the registration
9
+ * key (e.g. `'openai'`):
10
+ *
11
+ * ```typescript
12
+ * const client = MJGlobal.Instance.ClassFactory.CreateInstance<BaseRealtimeClient>(
13
+ * BaseRealtimeClient, startResult.Provider);
14
+ * ```
15
+ *
16
+ * Concrete drivers (e.g. {@link OpenAIRealtimeClient}) own ALL provider wire concerns:
17
+ * transport (WebRTC/WebSocket), event-name translation, the response state machine
18
+ * (never letting a tool-result reply collide with an in-flight response), narration-kind
19
+ * tagging, and audible-playback tracking. Hosts own POLICY: when to narrate, what
20
+ * instructions to speak, transcript persistence, and UI state.
21
+ *
22
+ * All `On*` methods register a single handler (matching the server `IRealtimeSession`
23
+ * style); registering again replaces the previous handler.
24
+ *
25
+ * ## DRIVER AUTHOR OBLIGATIONS
26
+ *
27
+ * Hard-won, provider-independent rules every client driver MUST honor (the client-side
28
+ * mirror of the obligations documented on the server's `BaseRealtimeModel` in
29
+ * `@memberjunction/ai`). Most were paid for in live debugging; do not relearn them:
30
+ *
31
+ * 1. **Silent exit from `'speaking'` after a tool call.** When the model emits a tool
32
+ * call, the host typically shows its own busy ("thinking") indicator while the tool
33
+ * executes. The driver must leave the `'speaking'` state *silently* (no state
34
+ * emission) at tool-call time so the turn's trailing frames don't clobber the host's
35
+ * indicator.
36
+ * 2. **Busy-flag release on tool-call emission (deadlock guard).** A tool-call frame
37
+ * means the model has yielded the floor pending the result. Any internal "response
38
+ * active" flag MUST be cleared then — otherwise the eventual {@link SendToolResult}
39
+ * (or a queued send) deadlocks waiting for a turn boundary that will never arrive
40
+ * until after the result is sent.
41
+ * 3. **Playback flush + honest {@link IsAudioPlaying} on interruption.** On a true
42
+ * barge-in the driver must flush any locally-owned audio playback and report
43
+ * `IsAudioPlaying === false` promptly — stale queued audio after an interruption is
44
+ * a product bug, not a nicety. The same flush applies to {@link CancelActiveResponse}.
45
+ * 4. **{@link SendText} must NOT echo a user transcript — and implies barge-in.** The
46
+ * driver must not synthesize a user-role transcript event for injected text (the host
47
+ * owns the local echo and would render the message twice), and an active spoken
48
+ * response is cancelled via the driver's own cancel path before the text is injected
49
+ * (see {@link CancelActiveResponse}).
50
+ * 5. **Tool-result delivery invariant.** Every result fed back via {@link SendToolResult}
51
+ * must EVENTUALLY be voiced/processed by the model and never dropped. If the provider
52
+ * rejects overlapping generation triggers, the driver queues the result's trigger
53
+ * behind the in-flight response and flushes it at the next turn boundary. The result
54
+ * must also EXPLICITLY trigger generation on providers that don't auto-continue after
55
+ * a tool response: OpenAI requires an explicit `response.create`; Gemini auto-continues
56
+ * UNLESS prior client content (a context note's `turnComplete: false`) left the turn
57
+ * open, in which case the driver must commit the turn or the model stays silent.
58
+ * 6. **Token/credential expiry surfaces as a FATAL error.** When the ephemeral credential
59
+ * dies (expiry, revocation, unexpected socket loss), the driver must surface it through
60
+ * {@link OnError} with `Fatal: true` so the host tears down cleanly instead of idling
61
+ * forever on a dead connection.
62
+ * 7. **`'listening'` only after the session config is applied.** The driver must not
63
+ * report the session as listening until the server-built session config (system prompt
64
+ * + tools) has actually been applied to the provider socket — otherwise early turns run
65
+ * against an unconfigured model.
66
+ * 8. **`SessionConfig` is a private pact.** The `ClientRealtimeSessionConfig.SessionConfig`
67
+ * payload is authored by the same-keyed SERVER driver and consumed only by this client
68
+ * driver; its shape may change between the two driver halves without notice. Hosts and
69
+ * intermediaries treat it as an opaque blob — and so must any code outside the matching
70
+ * driver pair.
71
+ * 9. **Audio metering is a CAPABILITY, not an obligation — but wire what you own.** A
72
+ * driver that owns (or can tap) an audio plane should attach
73
+ * {@link BaseRealtimeClient.attachInputAudioMeter} / {@link attachOutputAudioMeter} so
74
+ * {@link GetAudioActivity} reports real levels: the mic stream everywhere
75
+ * (`RealtimeAudioMeter.ForStream`), the shared `RealtimePcmPlayback.CreateMeter()` on
76
+ * client-owned-audio drivers, the remote WebRTC stream on peer-connection drivers.
77
+ * Meters must be released on disconnect ({@link closeAudioMeters}). A driver with no
78
+ * tappable plane simply attaches nothing — hosts fall back to turn-state animation.
79
+ */
80
+ export class BaseRealtimeClient {
81
+ constructor() {
82
+ // ── Audio-activity metering (capability surface — see driver obligation #9) ─
83
+ /** Meter over the USER's microphone, when the driver attached one. */
84
+ this.inputAudioMeter = null;
85
+ /** Meter over the AGENT's audio output, when the driver attached one. */
86
+ this.outputAudioMeter = null;
87
+ }
88
+ /**
89
+ * The session's current audible activity, or `null` when this driver attached no
90
+ * meters at all (hosts then keep their turn-state-driven visuals). Sampled by hosts
91
+ * per animation frame — implementations are cheap, allocation-light reads of an
92
+ * `AnalyserNode`; no per-call provider traffic.
93
+ */
94
+ GetAudioActivity() {
95
+ if (!this.inputAudioMeter && !this.outputAudioMeter) {
96
+ return null;
97
+ }
98
+ return {
99
+ InputLevel: this.inputAudioMeter ? this.inputAudioMeter.Level() : null,
100
+ OutputLevel: this.outputAudioMeter ? this.outputAudioMeter.Level() : null,
101
+ InputBins: this.inputAudioMeter ? this.inputAudioMeter.Bins(REALTIME_AUDIO_BIN_COUNT) : null,
102
+ OutputBins: this.outputAudioMeter ? this.outputAudioMeter.Bins(REALTIME_AUDIO_BIN_COUNT) : null
103
+ };
104
+ }
105
+ /** Attaches (replacing + closing any previous) the USER-microphone meter. `null` detaches. */
106
+ attachInputAudioMeter(meter) {
107
+ this.inputAudioMeter?.Close();
108
+ this.inputAudioMeter = meter;
109
+ }
110
+ /** Attaches (replacing + closing any previous) the AGENT-output meter. `null` detaches. */
111
+ attachOutputAudioMeter(meter) {
112
+ this.outputAudioMeter?.Close();
113
+ this.outputAudioMeter = meter;
114
+ }
115
+ /** Releases both meters — every driver calls this from its disconnect/teardown path. */
116
+ closeAudioMeters() {
117
+ this.attachInputAudioMeter(null);
118
+ this.attachOutputAudioMeter(null);
119
+ }
120
+ // ── Handler registration ──────────────────────────────────────────────────
121
+ /**
122
+ * Registers the (single) transcript handler. Receives interim deltas (`IsFinal: false`)
123
+ * and final turn transcripts (`IsFinal: true`) for both roles, tagged with their
124
+ * {@link RealtimeClientTranscript.Kind}.
125
+ */
126
+ OnTranscript(handler) {
127
+ this.transcriptHandler = handler;
128
+ }
129
+ /**
130
+ * Registers the (single) tool-call handler. The host executes the tool and calls
131
+ * {@link SendToolResult} with the outcome.
132
+ */
133
+ OnToolCall(handler) {
134
+ this.toolCallHandler = handler;
135
+ }
136
+ /**
137
+ * Registers the (single) state-change handler. States may be re-emitted (e.g. repeated
138
+ * `'listening'` on successive barge-ins); hosts should treat emissions as idempotent.
139
+ */
140
+ OnStateChange(handler) {
141
+ this.stateChangeHandler = handler;
142
+ }
143
+ /**
144
+ * Registers the (single) error handler. Receives both fatal transport errors (which are
145
+ * also followed by an `'error'` state change) and non-fatal provider error frames.
146
+ */
147
+ OnError(handler) {
148
+ this.errorHandler = handler;
149
+ }
150
+ /**
151
+ * Registers the (single) interruption handler.
152
+ *
153
+ * **True barge-in only:** fires ONLY when USER INPUT CUT OFF ACTIVE MODEL OUTPUT — a
154
+ * model response in flight or audio audibly playing when the user took the floor. A
155
+ * user simply taking their normal turn while the model is idle is NOT an interruption,
156
+ * and drivers must not report it as one (e.g. a raw "speech started" frame must be
157
+ * gated on whether a response is actually active or audio is playing).
158
+ *
159
+ * On interruption, drivers must also flush locally-owned playback and report
160
+ * {@link IsAudioPlaying} `=== false` promptly (driver obligation #3). Hosts typically
161
+ * use this hook to abort in-flight delegated work so a stale result is never narrated
162
+ * into a conversation that has moved on.
163
+ */
164
+ OnInterruption(handler) {
165
+ this.interruptionHandler = handler;
166
+ }
167
+ /**
168
+ * Registers the (single) usage handler.
169
+ *
170
+ * Emissions carry token **deltas** for the response/turn that just completed (see
171
+ * {@link RealtimeClientUsage} — deltas preferred; cumulative-only providers must convert
172
+ * in the driver). Hosts accumulate and relay/persist on their own cadence (e.g. the voice
173
+ * session service debounces a `RelayRealtimeUsage` mutation onto the co-agent prompt run).
174
+ *
175
+ * **Optional capability:** drivers whose provider exposes no usage telemetry simply never
176
+ * emit — registering a handler is always safe, but hosts must not assume emissions arrive.
177
+ * See {@link RealtimeClientUsage} for per-driver availability.
178
+ */
179
+ OnUsage(handler) {
180
+ this.usageHandler = handler;
181
+ }
182
+ // ── Protected emit helpers for concrete drivers ───────────────────────────
183
+ /** Emits a transcript event to the registered handler (if any). */
184
+ emitTranscript(transcript) {
185
+ this.transcriptHandler?.(transcript);
186
+ }
187
+ /** Emits a tool-call request to the registered handler (if any). */
188
+ emitToolCall(call) {
189
+ this.toolCallHandler?.(call);
190
+ }
191
+ /** Emits a state change to the registered handler (if any). */
192
+ emitStateChange(state) {
193
+ this.stateChangeHandler?.(state);
194
+ }
195
+ /** Emits an error to the registered handler (if any). */
196
+ emitError(error) {
197
+ this.errorHandler?.(error);
198
+ }
199
+ /** Emits a true barge-in interruption to the registered handler (if any). */
200
+ emitInterruption() {
201
+ this.interruptionHandler?.();
202
+ }
203
+ /** Emits a usage update (token deltas for a completed response/turn) to the registered handler (if any). */
204
+ emitUsage(usage) {
205
+ this.usageHandler?.(usage);
206
+ }
207
+ }
208
+ //# sourceMappingURL=baseRealtimeClient.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"baseRealtimeClient.js","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AA2HpF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AACH,MAAM,OAAgB,kBAAkB;IAAxC;QASI,+EAA+E;QAC/E,sEAAsE;QAC9D,oBAAe,GAA+B,IAAI,CAAC;QAC3D,yEAAyE;QACjE,qBAAgB,GAA+B,IAAI,CAAC;IAwPhE,CAAC;IAtPG;;;;;OAKG;IACI,gBAAgB;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACtE,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACzE,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;YAC5F,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;SAClG,CAAC;IACN,CAAC;IAED,8FAA8F;IACpF,qBAAqB,CAAC,KAAiC;QAC7D,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED,2FAA2F;IACjF,sBAAsB,CAAC,KAAiC;QAC9D,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED,wFAAwF;IAC9E,gBAAgB;QACtB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IAgHD,6EAA6E;IAE7E;;;;OAIG;IACI,YAAY,CAAC,OAAuD;QACvE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC;IACrC,CAAC;IAED;;;OAGG;IACI,UAAU,CAAC,OAA+C;QAC7D,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;IACnC,CAAC;IAED;;;OAGG;IACI,aAAa,CAAC,OAA6C;QAC9D,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAmB;QACrC,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED,6EAA6E;IAE7E,mEAAmE;IACzD,cAAc,CAAC,UAAoC;QACzD,IAAI,CAAC,iBAAiB,EAAE,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;IAED,oEAAoE;IAC1D,YAAY,CAAC,IAA4B;QAC/C,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,+DAA+D;IACrD,eAAe,CAAC,KAA0B;QAChD,IAAI,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,yDAAyD;IAC/C,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,6EAA6E;IACnE,gBAAgB;QACtB,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,4GAA4G;IAClG,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;CACJ"}
@@ -0,0 +1,11 @@
1
+ export * from './generic/baseRealtimeClient.js';
2
+ export * from './audio/pcmUtils.js';
3
+ export * from './audio/audioMeter.js';
4
+ export * from './audio/pcmPlayback.js';
5
+ export * from './audio/micCapture.js';
6
+ export * from './drivers/openAIRealtimeClient.js';
7
+ export * from './drivers/geminiRealtimeClient.js';
8
+ export * from './drivers/elevenLabsRealtimeClient.js';
9
+ export * from './drivers/assemblyAIRealtimeClient.js';
10
+ export * from './drivers/xaiRealtimeClient.js';
11
+ //# sourceMappingURL=index.d.ts.map