@memberjunction/ai-realtime-client 0.0.1 → 5.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +148 -28
  2. package/dist/audio/audioMeter.d.ts +101 -0
  3. package/dist/audio/audioMeter.d.ts.map +1 -0
  4. package/dist/audio/audioMeter.js +193 -0
  5. package/dist/audio/audioMeter.js.map +1 -0
  6. package/dist/audio/micCapture.d.ts +26 -0
  7. package/dist/audio/micCapture.d.ts.map +1 -0
  8. package/dist/audio/micCapture.js +69 -0
  9. package/dist/audio/micCapture.js.map +1 -0
  10. package/dist/audio/pcmPlayback.d.ts +73 -0
  11. package/dist/audio/pcmPlayback.d.ts.map +1 -0
  12. package/dist/audio/pcmPlayback.js +78 -0
  13. package/dist/audio/pcmPlayback.js.map +1 -0
  14. package/dist/audio/pcmUtils.d.ts +17 -0
  15. package/dist/audio/pcmUtils.d.ts.map +1 -0
  16. package/dist/audio/pcmUtils.js +46 -0
  17. package/dist/audio/pcmUtils.js.map +1 -0
  18. package/dist/drivers/assemblyAIRealtimeClient.d.ts +384 -0
  19. package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -0
  20. package/dist/drivers/assemblyAIRealtimeClient.js +732 -0
  21. package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -0
  22. package/dist/drivers/elevenLabsRealtimeClient.d.ts +362 -0
  23. package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -0
  24. package/dist/drivers/elevenLabsRealtimeClient.js +686 -0
  25. package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -0
  26. package/dist/drivers/geminiRealtimeClient.d.ts +406 -0
  27. package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -0
  28. package/dist/drivers/geminiRealtimeClient.js +675 -0
  29. package/dist/drivers/geminiRealtimeClient.js.map +1 -0
  30. package/dist/drivers/openAIRealtimeClient.d.ts +381 -0
  31. package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -0
  32. package/dist/drivers/openAIRealtimeClient.js +602 -0
  33. package/dist/drivers/openAIRealtimeClient.js.map +1 -0
  34. package/dist/drivers/xaiRealtimeClient.d.ts +430 -0
  35. package/dist/drivers/xaiRealtimeClient.d.ts.map +1 -0
  36. package/dist/drivers/xaiRealtimeClient.js +676 -0
  37. package/dist/drivers/xaiRealtimeClient.js.map +1 -0
  38. package/dist/generic/baseRealtimeClient.d.ts +382 -0
  39. package/dist/generic/baseRealtimeClient.d.ts.map +1 -0
  40. package/dist/generic/baseRealtimeClient.js +208 -0
  41. package/dist/generic/baseRealtimeClient.js.map +1 -0
  42. package/dist/index.d.ts +11 -0
  43. package/dist/index.d.ts.map +1 -0
  44. package/dist/index.js +11 -0
  45. package/dist/index.js.map +1 -0
  46. package/package.json +28 -7
@@ -0,0 +1 @@
1
+ {"version":3,"file":"geminiRealtimeClient.js","sourceRoot":"","sources":["../../src/drivers/geminiRealtimeClient.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EACH,WAAW,GASd,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,kBAAkB,EAAuB,MAAM,+BAA+B,CAAC;AACxF,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAwB,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAkB,MAAM,qBAAqB,CAAC;AAE1E,kFAAkF;AAElF,2EAA2E;AAC3E,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,qDAAqD;AACrD,MAAM,4BAA4B,GAAG,sBAAsB,CAAC;AAC5D,wEAAwE;AACxE,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAmExC,kFAAkF;AAElF;;;;;;GAMG;AACH,MAAM,OAAO,iBAAkB,SAAQ,mBAAmB;IACtD;QACI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACrC,CAAC;CACJ;AAED,kFAAkF;AAElF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAEI,IAAM,oBAAoB,GAA1B,MAAM,oBAAqB,SAAQ,kBAAkB;IAArD;;QACH,8EAA8E;QACtE,YAAO,GAAmC,IAAI,CAAC;QAC/C,cAAS,GAAuB,IAAI,CAAC;QACrC,eAAU,GAA6B,IAAI,CAAC;QAC5C,aAAQ,GAAgC,IAAI,CAAC;QAErD,8EAA8E;QAC9E,0EAA0E;QAClE,yBAAoB,GAAG,EAAE,CAAC;QAClC,wEAAwE;QAChE,oBAAe,GAAG,EAAE,CAAC;QAC7B,mFAAmF;QAC3E,mBAAc,GAAG,KAAK,CAAC;QAC/B,6FAA6F;QACrF,uBAAkB,GAA2B,QAAQ,CAAC;QAC9D,kFAAkF;QAC1E,gBAAW,GAAsB,EAAE,CAAC;QAC5C;;;WAGG;QACK,yBAAoB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzD;;;;;;;;;WASG;QACK,mBAAc,GAAG,KAAK,CAAC;QAC/B;;;;WAIG;QACK,iBAAY,GAAwB,QAAQ,CAAC;IAqkBzD,CAAC;IAnkBG,8EAA8E;IAE9E;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,MAAmC,EAAE,SAAsB;QAC5E,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC;YACzC,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,UAAU;YAClB,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC;YACzD,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC;YACpD,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;QAC1G,8EAA8E;QAC9E,kFAAkF;QAClF,gEAAgE;QAChE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,IAAI,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,UAAU;QACnB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACzB,CAAC;YAAC,MAAM,CAAC;gBACL,qBAAqB;YACzB,CAAC;YACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,6EAA6E;IAE7E;;;;;;;;;;;;;OAaG;IACI,QAAQ,CAAC,IAAY;QACxB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;IAC/E,CAAC;IAED;;;;;;;;;;;OAWG;IACI,oBAAoB;QACvB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,OAAO,CAAC,qCAAqC;QACjD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;;;;;;;;;;OAWG;IACI,eAAe,CAAC,IAAY;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE;YACnB,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;YACvG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC/B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;OAQG;IACI,mBAAmB,CAAC,YAAoB;QAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED;;;;;;;;;;OAUG;IACI,cAAc,CAAC,MAAc,EAAE,UAAkB;QACpD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACzD,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;IACjF,CAAC;IAED;;;;;OAKG;IACI,QAAQ,CAAC,KAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;QACtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACH,IAAW,cAAc;QACrB,OAAO,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,KAAK,CAAC;IAC7C,CAAC;IAED,6EAA6E;IAE7E;;;;;OAKG;IACO,KAAK,CAAC,kBAAkB,CAAC,IAA6B;QAC5D,MAAM,EAAE,GAAG,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QACpG,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE;gBACP,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,OAAO,EAAE,IAAI,CAAC,OAAO;aACxB;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;;;;;OAMG;IACO,KAAK,CAAC,gBAAgB,CAC5B,SAAsB,EACtB,UAAyC;QAEzC,OAAO,mBAAmB,CAAC,SAAS,EAAE,wBAAwB,EAAE,UAAU,CAAC,CAAC;IAChF,CAAC;IAED,0FAA0F;IAChF,cAAc;QACpB,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACnC,CAAC;IAED,8EAA8E;IAE9E;;;;;OAKG;IACK,kBAAkB,CAAC,MAAmC;QAC1D,MAAM,aAAa,GAAe,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,OAAO,aAAa,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACjG,MAAM,GAAG,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,UAAU,GACZ,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,GAAyB,CAAC,CAAC,CAAC,EAAE,CAAC;QACrG,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IACjC,CAAC;IAED,wFAAwF;IAChF,YAAY,CAAC,WAAmB;QACpC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC;YAC5B,KAAK,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,4BAA4B,EAAE;SACvE,CAAC,CAAC;IACP,CAAC;IAED,uEAAuE;IAC/D,oBAAoB,CAAC,KAAiB;QAC1C,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,gCAAgC,KAAK,CAAC,OAAO,IAAI,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,kFAAkF;IAC1E,oBAAoB;QACxB,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAClE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,8EAA8E;IAE9E;;;OAGG;IACK,mBAAmB,CAAC,OAA0B;QAClD,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACnB,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACK,mBAAmB,CAAC,aAA8D;QACtF,IAAI,CAAC,SAAS,CAAC;YACX,WAAW,EAAE,OAAO,aAAa,CAAC,gBAAgB,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;YAC5G,YAAY,EAAE,OAAO,aAAa,CAAC,kBAAkB,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;YACjH,GAAG,EAAE,aAAa;SACrB,CAAC,CAAC;IACP,CAAC;IAED,0FAA0F;IAClF,mBAAmB,CAAC,OAA0B;QAClD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;YAC7B,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAC9B,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACnE,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC9B,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACK,kBAAkB;QACtB,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED,uFAAuF;IAC/E,gBAAgB,CAAC,SAAkB;QACvC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC;YACnC,IAAI,IAAI,EAAE,CAAC;gBACP,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAC7B,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;YACtD,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,uBAAuB,CAAC,aAA4B;QACxD,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,eAAe,IAAI,aAAa,CAAC,IAAI,CAAC;YAC3C,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,4BAA4B,CAAC,aAA4B;QAC7D,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,oBAAoB,IAAI,aAAa,CAAC,IAAI,CAAC;YAChD,IAAI,CAAC,cAAc,CAAC;gBAChB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,IAAI,CAAC,kBAAkB;aAChC,CAAC,CAAC;QACP,CAAC;QACD,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,2BAA2B,EAAE,CAAC;QACvC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACK,mBAAmB,CAAC,aAAyC;QACjE,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/C,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QACpC,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,IAAI,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9G,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,kBAAkB;QACtB,IAAI,CAAC,2BAA2B,EAAE,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC,CAAC,4DAA4D;QACzF,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,qBAAqB;QACzB,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,0FAA0F;IAClF,sBAAsB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC;QAClC,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,CAAC;IACL,CAAC;IAED,0FAA0F;IAClF,2BAA2B;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,oBAAoB,CAAC;QACvC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACzG,CAAC;IACL,CAAC;IAED,8EAA8E;IAE9E;;;;;OAKG;IACK,YAAY,CAAC,IAAgB;QACjC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO;QACX,CAAC;QACD,IAAI,EAAE,CAAC;IACX,CAAC;IAED;;;OAGG;IACK,gBAAgB;QACpB,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YACtC,IAAI,EAAE,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,sBAAsB,CAAC,IAAY,EAAE,IAA4B,EAAE,YAAqB;QAC5F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,sFAAsF;QACtF,uFAAuF;QACvF,sFAAsF;QACtF,oFAAoF;QACpF,+EAA+E;QAC/E,qFAAqF;QACrF,oFAAoF;QACpF,OAAO,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI,YAAY,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACK,oBAAoB,CAAC,MAAc,EAAE,IAAY,EAAE,UAAkB;QACzE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,OAAO,CAAC,gBAAgB,CAAC;YACrB,iBAAiB,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,EAAE,CAAC;SACxF,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,CAAC,iBAAiB,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,MAAc;QAClC,IAAI,CAAC;YACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1E,OAAO,MAAoB,CAAC;YAChC,CAAC;YACD,OAAO,EAAE,MAAM,EAAE,MAAmB,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,8EAA8E;IAE9E,0EAA0E;IAClE,kBAAkB;QACtB,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,CAAC;IACtC,CAAC;IAED,4EAA4E;IACpE,QAAQ,CAAC,KAA0B;QACvC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;CACJ,CAAA;AA5mBY,oBAAoB;IADhC,aAAa,CAAC,kBAAkB,EAAE,QAAQ,CAAC;GAC/B,oBAAoB,CA4mBhC;;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB;IACpC,oEAAoE;AACxE,CAAC"}
@@ -0,0 +1,381 @@
1
+ import { ClientRealtimeSessionConfig, JSONObject } from '@memberjunction/ai';
2
+ import { BaseRealtimeClient } from '../generic/baseRealtimeClient.js';
3
+ /** Streaming delta of the assistant's spoken-text transcript.
4
+ * GA (gpt-realtime) emits `response.output_audio_transcript.delta`; the older beta
5
+ * emitted `response.audio_transcript.delta`. We accept both so captions populate
6
+ * regardless of the model generation. */
7
+ export interface OAIResponseAudioTranscriptDelta {
8
+ type: 'response.output_audio_transcript.delta' | 'response.audio_transcript.delta';
9
+ delta: string;
10
+ response_id?: string;
11
+ item_id?: string;
12
+ }
13
+ /** Final assistant transcript for a turn (GA or beta event name). */
14
+ export interface OAIResponseAudioTranscriptDone {
15
+ type: 'response.output_audio_transcript.done' | 'response.audio_transcript.done';
16
+ transcript: string;
17
+ response_id?: string;
18
+ item_id?: string;
19
+ }
20
+ /** Final transcription of the user's spoken input for a turn. */
21
+ export interface OAIInputAudioTranscriptionCompleted {
22
+ type: 'conversation.item.input_audio_transcription.completed';
23
+ transcript: string;
24
+ item_id?: string;
25
+ }
26
+ /** The model finished assembling a function (tool) call and wants it executed. */
27
+ export interface OAIFunctionCallArgumentsDone {
28
+ type: 'response.function_call_arguments.done';
29
+ call_id: string;
30
+ name: string;
31
+ /** JSON-encoded arguments. */
32
+ arguments: string;
33
+ }
34
+ /** The provider detected the user starting to speak (barge-in). */
35
+ export interface OAIInputAudioBufferSpeechStarted {
36
+ type: 'input_audio_buffer.speech_started';
37
+ }
38
+ /** A new response (turn) started — tracked so we never start a second overlapping response. */
39
+ export interface OAIResponseCreated {
40
+ type: 'response.created';
41
+ }
42
+ /**
43
+ * A full response (turn) completed — carries the GA usage payload for THIS response
44
+ * (`input_tokens` / `output_tokens` plus per-modality detail objects), i.e. per-response
45
+ * DELTAS, which is exactly the `OnUsage` contract shape.
46
+ */
47
+ export interface OAIResponseDone {
48
+ type: 'response.done';
49
+ response?: {
50
+ usage?: {
51
+ input_tokens?: number;
52
+ output_tokens?: number;
53
+ [detail: string]: unknown;
54
+ };
55
+ };
56
+ }
57
+ /**
58
+ * WebRTC-only playback events: the client audio buffer started/stopped PLAYING.
59
+ * Critical distinction from response.done (generation finished): audio plays at
60
+ * realtime while generation runs ahead, so the model can be "idle" while speech
61
+ * is still audibly coming out of the speaker.
62
+ */
63
+ export interface OAIOutputAudioBufferStarted {
64
+ type: 'output_audio_buffer.started';
65
+ }
66
+ export interface OAIOutputAudioBufferStopped {
67
+ type: 'output_audio_buffer.stopped' | 'output_audio_buffer.cleared';
68
+ }
69
+ /** Provider-side error frame. */
70
+ export interface OAIErrorEvent {
71
+ type: 'error';
72
+ error?: {
73
+ message?: string;
74
+ code?: string;
75
+ };
76
+ }
77
+ /** Events whose `type` we don't explicitly handle still parse to this shape. */
78
+ export interface OAIUnknownEvent {
79
+ type: string;
80
+ }
81
+ export type OpenAIRealtimeEvent = OAIResponseAudioTranscriptDelta | OAIResponseAudioTranscriptDone | OAIInputAudioTranscriptionCompleted | OAIFunctionCallArgumentsDone | OAIInputAudioBufferSpeechStarted | OAIResponseCreated | OAIResponseDone | OAIOutputAudioBufferStarted | OAIOutputAudioBufferStopped | OAIErrorEvent | OAIUnknownEvent;
82
+ /** Applies the server-built session config (instructions + tools) to the live session. */
83
+ export interface OAISessionUpdateEvent {
84
+ type: 'session.update';
85
+ session: JSONObject;
86
+ }
87
+ /** A user or system `message` conversation item. */
88
+ export interface OAIMessageItem {
89
+ type: 'message';
90
+ role: 'user' | 'system';
91
+ content: Array<{
92
+ type: 'input_text';
93
+ text: string;
94
+ }>;
95
+ }
96
+ /** The output of an executed function (tool) call, correlated by `call_id`. */
97
+ export interface OAIFunctionCallOutputItem {
98
+ type: 'function_call_output';
99
+ call_id: string;
100
+ output: string;
101
+ }
102
+ /** Creates a conversation item (message or tool output). */
103
+ export interface OAIConversationItemCreateEvent {
104
+ type: 'conversation.item.create';
105
+ item: OAIMessageItem | OAIFunctionCallOutputItem;
106
+ }
107
+ /** Asks the model to produce a response, optionally with one-off instructions. */
108
+ export interface OAIResponseCreateEvent {
109
+ type: 'response.create';
110
+ response?: {
111
+ instructions: string;
112
+ };
113
+ }
114
+ /** Cancels the model's in-flight response (generation stops; a response.done follows). */
115
+ export interface OAIResponseCancelEvent {
116
+ type: 'response.cancel';
117
+ }
118
+ /**
119
+ * WebRTC-only client event: stops + clears the provider-managed output audio buffer so
120
+ * already-generated speech stops playing immediately (the server replies with an
121
+ * `output_audio_buffer.cleared` frame). Sent on typed-message barge-in.
122
+ */
123
+ export interface OAIOutputAudioBufferClearEvent {
124
+ type: 'output_audio_buffer.clear';
125
+ }
126
+ export type OpenAIRealtimeClientEvent = OAISessionUpdateEvent | OAIConversationItemCreateEvent | OAIResponseCreateEvent | OAIResponseCancelEvent | OAIOutputAudioBufferClearEvent;
127
+ /** The subset of `RTCDataChannel` this client uses. */
128
+ export interface IRealtimeDataChannel {
129
+ /** The channel's connection state (`'open'` once usable). */
130
+ readonly readyState: RTCDataChannelState;
131
+ onopen: ((event: Event) => void) | null;
132
+ onmessage: ((event: MessageEvent) => void) | null;
133
+ onerror: ((event: Event) => void) | null;
134
+ onclose: ((event: Event) => void) | null;
135
+ send(data: string): void;
136
+ close(): void;
137
+ }
138
+ /** The subset of `RTCPeerConnection` this client uses. */
139
+ export interface IRealtimePeerConnection {
140
+ ontrack: ((event: RTCTrackEvent) => void) | null;
141
+ addTrack(track: MediaStreamTrack, stream: MediaStream): void;
142
+ createDataChannel(label: string): IRealtimeDataChannel;
143
+ createOffer(): Promise<RTCSessionDescriptionInit>;
144
+ setLocalDescription(description: RTCSessionDescriptionInit): Promise<void>;
145
+ setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void>;
146
+ close(): void;
147
+ }
148
+ /** The subset of the hidden `<audio>` sink element this client uses. */
149
+ export interface IRealtimeAudioSink {
150
+ srcObject: MediaProvider | null;
151
+ remove(): void;
152
+ }
153
+ /**
154
+ * OpenAI implementation of {@link BaseRealtimeClient}: a **browser-direct** WebRTC
155
+ * connection to OpenAI's Realtime API, authenticated with the server-minted ephemeral
156
+ * client secret.
157
+ *
158
+ * Registered with the ClassFactory under the key `'openai'` — the `Provider` string the
159
+ * server's `OpenAIRealtime` driver stamps on its `ClientRealtimeSessionConfig` — so hosts
160
+ * resolve it without referencing this class directly.
161
+ *
162
+ * Owns ALL OpenAI wire concerns extracted from the original voice-session orchestration:
163
+ * - WebRTC connect: mic tracks → peer connection, remote audio → hidden `<audio>` sink,
164
+ * the `'oai-events'` data channel, and the GA SDP handshake (see {@link performSdpHandshake}).
165
+ * - Event translation: GA + beta transcript event names, input transcription, tool calls,
166
+ * barge-in, playback-buffer events, and provider error frames.
167
+ * - The response state machine: `responseActive` set on `response.created` / cleared on
168
+ * `response.done`; tool-result `response.create`s queued while a response is in flight
169
+ * and flushed on `response.done` so the model ALWAYS voices delegated results.
170
+ * - Narration-kind tagging: {@link RequestSpokenUpdate} marks the NEXT response as
171
+ * `'narration'` so its transcripts are emitted with `Kind: 'narration'` (ephemeral).
172
+ * - Audible-playback tracking ({@link IsAudioPlaying}) from `output_audio_buffer.*` events.
173
+ */
174
+ export declare class OpenAIRealtimeClient extends BaseRealtimeClient {
175
+ private peerConnection;
176
+ /** Protected so test subclasses can inspect/inject; production code treats it as private. */
177
+ protected dataChannel: IRealtimeDataChannel | null;
178
+ private remoteAudioEl;
179
+ private micStream;
180
+ /**
181
+ * The server-built session config applied verbatim via `session.update` when the data
182
+ * channel opens. Protected so test subclasses can seed it without a full Connect.
183
+ */
184
+ protected sessionConfig: JSONObject | null;
185
+ /** Accumulates the in-flight assistant transcript across delta frames. */
186
+ private pendingAssistantText;
187
+ /** True while the model has a response in flight; gates narration + queues the tool result. */
188
+ private responseActive;
189
+ /** Set when a tool result is ready while a response is active; sent on the next response.done. */
190
+ private pendingResultResponse;
191
+ /**
192
+ * Set by {@link RequestSpokenUpdate} just before it sends its `response.create`, and
193
+ * CONSUMED by the very next `response.created` frame, which stamps
194
+ * {@link activeResponseKind} for that turn. Narration is only requested while the model
195
+ * is idle (hosts gate on {@link IsBusy}), so under normal ordering the next
196
+ * `response.created` is ours.
197
+ */
198
+ private pendingNarrationKind;
199
+ /**
200
+ * The kind of the response currently in flight. Event ordering (confirmed against the
201
+ * live API): `response.created` → transcript deltas → `*_audio_transcript.done` →
202
+ * `response.done`. The transcript-done frame therefore arrives while the kind is still
203
+ * set, letting {@link onAssistantDone} classify the turn; `response.done` then resets
204
+ * the kind to `'normal'`.
205
+ */
206
+ private activeResponseKind;
207
+ /**
208
+ * True while the model's audio is audibly PLAYING in the browser (WebRTC
209
+ * `output_audio_buffer` started/stopped). Distinct from {@link responseActive} —
210
+ * generation finishes ahead of playback.
211
+ */
212
+ private audioPlaying;
213
+ /**
214
+ * The client's own view of the session state — mirrors what {@link emitStateChange}
215
+ * last reported, EXCEPT after a tool call is emitted: the host typically shows its own
216
+ * busy state then, so the client silently leaves `'speaking'` (no emission) to preserve
217
+ * the host's indicator until the result reply starts (see {@link handleEvent}).
218
+ */
219
+ private currentState;
220
+ /**
221
+ * Opens the client-direct OpenAI Realtime WebRTC connection: mic tracks onto a peer
222
+ * connection, a hidden remote-audio sink, the `'oai-events'` data channel, and the SDP
223
+ * handshake. `config.SessionConfig` is applied verbatim via `session.update` once the
224
+ * data channel opens; the client reports `'listening'` at that point.
225
+ */
226
+ Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
227
+ /**
228
+ * Tears down the channel, peer connection, mic tracks, and audio sink, resets the
229
+ * response state machine, and emits a final `'closed'` (unless already `'error'`).
230
+ */
231
+ Disconnect(): Promise<void>;
232
+ /**
233
+ * Injects typed text as a user-role `message` conversation item, then triggers a reply
234
+ * through the SAME collision-safe path tool results use ({@link requestResultResponse}).
235
+ * No-op when the data channel isn't open.
236
+ *
237
+ * **SendText implies barge-in** (base-contract rule): an active spoken response is
238
+ * cancelled via {@link CancelActiveResponse} before the text is injected, so the typed
239
+ * turn takes the floor immediately instead of waiting behind stale speech. When nothing
240
+ * is active the cancel is a no-op and the reply triggers immediately.
241
+ */
242
+ SendText(text: string): void;
243
+ /**
244
+ * @inheritdoc
245
+ *
246
+ * Sends `response.cancel` (only when a response is actually in flight) and — when audio
247
+ * is audibly playing — the WebRTC-only `output_audio_buffer.clear` client event so
248
+ * already-generated speech stops coming out of the speaker immediately. Resets the local
249
+ * response state machine (active flag, narration kind, accumulated transcript) but
250
+ * PRESERVES any queued tool-result trigger: delegated work is never affected by a
251
+ * floor-control cancel, and the queued trigger still fires on the cancelled response's
252
+ * trailing `response.done`. No-op when idle or when the channel is not open.
253
+ */
254
+ CancelActiveResponse(): void;
255
+ /**
256
+ * Injects a system-role context item the model can draw on the next time it speaks,
257
+ * WITHOUT forcing a reply.
258
+ *
259
+ * NOTE: role must be 'system' — gpt-realtime rejects 'developer' items
260
+ * ("Developer messages are only supported for quicksilver sessions").
261
+ */
262
+ SendContextNote(text: string): void;
263
+ /**
264
+ * Triggers ONE short spoken update with the given instructions. Marks the upcoming
265
+ * response as `'narration'` (flag consumed by the next `response.created`) so its
266
+ * transcripts are emitted with `Kind: 'narration'` — ephemeral by contract. Sets
267
+ * {@link responseActive} eagerly so a tool result landing mid-narration queues
268
+ * instead of colliding.
269
+ *
270
+ * **Skips when busy** (base-contract collision rule — drivers MUST queue or skip):
271
+ * a `response.create` sent while a response is in flight would be rejected/garbled by
272
+ * the provider, and narration is disposable by contract, so the update is dropped with
273
+ * a debug log rather than queued to come out late and stale. Hosts SHOULD still gate
274
+ * on {@link IsBusy} / {@link IsAudioPlaying} for timing quality.
275
+ */
276
+ RequestSpokenUpdate(instructions: string): void;
277
+ /**
278
+ * Sends the tool result back as a `function_call_output` conversation item, then
279
+ * triggers a reply — immediately if the model is idle, otherwise queued until the
280
+ * current response (e.g. a progress narration) finishes. Without the queueing the
281
+ * result's `response.create` would collide with an in-flight narration and be dropped,
282
+ * leaving the model silent when delegated work comes back.
283
+ */
284
+ SendToolResult(callID: string, outputJson: string): void;
285
+ /** Mute / unmute by toggling the mic tracks' `enabled` flag (transport stays up). */
286
+ SetMuted(muted: boolean): void;
287
+ /** @inheritdoc */
288
+ get IsBusy(): boolean;
289
+ /** @inheritdoc */
290
+ get IsAudioPlaying(): boolean;
291
+ /** Creates the peer connection. Production returns a real `RTCPeerConnection`. */
292
+ protected createPeerConnection(): IRealtimePeerConnection;
293
+ /** Creates a hidden `<audio>` element to play the model's audio output. */
294
+ protected createAudioSink(): IRealtimeAudioSink;
295
+ /**
296
+ * POSTs the raw SDP offer to OpenAI's Realtime WebRTC endpoint and returns the answer SDP.
297
+ *
298
+ * GA browser flow (confirmed against the OpenAI Realtime WebRTC guide): POST to
299
+ * `https://api.openai.com/v1/realtime/calls` with **no** query params and **no**
300
+ * `OpenAI-Beta` header. The ephemeral client secret already encodes the model + session
301
+ * config (set server-side at mint), so the browser must not specify the model — passing
302
+ * `?model=` returns an empty 400. The answer comes back as raw `application/sdp`.
303
+ *
304
+ * @param offerSdp The local SDP offer.
305
+ * @param ephemeralToken The server-minted ephemeral client secret.
306
+ * @returns The answer SDP.
307
+ */
308
+ protected postSdpOffer(offerSdp: string, ephemeralToken: string): Promise<string>;
309
+ /** Streams the mic tracks to the provider. */
310
+ private attachMicrophone;
311
+ /** Routes the provider's audio track into the hidden `<audio>` sink. */
312
+ private attachRemoteAudio;
313
+ /**
314
+ * Adopts + wires the events data channel: applies the session config and reports
315
+ * `'listening'` on open; translates inbound frames; reports transport errors / closure.
316
+ * Protected so test subclasses can inject a fake channel directly.
317
+ */
318
+ protected adoptDataChannel(channel: IRealtimeDataChannel): void;
319
+ /** Performs the offer/answer SDP exchange (POST seam: {@link postSdpOffer}). */
320
+ private performSdpHandshake;
321
+ /**
322
+ * Sends the server-controlled session config (instructions + tools) as a
323
+ * `session.update` so the co-agent's identity and tool set apply. Skipped when the
324
+ * host supplied no config (e.g. it failed to parse the server payload — the host
325
+ * already logged that; sending an EMPTY `session.update` would be wrong).
326
+ */
327
+ private applySessionConfig;
328
+ /** Parses an inbound channel message and dispatches it to the typed handler. */
329
+ private handleChannelMessage;
330
+ /** Dispatches a typed OpenAI realtime server event to the appropriate behavior. */
331
+ private handleEvent;
332
+ /** Appends an assistant transcript delta, reflects `'speaking'`, and emits the delta. */
333
+ private onAssistantDelta;
334
+ /**
335
+ * Finalizes the assistant turn: emits the final transcript tagged with the ACTIVE
336
+ * response kind (the transcript-done frame arrives BEFORE `response.done`, so
337
+ * {@link activeResponseKind} still reflects this turn), then returns to `'listening'`.
338
+ * Empty turns emit nothing.
339
+ */
340
+ private onAssistantDone;
341
+ /** Emits the final transcription of the user's spoken input (always `Kind: 'normal'`). */
342
+ private onUserTranscript;
343
+ /**
344
+ * Surfaces a completed tool call to the host. The client silently leaves the
345
+ * `'speaking'` state (NO emission) so a host-rendered busy indicator (e.g. "thinking")
346
+ * isn't clobbered by this turn's trailing `response.done` / playback-stopped frames —
347
+ * those only transition when the client still considers itself `'speaking'`.
348
+ */
349
+ private onToolCallFrame;
350
+ /**
351
+ * Emits the completed response's usage to the host as a DELTA (the GA `response.done`
352
+ * usage payload covers exactly this response, so it is already incremental — the
353
+ * `OnUsage` contract's preferred shape). Frames without a usage payload emit nothing.
354
+ */
355
+ private emitResponseUsage;
356
+ /** Surfaces a provider error frame (non-fatal; the session continues). */
357
+ private onErrorFrame;
358
+ /**
359
+ * Asks the model to speak (a tool result or typed-text reply) — immediately if it's
360
+ * idle, otherwise queued until the current response finishes. An immediate trigger
361
+ * also CONSUMES any queued trigger debt: every payload item is already in the
362
+ * conversation, so one `response.create` voices everything (e.g. typed text barging
363
+ * in over a narration that had tool results queued behind it).
364
+ */
365
+ private requestResultResponse;
366
+ /** On a turn completing, fire any queued tool-result response so the answer is spoken. */
367
+ private flushPendingResultResponse;
368
+ /** Resets the per-session response state machine (used on Disconnect). */
369
+ private resetResponseState;
370
+ /** Updates the client's own state view and emits the change to the host. */
371
+ private setState;
372
+ /** Serializes + sends a client event over the data channel (only when open). */
373
+ private sendEvent;
374
+ }
375
+ /**
376
+ * Tree-shaking prevention: bundlers cannot see that {@link OpenAIRealtimeClient} is
377
+ * instantiated dynamically through the ClassFactory, so a consumer must call this no-op
378
+ * to create a static code path that keeps the `@RegisterClass` side effect alive.
379
+ */
380
+ export declare function LoadOpenAIRealtimeClient(): void;
381
+ //# sourceMappingURL=openAIRealtimeClient.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openAIRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/openAIRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EACH,kBAAkB,EAErB,MAAM,+BAA+B,CAAC;AAMvC;;;0CAG0C;AAC1C,MAAM,WAAW,+BAA+B;IAC5C,IAAI,EAAE,wCAAwC,GAAG,iCAAiC,CAAC;IACnF,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qEAAqE;AACrE,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,uCAAuC,GAAG,gCAAgC,CAAC;IACjF,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iEAAiE;AACjE,MAAM,WAAW,mCAAmC;IAChD,IAAI,EAAE,uDAAuD,CAAC;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kFAAkF;AAClF,MAAM,WAAW,4BAA4B;IACzC,IAAI,EAAE,uCAAuC,CAAC;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,mEAAmE;AACnE,MAAM,WAAW,gCAAgC;IAC7C,IAAI,EAAE,mCAAmC,CAAC;CAC7C;AAED,+FAA+F;AAC/F,MAAM,WAAW,kBAAkB;IAC/B,IAAI,EAAE,kBAAkB,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE;YAAE,YAAY,CAAC,EAAE,MAAM,CAAC;YAAC,aAAa,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,CAAA;KAAE,CAAC;CACvG;AAED;;;;;GAKG;AACH,MAAM,WAAW,2BAA2B;IACxC,IAAI,EAAE,6BAA6B,CAAC;CACvC;AACD,MAAM,WAAW,2BAA2B;IACxC,IAAI,EAAE,6BAA6B,GAAG,6BAA6B,CAAC;CACvE;AAED,iCAAiC;AACjC,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/C;AAED,gFAAgF;AAChF,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,mBAAmB,GACzB,+BAA+B,GAC/B,8BAA8B,GAC9B,mCAAmC,GACnC,4BAA4B,GAC5B,gCAAgC,GAChC,kBAAkB,GAClB,eAAe,GACf,2BAA2B,GAC3B,2BAA2B,GAC3B,aAAa,GACb,eAAe,CAAC;AAItB,0FAA0F;AAC1F,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,gBAAgB,CAAC;IACvB,OAAO,EAAE,UAAU,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,YAAY,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,+EAA+E;AAC/E,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,4DAA4D;AAC5D,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,0BAA0B,CAAC;IACjC,IAAI,EAAE,cAAc,GAAG,yBAAyB,CAAC;CACpD;AAED,kFAAkF;AAClF,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,iBAAiB,CAAC;IACxB,QAAQ,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC;AAED,0FAA0F;AAC1F,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,iBAAiB,CAAC;CAC3B;AAED;;;;GAIG;AACH,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,2BAA2B,CAAC;CACrC;AAED,MAAM,MAAM,yBAAyB,GAC/B,qBAAqB,GACrB,8BAA8B,GAC9B,sBAAsB,GACtB,sBAAsB,GACtB,8BAA8B,CAAC;AAQrC,uDAAuD;AACvD,MAAM,WAAW,oBAAoB;IACjC,6DAA6D;IAC7D,QAAQ,CAAC,UAAU,EAAE,mBAAmB,CAAC;IACzC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IACxC,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IACzC,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,IAAI,IAAI,CAAC;CACjB;AAED,0DAA0D;AAC1D,MAAM,WAAW,uBAAuB;IACpC,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC7D,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,oBAAoB,CAAC;IACvD,WAAW,IAAI,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAClD,mBAAmB,CAAC,WAAW,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3E,oBAAoB,CAAC,WAAW,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,KAAK,IAAI,IAAI,CAAC;CACjB;AAED,wEAAwE;AACxE,MAAM,WAAW,kBAAkB;IAC/B,SAAS,EAAE,aAAa,GAAG,IAAI,CAAC;IAChC,MAAM,IAAI,IAAI,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBACa,oBAAqB,SAAQ,kBAAkB;IAExD,OAAO,CAAC,cAAc,CAAwC;IAC9D,6FAA6F;IAC7F,SAAS,CAAC,WAAW,EAAE,oBAAoB,GAAG,IAAI,CAAQ;IAC1D,OAAO,CAAC,aAAa,CAAmC;IACxD,OAAO,CAAC,SAAS,CAA4B;IAC7C;;;OAGG;IACH,SAAS,CAAC,aAAa,EAAE,UAAU,GAAG,IAAI,CAAQ;IAGlD,0EAA0E;IAC1E,OAAO,CAAC,oBAAoB,CAAM;IAClC,+FAA+F;IAC/F,OAAO,CAAC,cAAc,CAAS;IAC/B,kGAAkG;IAClG,OAAO,CAAC,qBAAqB,CAAS;IACtC;;;;;;OAMG;IACH,OAAO,CAAC,oBAAoB,CAAS;IACrC;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB,CAAoC;IAC9D;;;;OAIG;IACH,OAAO,CAAC,YAAY,CAAS;IAC7B;;;;;OAKG;IACH,OAAO,CAAC,YAAY,CAAiC;IAIrD;;;;;OAKG;IACU,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBhG;;;OAGG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA4BxC;;;;;;;;;OASG;IACI,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAiBnC;;;;;;;;;;OAUG;IACI,oBAAoB,IAAI,IAAI;IAwBnC;;;;;;OAMG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAe1C;;;;;;;;;;;;OAYG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAiBtD;;;;;;OAMG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAgB/D,qFAAqF;IAC9E,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAOrC,kBAAkB;IAClB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED,kBAAkB;IAClB,IAAW,cAAc,IAAI,OAAO,CAEnC;IAID,kFAAkF;IAClF,SAAS,CAAC,oBAAoB,IAAI,uBAAuB;IAIzD,2EAA2E;IAC3E,SAAS,CAAC,eAAe,IAAI,kBAAkB;IAQ/C;;;;;;;;;;;;OAYG;cACa,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBvF,8CAA8C;IAC9C,OAAO,CAAC,gBAAgB;IAMxB,wEAAwE;IACxE,OAAO,CAAC,iBAAiB;IAYzB;;;;OAIG;IACH,SAAS,CAAC,gBAAgB,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI;IAoB/D,gFAAgF;YAClE,mBAAmB;IAOjC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAS1B,gFAAgF;IAChF,OAAO,CAAC,oBAAoB;IAa5B,mFAAmF;IACnF,OAAO,CAAC,WAAW;IAqEnB,yFAAyF;IACzF,OAAO,CAAC,gBAAgB;IAQxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAWvB,0FAA0F;IAC1F,OAAO,CAAC,gBAAgB;IAIxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAYzB,0EAA0E;IAC1E,OAAO,CAAC,YAAY;IAUpB;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;IAc7B,0FAA0F;IAC1F,OAAO,CAAC,0BAA0B;IAUlC,0EAA0E;IAC1E,OAAO,CAAC,kBAAkB;IAW1B,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;IAKhB,gFAAgF;IAChF,OAAO,CAAC,SAAS;CAKpB;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,IAAI,IAAI,CAE/C"}