@bddh/starling-realtime-client 2.0.0-beta.3 → 2.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cjs/Client.d.ts +13 -11
- package/cjs/Client.js +1 -1
- package/cjs/Server.d.ts +4 -2
- package/cjs/Server.js +1 -1
- package/cjs/interface.d.ts +2 -2
- package/es/Client.d.ts +13 -11
- package/es/Client.js +1 -1
- package/es/Server.d.ts +4 -2
- package/es/Server.js +1 -1
- package/es/interface.d.ts +2 -2
- package/package.json +2 -2
package/cjs/Client.d.ts
CHANGED
|
@@ -3,29 +3,31 @@
|
|
|
3
3
|
* @description 云渲染数字人sdk
|
|
4
4
|
* @author zhangyue49
|
|
5
5
|
*/
|
|
6
|
-
import { DhRealtimeHumanType, ConnectDataType } from './interface';
|
|
6
|
+
import { DhRealtimeHumanType, ConnectDataType, CallbackMsgType } from './interface';
|
|
7
7
|
import { DHServerInterface, RenderCallback } from './Server';
|
|
8
8
|
export default class DhRealtimeHuman {
|
|
9
|
-
private
|
|
10
|
-
private
|
|
11
|
-
private
|
|
12
|
-
private
|
|
13
|
-
private readonly onDigitalHumanCallback
|
|
9
|
+
private token;
|
|
10
|
+
private appKey;
|
|
11
|
+
private appId;
|
|
12
|
+
private connectParams;
|
|
13
|
+
private readonly onDigitalHumanCallback?;
|
|
14
14
|
private dhServer;
|
|
15
|
-
private rtcServer
|
|
15
|
+
private rtcServer?;
|
|
16
16
|
private rtcConnectParams;
|
|
17
|
-
private
|
|
17
|
+
private cutoutInstance?;
|
|
18
18
|
private readonly isHuawei;
|
|
19
|
-
private
|
|
20
|
-
private
|
|
19
|
+
private rtcServerUrl;
|
|
20
|
+
private recordController;
|
|
21
21
|
private wsReadyState;
|
|
22
22
|
constructor(props: DhRealtimeHumanType);
|
|
23
|
+
init(props: DhRealtimeHumanType): void;
|
|
24
|
+
onDigitalHumanCallbackPlus: (data: CallbackMsgType) => void;
|
|
23
25
|
onRemoteVideoComing: (id: number) => void;
|
|
24
26
|
onLocalStream: () => void;
|
|
25
27
|
onRemoteVideoOn: () => void;
|
|
26
28
|
connectListener: (data: ConnectDataType) => void;
|
|
27
29
|
createServer: (sessionId?: string) => void;
|
|
28
|
-
checkHumanInstansce: () =>
|
|
30
|
+
checkHumanInstansce: () => boolean;
|
|
29
31
|
sendMessage: (message: DHServerInterface | ArrayBuffer, listener: RenderCallback | null) => Promise<void> | null;
|
|
30
32
|
interrupt: () => Promise<void>;
|
|
31
33
|
textRender: (renderData: {
|
package/cjs/Client.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("./_virtual/_rollupPluginBabelHelpers.js"),t=require("@bddh/starling-json"),r=require("lodash/isEmpty"),n=require("lodash/omit"),
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("./_virtual/_rollupPluginBabelHelpers.js"),t=require("@bddh/starling-json"),r=require("lodash/isEmpty"),n=require("lodash/omit"),a=require("lodash/pick"),o=require("uuid"),i=require("./interface.js"),s=require("./Server.js"),c=require("./BrtcClient.js"),u=require("./DomControl.js"),d=require("./Cutout.js"),l=require("./RecordController.js"),m=function(){function m(r){var c=this;e.classCallCheck(this,m),e.defineProperty(this,"token",void 0),e.defineProperty(this,"appKey",void 0),e.defineProperty(this,"appId",void 0),e.defineProperty(this,"connectParams",void 0),e.defineProperty(this,"onDigitalHumanCallback",void 0),e.defineProperty(this,"dhServer",void 0),e.defineProperty(this,"rtcServer",void 0),e.defineProperty(this,"rtcConnectParams",void 0),e.defineProperty(this,"cutoutInstance",void 0),e.defineProperty(this,"isHuawei",u.checkIsHuawei()),e.defineProperty(this,"rtcServerUrl",void 0),e.defineProperty(this,"recordController",null),e.defineProperty(this,"wsReadyState",-1),e.defineProperty(this,"onDigitalHumanCallbackPlus",(function(e){var t;e.status===i.statusEnum.DH_LIB_FULL_STATUS&&"wsState"===(null===(t=e.content)||void 0===t?void 0:t.type)&&(c.wsReadyState=e.content.readyState),c.onDigitalHumanCallback&&c.onDigitalHumanCallback(e)})),e.defineProperty(this,"onRemoteVideoComing",(function(e){c.rtcConnectParams&&c.rtcConnectParams.feedId&&e===+c.rtcConnectParams.feedId&&c.rtcServer&&c.rtcServer.client&&c.rtcServer.client.subscribeStreaming("remotevideo500",+c.rtcConnectParams.feedId)})),e.defineProperty(this,"onLocalStream",(function(){(c.connectParams||{}).pullAudioFromRtc&&c.rtcServer&&c.rtcServer.muteMicphone(!0)})),e.defineProperty(this,"onRemoteVideoOn",(function(){setTimeout(e.asyncToGenerator(e.regeneratorRuntime().mark((function t(){var r,n,a,o,i;return e.regeneratorRuntime().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return r=document.getElementById(u.RTC_VIDEO_ID),e.next=3,c.cutoutInstance&&c.cutoutInstance.updateSource(u.RTC_VIDEO_ID);case 3:r&&(r.setAttribute("data-playing","true"),n=r.videoWidth,a=r.videoHeight,o=document.getElementById(u.DEFAULT_CANVAS_ID),i=o.getContext("2d"),o&&i&&(o.width=n,o.height=a,i.drawImage(r,0,0,n,a)));case 4:case"end":return e.stop()}}),t)}))),100)})),e.defineProperty(this,"connectListener",(function(e){var r=e.code,n=e.body,a=e.message;if(0===r&&c.onDigitalHumanCallback&&c.onDigitalHumanCallback({status:i.statusEnum.DH_LIB_WS_SUCCESS,content:e}),0===r){var o=t.parse(n),s=o.rtcServerUrl,u=o.appId,d=o.clientToken,l=o.roomName,m=o.clientId,p=o.feedId,h=o.chargeId;u&&l&&(c.rtcConnectParams={server:c.rtcServerUrl||s,appId:u,token:d,roomName:l,userId:m,feedId:p,chargeId:h},c.rtcServer&&c.rtcServer.openRtc(c.rtcConnectParams))}else c.onDigitalHumanCallback&&c.onDigitalHumanCallback({status:i.statusEnum.DH_LIB_ERROR,content:{type:i.errorTypeEnum.CONNECT_ERROR,msg:{code:r,errMsg:a}}})})),e.defineProperty(this,"createServer",(function(t){var r,o=e.objectSpread2(e.objectSpread2(e.objectSpread2({token:c.token,appKey:c.appKey,appId:c.appId},t?{sessionId:t}:{}),a(c.connectParams,i.CONNECT_MAIN_KEY)),{},{parameters:n(c.connectParams,i.CONNECT_MAIN_KEY),onConnect:c.connectListener,onDigitalHumanCallback:c.onDigitalHumanCallbackPlus});null!==(r=c.dhServer)&&void 0!==r&&r.preConnectSuccess?c.dhServer&&c.dhServer.sendPreConnect(o):c.dhServer&&c.dhServer.createSocket(o)})),e.defineProperty(this,"checkHumanInstansce",(function(){return c.dhServer?!!c.dhServer.preConnectSuccess||(c.onDigitalHumanCallback&&c.onDigitalHumanCallback({status:i.statusEnum.DH_LIB_ERROR,content:{type:i.errorTypeEnum.CONNECT_ERROR,msg:{code:4001,errMsg:"webSocket连接中"}}}),!1):(c.onDigitalHumanCallback&&c.onDigitalHumanCallback({status:i.statusEnum.DH_LIB_ERROR,content:{type:i.errorTypeEnum.CONNECT_ERROR,msg:{code:4001,errMsg:"请先初始化数字人示例"}}}),!1)})),e.defineProperty(this,"sendMessage",(function(e,t){return c.dhServer&&c.dhServer.sendMessage(e,t)})),e.defineProperty(this,"interrupt",e.asyncToGenerator(e.regeneratorRuntime().mark((function t(){return e.regeneratorRuntime().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.abrupt("return",new Promise((function(e){c.dhServer&&c.dhServer.sendMessage({requestId:o.v4(),action:"TEXT_RENDER",body:"<interrupt></interrupt>",clientTs:(new Date).getTime()},(function(){return e()}))})));case 1:case"end":return e.stop()}}),t)})))),e.defineProperty(this,"textRender",(function(e){var t=e.requestId,r=e.body,n=e.onCallbackMsg;c.checkHumanInstansce()&&c.dhServer&&c.dhServer.sendMessage({requestId:t||o.v4(),action:"TEXT_RENDER",body:r,clientTs:(new Date).getTime()},n)})),e.defineProperty(this,"textStreamRender",(function(e){var r=e.requestId,n=e.body,a=e.onCallbackMsg,i=t.parse(n).first;c.checkHumanInstansce()&&c.dhServer&&c.dhServer.sendMessage({requestId:r||o.v4(),action:"TEXT_STREAM_RENDER",body:n,clientTs:(new Date).getTime()},i?a:null)})),e.defineProperty(this,"audioRender",(function(e){var t=e.body,r=e.onCallbackMsg;c.checkHumanInstansce()&&c.dhServer&&c.dhServer.sendMessage({requestId:o.v4(),action:"AUDIO_RENDER",body:t,clientTs:(new Date).getTime()},r)})),e.defineProperty(this,"audioStreamRender",(function(e){var t=e.requestId,r=e.body,n=e.onCallbackMsg;c.checkHumanInstansce()&&c.dhServer&&c.dhServer.sendMessage({requestId:t||o.v4(),action:"AUDIO_STREAM_RENDER",body:r,clientTs:(new Date).getTime()},n)})),e.defineProperty(this,"textQuery",(function(e){var t=e.requestId,r=e.body,n=e.onCallbackMsg;c.dhServer&&c.dhServer.sendMessage({requestId:t||o.v4(),action:"TEXT_QUERY",body:r,clientTs:(new Date).getTime()},n)})),e.defineProperty(this,"destory",e.asyncToGenerator(e.regeneratorRuntime().mark((function t(){return e.regeneratorRuntime().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.next=2,c.dhServer&&c.dhServer.closeSocket();case 2:c.rtcServer&&c.rtcServer.closeRtc(),u.removeHumanDom(),c.dhServer=null,c.rtcServer=null;case 6:case"end":return e.stop()}}),t)})))),e.defineProperty(this,"pauseHuman",(function(){return u.playHumanVideo(!1)})),e.defineProperty(this,"playHuman",(function(){return u.playHumanVideo(!0)})),e.defineProperty(this,"muteHuman",(function(){return u.muteHumanVideo(!0)})),e.defineProperty(this,"unMuteHuman",(function(){var e=u.muteHumanVideo(!1);try{return c.isHuawei?(e.pause(),e.play()):e.paused&&e.play()}catch(e){console.info(e)}}));var d=r.isPreOpenSession,l=void 0!==d&&d,p=r.wsUrl,h=r.renderParams,v=r.onDigitalHumanCallback;this.dhServer=new s.default(p||"wss://open.xiling.baidu.com/cloud/digital-human-demonstration"),this.onDigitalHumanCallback=function(e){e.status!==i.statusEnum.DH_LIB_FULL_STATUS?v&&v(e):(null==h?void 0:h.fullStatus)&&v&&v(e)},l?this.dhServer.createSocket({onDigitalHumanCallback:this.onDigitalHumanCallbackPlus},l):this.init(r)}return e.createClass(m,[{key:"init",value:function(t){var n=this,a=t.wrapperId,o=t.token,s=t.appKey,m=t.appId,p=t.connectParams,h=t.renderParams,v=void 0===h?{}:h,f=t.rtcConnectParams,y=t.brtcParams,S=t.rtcServerUrl;if(this.token=o,this.appKey=s,this.appId=m,this.rtcServerUrl=S,this.connectParams=e.objectSpread2({autoChromaKey:(null==v?void 0:v.autoChromaKey)||!1},p),!m&&!s&&!o)throw new Error("appId and appKey or token is required");var g=v.closeLog,C=(u.createHumanDom({autoChromaKey:!!v.autoChromaKey||!r(v.chromaEffects)})||{}).playerWrapper,R=document.getElementById(a||"");R||console.error("dom with wrapperId is not find"),C&&R&&R.appendChild(C),u.appendRtcDomStyle();var I=e.objectSpread2({remotevideoviewid:a,showvideobps:!1,usingvideo:!1,usingaudio:!(null==p||!p.pullAudioFromRtc),aspublisher:!(null==p||!p.pullAudioFromRtc),autopublish:!(null==p||!p.pullAudioFromRtc),usingdatachannel:!0,autoplaymuted:!1},y);this.rtcServer=new c.default({closeLog:g,videoWrapperId:u.RTC_VIDEO_ID,rtcConfig:I,bizConfig:{},rtcCallback:{onRemoteVideoComing:this.onRemoteVideoComing,onDigitalHumanCallback:this.onDigitalHumanCallback,onLocalStream:this.onLocalStream}}),this.cutoutInstance=new d.default({sourceId:u.RTC_VIDEO_ID,effects:null==v?void 0:v.chromaEffects}),r(f)||(this.rtcConnectParams=f,this.rtcServer.openRtc(f)),null!=p&&p.pullAudioFromRtc?this.recordController=null:this.recordController=new l.default({pickAudioMode:null==p?void 0:p.pickAudioMode,onAudioData:function(e){1===n.wsReadyState&&n.sendMessage(e,null)},onError:function(e){n.onDigitalHumanCallback&&n.onDigitalHumanCallback({status:i.statusEnum.DH_LIB_ERROR,content:{type:i.errorTypeEnum.CONNECT_ERROR,msg:{code:-1,errMsg:e.message}}})}})}},{key:"startRecord",value:function(){var e;null!==(e=this.connectParams)&&void 0!==e&&e.pullAudioFromRtc?this.rtcServer&&this.rtcServer.muteMicphone(!1):this.recordController&&this.recordController.start()}},{key:"stopRecord",value:function(){var e;if(null!==(e=this.connectParams)&&void 0!==e&&e.pullAudioFromRtc)return this.rtcServer&&this.rtcServer.muteMicphone(!0),void this.sendMessage({action:"AUDIO_QUERY_INTERRUPT",requestId:o.v4(),clientTs:Date.now(),body:""},null);this.recordController&&this.recordController.stop(),this.sendMessage({action:"AUDIO_QUERY_INTERRUPT",requestId:o.v4(),clientTs:Date.now(),body:""},null)}}]),m}();exports.default=m;
|
package/cjs/Server.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export interface createSocketParamsType {
|
|
|
32
32
|
sessionId?: string;
|
|
33
33
|
parameters?: ConnectParamsType;
|
|
34
34
|
reConnect?: boolean;
|
|
35
|
-
onConnect
|
|
35
|
+
onConnect?: ConnectListener;
|
|
36
36
|
onDigitalHumanCallback?: (data: CallbackMsgType) => void;
|
|
37
37
|
}
|
|
38
38
|
export interface RenderCallbackRes {
|
|
@@ -45,12 +45,14 @@ export interface RenderCallbackRes {
|
|
|
45
45
|
export type RenderCallback = (res: RenderCallbackRes) => void;
|
|
46
46
|
declare class Server {
|
|
47
47
|
dhServerSocket: WebSocketClient | null;
|
|
48
|
+
preConnectSuccess: boolean;
|
|
48
49
|
private socketClosingPromise;
|
|
49
50
|
private readonly url;
|
|
50
51
|
private firstConnect;
|
|
51
52
|
private connectParams;
|
|
52
53
|
constructor(url: string);
|
|
53
|
-
|
|
54
|
+
sendPreConnect: (props: createSocketParamsType, resolve?: ((value: any) => void) | undefined) => void;
|
|
55
|
+
createSocket: (props: createSocketParamsType, isPreOpenSession?: boolean) => Promise<WebSocketClient | null>;
|
|
54
56
|
closeSocket: () => Promise<void>;
|
|
55
57
|
handleConnect: () => Promise<void>;
|
|
56
58
|
sendMessage(message: DHServerInterface | ArrayBuffer, listener: RenderCallback | null): Promise<void>;
|
package/cjs/Server.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("./_virtual/_rollupPluginBabelHelpers.js"),r=require("uuid"),t=require("@bddh/starling-web-socket/es/create-web-socket"),n=require("lodash/pick"),o=require("lodash/omit"),s=require("lodash/get"),
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("./_virtual/_rollupPluginBabelHelpers.js"),r=require("uuid"),t=require("@bddh/starling-web-socket/es/create-web-socket"),n=require("lodash/pick"),o=require("lodash/omit"),s=require("lodash/get"),c=require("./interface.js"),a=function(a){function i(a){var u,d=this;e.classCallCheck(this,i),e.defineProperty(this,"dhServerSocket",null),e.defineProperty(this,"preConnectSuccess",!1),e.defineProperty(this,"socketClosingPromise",null),e.defineProperty(this,"url",void 0),e.defineProperty(this,"firstConnect",!0),e.defineProperty(this,"connectParams",null),e.defineProperty(this,"sendPreConnect",(function(e,t){var n,o=e.token,s=e.sessionId,c=e.onConnect;d.firstConnect&&(null===(n=d.dhServerSocket)||void 0===n||n.request({action:"RE_CONNECT",requestId:r.v4(),body:JSON.stringify({token:o,sessionId:s})}).on((function(e,r){null==c||c(e),0!==e.code&&(r(),d.handleConnect()),null==t||t(d.dhServerSocket)})),d.firstConnect=!1)})),e.defineProperty(this,"createSocket",(function(r,n){return(u=u||e.asyncToGenerator(e.regeneratorRuntime().mark((function r(n,o){var a,i;return e.regeneratorRuntime().wrap((function(r){for(;;)switch(r.prev=r.next){case 0:if(d.connectParams=n,a=n.sessionId,i=n.onDigitalHumanCallback,!d.socketClosingPromise){r.next=5;break}return r.next=5,d.socketClosingPromise;case 5:if(!d.dhServerSocket){r.next=8;break}return r.next=8,d.closeSocket();case 8:return r.abrupt("return",new Promise((function(r,u){var p={keepAlive:"heartbeat",combine:function(e,r){return e&&r&&e.requestId===r.requestId},onClose:function(){d.dhServerSocket=null,d.socketClosingPromise=null},onError:function(e){u(e)}},l=o?p:e.objectSpread2(e.objectSpread2({},p),{},{reopen:s(d.connectParams,"reConnect",!0),onOpen:function(){d.dhServerSocket=S,a?d.sendPreConnect(n,r):d.handleConnect()}}),S=t.createWebSocket(d.url,l);S.onReadyStateChange((function(e){0===e&&i&&i({status:c.statusEnum.DH_LIB_INIT}),1===e&&o&&(d.preConnectSuccess=!0),3===e&&(d.preConnectSuccess=!1),i&&i({status:c.statusEnum.DH_LIB_FULL_STATUS,content:{type:"wsState",readyState:e,body:c.WS_STATUS_MAP[e]}})})),S.onMessage((function(e){var r=e.action;[c.errorTypeEnum.TIMEOUT_EXIT,c.errorTypeEnum.DISCONNECT_ALERT].includes(r)&&i&&i({status:c.statusEnum.DH_LIB_STATUS,content:{type:r}})})),o&&(d.dhServerSocket=S),S.open()})));case 9:case"end":return r.stop()}}),r)})))).apply(this,arguments)})),e.defineProperty(this,"closeSocket",e.asyncToGenerator(e.regeneratorRuntime().mark((function r(){return e.regeneratorRuntime().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(d.dhServerSocket){e.next=2;break}return e.abrupt("return");case 2:return d.socketClosingPromise=d.dhServerSocket.close(),e.next=5,d.socketClosingPromise;case 5:d.dhServerSocket=null;case 6:case"end":return e.stop()}}),r)})))),e.defineProperty(this,"handleConnect",e.asyncToGenerator(e.regeneratorRuntime().mark((function t(){var s,a,i,u,p,l;return e.regeneratorRuntime().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(d.connectParams){t.next=2;break}return t.abrupt("return");case 2:if(s=d.connectParams,a=s.appId,i=s.appKey,u=s.token,p=s.parameters,l=s.onConnect,a||i||u){t.next=5;break}return t.abrupt("return");case 5:d.dhServerSocket&&d.dhServerSocket.request({action:"CONNECT",clientTs:(new Date).getTime(),requestId:r.v4(),body:JSON.stringify(e.objectSpread2(e.objectSpread2({token:u,appId:a,appKey:i},n(d.connectParams,c.CONNECT_MAIN_KEY)),{},{parameters:e.objectSpread2(e.objectSpread2({},o(p,["parameters"])),p&&p.parameters?p.parameters:{})}))}).on((function(e,r){null==l||l(e),0!==e.code&&r()}));case 6:case"end":return t.stop()}}),t)})))),this.url=a}return e.createClass(i,[{key:"sendMessage",value:function(t,n){return(a=a||e.asyncToGenerator(e.regeneratorRuntime().mark((function t(n,o){var s;return e.regeneratorRuntime().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:try{this.dhServerSocket&&this.dhServerSocket&&this.dhServerSocket.request(n).on((function(e){o&&o(e)}))}catch(t){s="object"===e.typeof(n)&&"requestId"in n?n.requestId:r.v4(),o&&o({code:-11,body:null,action:"RENDER_ERROR",requestId:s,message:JSON.stringify(t)})}case 1:case"end":return t.stop()}}),t,this)})))).apply(this,arguments)}}]),i}();exports.default=a;
|
package/cjs/interface.d.ts
CHANGED
|
@@ -13,7 +13,6 @@ export interface ConnectParamsType {
|
|
|
13
13
|
cameraId?: string | number;
|
|
14
14
|
characterOffset?: string;
|
|
15
15
|
pickAudioMode?: 'pressButton' | 'free';
|
|
16
|
-
fullDuplex: boolean;
|
|
17
16
|
autoChromaKey?: boolean;
|
|
18
17
|
usingVideo?: boolean;
|
|
19
18
|
ttsPer?: string | number;
|
|
@@ -98,10 +97,11 @@ export interface DhRealtimeHumanType {
|
|
|
98
97
|
rtcServerUrl?: string;
|
|
99
98
|
appKey?: string;
|
|
100
99
|
appId?: string;
|
|
101
|
-
wrapperId
|
|
100
|
+
wrapperId?: string;
|
|
102
101
|
connectParams?: ConnectParamsType;
|
|
103
102
|
renderParams?: RenderParamsType;
|
|
104
103
|
rtcConnectParams?: RtcConnectionType;
|
|
105
104
|
brtcParams?: any;
|
|
105
|
+
isPreOpenSession?: boolean;
|
|
106
106
|
onDigitalHumanCallback: (data: CallbackMsgType) => void;
|
|
107
107
|
}
|
package/es/Client.d.ts
CHANGED
|
@@ -3,29 +3,31 @@
|
|
|
3
3
|
* @description 云渲染数字人sdk
|
|
4
4
|
* @author zhangyue49
|
|
5
5
|
*/
|
|
6
|
-
import { DhRealtimeHumanType, ConnectDataType } from './interface';
|
|
6
|
+
import { DhRealtimeHumanType, ConnectDataType, CallbackMsgType } from './interface';
|
|
7
7
|
import { DHServerInterface, RenderCallback } from './Server';
|
|
8
8
|
export default class DhRealtimeHuman {
|
|
9
|
-
private
|
|
10
|
-
private
|
|
11
|
-
private
|
|
12
|
-
private
|
|
13
|
-
private readonly onDigitalHumanCallback
|
|
9
|
+
private token;
|
|
10
|
+
private appKey;
|
|
11
|
+
private appId;
|
|
12
|
+
private connectParams;
|
|
13
|
+
private readonly onDigitalHumanCallback?;
|
|
14
14
|
private dhServer;
|
|
15
|
-
private rtcServer
|
|
15
|
+
private rtcServer?;
|
|
16
16
|
private rtcConnectParams;
|
|
17
|
-
private
|
|
17
|
+
private cutoutInstance?;
|
|
18
18
|
private readonly isHuawei;
|
|
19
|
-
private
|
|
20
|
-
private
|
|
19
|
+
private rtcServerUrl;
|
|
20
|
+
private recordController;
|
|
21
21
|
private wsReadyState;
|
|
22
22
|
constructor(props: DhRealtimeHumanType);
|
|
23
|
+
init(props: DhRealtimeHumanType): void;
|
|
24
|
+
onDigitalHumanCallbackPlus: (data: CallbackMsgType) => void;
|
|
23
25
|
onRemoteVideoComing: (id: number) => void;
|
|
24
26
|
onLocalStream: () => void;
|
|
25
27
|
onRemoteVideoOn: () => void;
|
|
26
28
|
connectListener: (data: ConnectDataType) => void;
|
|
27
29
|
createServer: (sessionId?: string) => void;
|
|
28
|
-
checkHumanInstansce: () =>
|
|
30
|
+
checkHumanInstansce: () => boolean;
|
|
29
31
|
sendMessage: (message: DHServerInterface | ArrayBuffer, listener: RenderCallback | null) => Promise<void> | null;
|
|
30
32
|
interrupt: () => Promise<void>;
|
|
31
33
|
textRender: (renderData: {
|
package/es/Client.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{createClass as e,classCallCheck as t,defineProperty as r,objectSpread2 as n,asyncToGenerator as
|
|
1
|
+
import{createClass as e,classCallCheck as t,defineProperty as r,objectSpread2 as n,asyncToGenerator as a,regeneratorRuntime as o}from"./_virtual/_rollupPluginBabelHelpers.js";import i from"@bddh/starling-json";import s from"lodash/isEmpty";import c from"lodash/omit";import l from"lodash/pick";import{v4 as u}from"uuid";import{statusEnum as d,errorTypeEnum as m,CONNECT_MAIN_KEY as h}from"./interface.js";import p from"./Server.js";import v from"./BrtcClient.js";import{createHumanDom as S,appendRtcDomStyle as g,RTC_VIDEO_ID as C,checkIsHuawei as f,playHumanVideo as R,muteHumanVideo as I,removeHumanDom as b,DEFAULT_CANVAS_ID as k}from"./DomControl.js";import y from"./Cutout.js";import D from"./RecordController.js";var E=function(){function E(e){var s=this;t(this,E),r(this,"token",void 0),r(this,"appKey",void 0),r(this,"appId",void 0),r(this,"connectParams",void 0),r(this,"onDigitalHumanCallback",void 0),r(this,"dhServer",void 0),r(this,"rtcServer",void 0),r(this,"rtcConnectParams",void 0),r(this,"cutoutInstance",void 0),r(this,"isHuawei",f()),r(this,"rtcServerUrl",void 0),r(this,"recordController",null),r(this,"wsReadyState",-1),r(this,"onDigitalHumanCallbackPlus",(function(e){var t;e.status===d.DH_LIB_FULL_STATUS&&"wsState"===(null===(t=e.content)||void 0===t?void 0:t.type)&&(s.wsReadyState=e.content.readyState),s.onDigitalHumanCallback&&s.onDigitalHumanCallback(e)})),r(this,"onRemoteVideoComing",(function(e){s.rtcConnectParams&&s.rtcConnectParams.feedId&&e===+s.rtcConnectParams.feedId&&s.rtcServer&&s.rtcServer.client&&s.rtcServer.client.subscribeStreaming("remotevideo500",+s.rtcConnectParams.feedId)})),r(this,"onLocalStream",(function(){(s.connectParams||{}).pullAudioFromRtc&&s.rtcServer&&s.rtcServer.muteMicphone(!0)})),r(this,"onRemoteVideoOn",(function(){setTimeout(a(o().mark((function e(){var t,r,n,a,i;return o().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return t=document.getElementById(C),e.next=3,s.cutoutInstance&&s.cutoutInstance.updateSource(C);case 3:t&&(t.setAttribute("data-playing","true"),r=t.videoWidth,n=t.videoHeight,a=document.getElementById(k),i=a.getContext("2d"),a&&i&&(a.width=r,a.height=n,i.drawImage(t,0,0,r,n)));case 4:case"end":return e.stop()}}),e)}))),100)})),r(this,"connectListener",(function(e){var t=e.code,r=e.body,n=e.message;if(0===t&&s.onDigitalHumanCallback&&s.onDigitalHumanCallback({status:d.DH_LIB_WS_SUCCESS,content:e}),0===t){var a=i.parse(r),o=a.rtcServerUrl,c=a.appId,l=a.clientToken,u=a.roomName,h=a.clientId,p=a.feedId,v=a.chargeId;c&&u&&(s.rtcConnectParams={server:s.rtcServerUrl||o,appId:c,token:l,roomName:u,userId:h,feedId:p,chargeId:v},s.rtcServer&&s.rtcServer.openRtc(s.rtcConnectParams))}else s.onDigitalHumanCallback&&s.onDigitalHumanCallback({status:d.DH_LIB_ERROR,content:{type:m.CONNECT_ERROR,msg:{code:t,errMsg:n}}})})),r(this,"createServer",(function(e){var t,r=n(n(n({token:s.token,appKey:s.appKey,appId:s.appId},e?{sessionId:e}:{}),l(s.connectParams,h)),{},{parameters:c(s.connectParams,h),onConnect:s.connectListener,onDigitalHumanCallback:s.onDigitalHumanCallbackPlus});null!==(t=s.dhServer)&&void 0!==t&&t.preConnectSuccess?s.dhServer&&s.dhServer.sendPreConnect(r):s.dhServer&&s.dhServer.createSocket(r)})),r(this,"checkHumanInstansce",(function(){return s.dhServer?!!s.dhServer.preConnectSuccess||(s.onDigitalHumanCallback&&s.onDigitalHumanCallback({status:d.DH_LIB_ERROR,content:{type:m.CONNECT_ERROR,msg:{code:4001,errMsg:"webSocket连接中"}}}),!1):(s.onDigitalHumanCallback&&s.onDigitalHumanCallback({status:d.DH_LIB_ERROR,content:{type:m.CONNECT_ERROR,msg:{code:4001,errMsg:"请先初始化数字人示例"}}}),!1)})),r(this,"sendMessage",(function(e,t){return s.dhServer&&s.dhServer.sendMessage(e,t)})),r(this,"interrupt",a(o().mark((function e(){return o().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.abrupt("return",new Promise((function(e){s.dhServer&&s.dhServer.sendMessage({requestId:u(),action:"TEXT_RENDER",body:"<interrupt></interrupt>",clientTs:(new Date).getTime()},(function(){return e()}))})));case 1:case"end":return e.stop()}}),e)})))),r(this,"textRender",(function(e){var t=e.requestId,r=e.body,n=e.onCallbackMsg;s.checkHumanInstansce()&&s.dhServer&&s.dhServer.sendMessage({requestId:t||u(),action:"TEXT_RENDER",body:r,clientTs:(new Date).getTime()},n)})),r(this,"textStreamRender",(function(e){var t=e.requestId,r=e.body,n=e.onCallbackMsg,a=i.parse(r).first;s.checkHumanInstansce()&&s.dhServer&&s.dhServer.sendMessage({requestId:t||u(),action:"TEXT_STREAM_RENDER",body:r,clientTs:(new Date).getTime()},a?n:null)})),r(this,"audioRender",(function(e){var t=e.body,r=e.onCallbackMsg;s.checkHumanInstansce()&&s.dhServer&&s.dhServer.sendMessage({requestId:u(),action:"AUDIO_RENDER",body:t,clientTs:(new Date).getTime()},r)})),r(this,"audioStreamRender",(function(e){var t=e.requestId,r=e.body,n=e.onCallbackMsg;s.checkHumanInstansce()&&s.dhServer&&s.dhServer.sendMessage({requestId:t||u(),action:"AUDIO_STREAM_RENDER",body:r,clientTs:(new Date).getTime()},n)})),r(this,"textQuery",(function(e){var t=e.requestId,r=e.body,n=e.onCallbackMsg;s.dhServer&&s.dhServer.sendMessage({requestId:t||u(),action:"TEXT_QUERY",body:r,clientTs:(new Date).getTime()},n)})),r(this,"destory",a(o().mark((function e(){return o().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.next=2,s.dhServer&&s.dhServer.closeSocket();case 2:s.rtcServer&&s.rtcServer.closeRtc(),b(),s.dhServer=null,s.rtcServer=null;case 6:case"end":return e.stop()}}),e)})))),r(this,"pauseHuman",(function(){return R(!1)})),r(this,"playHuman",(function(){return R(!0)})),r(this,"muteHuman",(function(){return I(!0)})),r(this,"unMuteHuman",(function(){var e=I(!1);try{return s.isHuawei?(e.pause(),e.play()):e.paused&&e.play()}catch(e){console.info(e)}}));var v=e.isPreOpenSession,S=void 0!==v&&v,g=e.wsUrl,y=e.renderParams,D=e.onDigitalHumanCallback;this.dhServer=new p(g||"wss://open.xiling.baidu.com/cloud/digital-human-demonstration"),this.onDigitalHumanCallback=function(e){e.status!==d.DH_LIB_FULL_STATUS?D&&D(e):(null==y?void 0:y.fullStatus)&&D&&D(e)},S?this.dhServer.createSocket({onDigitalHumanCallback:this.onDigitalHumanCallbackPlus},S):this.init(e)}return e(E,[{key:"init",value:function(e){var t=this,r=e.wrapperId,a=e.token,o=e.appKey,i=e.appId,c=e.connectParams,l=e.renderParams,u=void 0===l?{}:l,h=e.rtcConnectParams,p=e.brtcParams,f=e.rtcServerUrl;if(this.token=a,this.appKey=o,this.appId=i,this.rtcServerUrl=f,this.connectParams=n({autoChromaKey:(null==u?void 0:u.autoChromaKey)||!1},c),!i&&!o&&!a)throw new Error("appId and appKey or token is required");var R=u.closeLog,I=(S({autoChromaKey:!!u.autoChromaKey||!s(u.chromaEffects)})||{}).playerWrapper,b=document.getElementById(r||"");b||console.error("dom with wrapperId is not find"),I&&b&&b.appendChild(I),g();var k=n({remotevideoviewid:r,showvideobps:!1,usingvideo:!1,usingaudio:!(null==c||!c.pullAudioFromRtc),aspublisher:!(null==c||!c.pullAudioFromRtc),autopublish:!(null==c||!c.pullAudioFromRtc),usingdatachannel:!0,autoplaymuted:!1},p);this.rtcServer=new v({closeLog:R,videoWrapperId:C,rtcConfig:k,bizConfig:{},rtcCallback:{onRemoteVideoComing:this.onRemoteVideoComing,onDigitalHumanCallback:this.onDigitalHumanCallback,onLocalStream:this.onLocalStream}}),this.cutoutInstance=new y({sourceId:C,effects:null==u?void 0:u.chromaEffects}),s(h)||(this.rtcConnectParams=h,this.rtcServer.openRtc(h)),null!=c&&c.pullAudioFromRtc?this.recordController=null:this.recordController=new D({pickAudioMode:null==c?void 0:c.pickAudioMode,onAudioData:function(e){1===t.wsReadyState&&t.sendMessage(e,null)},onError:function(e){t.onDigitalHumanCallback&&t.onDigitalHumanCallback({status:d.DH_LIB_ERROR,content:{type:m.CONNECT_ERROR,msg:{code:-1,errMsg:e.message}}})}})}},{key:"startRecord",value:function(){var e;null!==(e=this.connectParams)&&void 0!==e&&e.pullAudioFromRtc?this.rtcServer&&this.rtcServer.muteMicphone(!1):this.recordController&&this.recordController.start()}},{key:"stopRecord",value:function(){var e;if(null!==(e=this.connectParams)&&void 0!==e&&e.pullAudioFromRtc)return this.rtcServer&&this.rtcServer.muteMicphone(!0),void this.sendMessage({action:"AUDIO_QUERY_INTERRUPT",requestId:u(),clientTs:Date.now(),body:""},null);this.recordController&&this.recordController.stop(),this.sendMessage({action:"AUDIO_QUERY_INTERRUPT",requestId:u(),clientTs:Date.now(),body:""},null)}}]),E}();export{E as default};
|
package/es/Server.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export interface createSocketParamsType {
|
|
|
32
32
|
sessionId?: string;
|
|
33
33
|
parameters?: ConnectParamsType;
|
|
34
34
|
reConnect?: boolean;
|
|
35
|
-
onConnect
|
|
35
|
+
onConnect?: ConnectListener;
|
|
36
36
|
onDigitalHumanCallback?: (data: CallbackMsgType) => void;
|
|
37
37
|
}
|
|
38
38
|
export interface RenderCallbackRes {
|
|
@@ -45,12 +45,14 @@ export interface RenderCallbackRes {
|
|
|
45
45
|
export type RenderCallback = (res: RenderCallbackRes) => void;
|
|
46
46
|
declare class Server {
|
|
47
47
|
dhServerSocket: WebSocketClient | null;
|
|
48
|
+
preConnectSuccess: boolean;
|
|
48
49
|
private socketClosingPromise;
|
|
49
50
|
private readonly url;
|
|
50
51
|
private firstConnect;
|
|
51
52
|
private connectParams;
|
|
52
53
|
constructor(url: string);
|
|
53
|
-
|
|
54
|
+
sendPreConnect: (props: createSocketParamsType, resolve?: ((value: any) => void) | undefined) => void;
|
|
55
|
+
createSocket: (props: createSocketParamsType, isPreOpenSession?: boolean) => Promise<WebSocketClient | null>;
|
|
54
56
|
closeSocket: () => Promise<void>;
|
|
55
57
|
handleConnect: () => Promise<void>;
|
|
56
58
|
sendMessage(message: DHServerInterface | ArrayBuffer, listener: RenderCallback | null): Promise<void>;
|
package/es/Server.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{createClass as e,asyncToGenerator as t,classCallCheck as
|
|
1
|
+
import{createClass as e,asyncToGenerator as t,classCallCheck as n,defineProperty as r,regeneratorRuntime as o,objectSpread2 as s,typeof as c}from"./_virtual/_rollupPluginBabelHelpers.js";import{v4 as a}from"uuid";import{createWebSocket as i}from"@bddh/starling-web-socket/es/create-web-socket";import u from"lodash/pick";import l from"lodash/omit";import d from"lodash/get";import{errorTypeEnum as p,CONNECT_MAIN_KEY as f,statusEnum as h,WS_STATUS_MAP as S}from"./interface.js";var k=function(k){function m(e){var c,k=this;n(this,m),r(this,"dhServerSocket",null),r(this,"preConnectSuccess",!1),r(this,"socketClosingPromise",null),r(this,"url",void 0),r(this,"firstConnect",!0),r(this,"connectParams",null),r(this,"sendPreConnect",(function(e,t){var n,r=e.token,o=e.sessionId,s=e.onConnect;k.firstConnect&&(null===(n=k.dhServerSocket)||void 0===n||n.request({action:"RE_CONNECT",requestId:a(),body:JSON.stringify({token:r,sessionId:o})}).on((function(e,n){null==s||s(e),0!==e.code&&(n(),k.handleConnect()),null==t||t(k.dhServerSocket)})),k.firstConnect=!1)})),r(this,"createSocket",(function(e,n){return(c=c||t(o().mark((function e(t,n){var r,c;return o().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(k.connectParams=t,r=t.sessionId,c=t.onDigitalHumanCallback,!k.socketClosingPromise){e.next=5;break}return e.next=5,k.socketClosingPromise;case 5:if(!k.dhServerSocket){e.next=8;break}return e.next=8,k.closeSocket();case 8:return e.abrupt("return",new Promise((function(e,o){var a={keepAlive:"heartbeat",combine:function(e,t){return e&&t&&e.requestId===t.requestId},onClose:function(){k.dhServerSocket=null,k.socketClosingPromise=null},onError:function(e){o(e)}},u=n?a:s(s({},a),{},{reopen:d(k.connectParams,"reConnect",!0),onOpen:function(){k.dhServerSocket=l,r?k.sendPreConnect(t,e):k.handleConnect()}}),l=i(k.url,u);l.onReadyStateChange((function(e){0===e&&c&&c({status:h.DH_LIB_INIT}),1===e&&n&&(k.preConnectSuccess=!0),3===e&&(k.preConnectSuccess=!1),c&&c({status:h.DH_LIB_FULL_STATUS,content:{type:"wsState",readyState:e,body:S[e]}})})),l.onMessage((function(e){var t=e.action;[p.TIMEOUT_EXIT,p.DISCONNECT_ALERT].includes(t)&&c&&c({status:h.DH_LIB_STATUS,content:{type:t}})})),n&&(k.dhServerSocket=l),l.open()})));case 9:case"end":return e.stop()}}),e)})))).apply(this,arguments)})),r(this,"closeSocket",t(o().mark((function e(){return o().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(k.dhServerSocket){e.next=2;break}return e.abrupt("return");case 2:return k.socketClosingPromise=k.dhServerSocket.close(),e.next=5,k.socketClosingPromise;case 5:k.dhServerSocket=null;case 6:case"end":return e.stop()}}),e)})))),r(this,"handleConnect",t(o().mark((function e(){var t,n,r,c,i,d;return o().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(k.connectParams){e.next=2;break}return e.abrupt("return");case 2:if(t=k.connectParams,n=t.appId,r=t.appKey,c=t.token,i=t.parameters,d=t.onConnect,n||r||c){e.next=5;break}return e.abrupt("return");case 5:k.dhServerSocket&&k.dhServerSocket.request({action:"CONNECT",clientTs:(new Date).getTime(),requestId:a(),body:JSON.stringify(s(s({token:c,appId:n,appKey:r},u(k.connectParams,f)),{},{parameters:s(s({},l(i,["parameters"])),i&&i.parameters?i.parameters:{})}))}).on((function(e,t){null==d||d(e),0!==e.code&&t()}));case 6:case"end":return e.stop()}}),e)})))),this.url=e}return e(m,[{key:"sendMessage",value:function(e,n){return(k=k||t(o().mark((function e(t,n){var r;return o().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:try{this.dhServerSocket&&this.dhServerSocket&&this.dhServerSocket.request(t).on((function(e){n&&n(e)}))}catch(e){r="object"===c(t)&&"requestId"in t?t.requestId:a(),n&&n({code:-11,body:null,action:"RENDER_ERROR",requestId:r,message:JSON.stringify(e)})}case 1:case"end":return e.stop()}}),e,this)})))).apply(this,arguments)}}]),m}();export{k as default};
|
package/es/interface.d.ts
CHANGED
|
@@ -13,7 +13,6 @@ export interface ConnectParamsType {
|
|
|
13
13
|
cameraId?: string | number;
|
|
14
14
|
characterOffset?: string;
|
|
15
15
|
pickAudioMode?: 'pressButton' | 'free';
|
|
16
|
-
fullDuplex: boolean;
|
|
17
16
|
autoChromaKey?: boolean;
|
|
18
17
|
usingVideo?: boolean;
|
|
19
18
|
ttsPer?: string | number;
|
|
@@ -98,10 +97,11 @@ export interface DhRealtimeHumanType {
|
|
|
98
97
|
rtcServerUrl?: string;
|
|
99
98
|
appKey?: string;
|
|
100
99
|
appId?: string;
|
|
101
|
-
wrapperId
|
|
100
|
+
wrapperId?: string;
|
|
102
101
|
connectParams?: ConnectParamsType;
|
|
103
102
|
renderParams?: RenderParamsType;
|
|
104
103
|
rtcConnectParams?: RtcConnectionType;
|
|
105
104
|
brtcParams?: any;
|
|
105
|
+
isPreOpenSession?: boolean;
|
|
106
106
|
onDigitalHumanCallback: (data: CallbackMsgType) => void;
|
|
107
107
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bddh/starling-realtime-client",
|
|
3
|
-
"version": "2.0.0-beta.
|
|
3
|
+
"version": "2.0.0-beta.5",
|
|
4
4
|
"keywords": [
|
|
5
5
|
"react",
|
|
6
6
|
"indexDB"
|
|
@@ -44,5 +44,5 @@
|
|
|
44
44
|
"lodash": "^4.17.21",
|
|
45
45
|
"uuid": "^9.0.0"
|
|
46
46
|
},
|
|
47
|
-
"gitHead": "
|
|
47
|
+
"gitHead": "fd1f9d6d3ffacece6b340e8f102df30034c5e000"
|
|
48
48
|
}
|