fmode-ng 0.0.66 → 0.0.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm2022/lib/aigc/service-fmai/service-chat/chat-class.mjs +1 -1
- package/esm2022/lib/aigc/voice/stream.player.mjs +1 -1
- package/esm2022/lib/aigc/voice/tts/fmode-tts-class.mjs +1 -1
- package/fesm2022/fmode-ng.mjs +1 -1
- package/fesm2022/fmode-ng.mjs.map +1 -1
- package/lib/aigc/voice/stream.player.d.ts +1 -1
- package/package.json +1 -1
|
@@ -5,6 +5,6 @@
|
|
|
5
5
|
* 保留所有权利 All Rights Reserved.
|
|
6
6
|
* /home/ryan/workspace/nova/nova-admin/dist/fmode-ng/esm2022/lib/aigc/service-fmai/service-chat/chat-class.mjs
|
|
7
7
|
*/
|
|
8
|
-
import{bufferTime,concatMap,Observable,delay,finalize}from"rxjs";import Parse from"parse";import{AgentPrompt}from"../../agent";import{FmodeTTS}from"../../voice/tts";import{PromptTemplate}from"@langchain/core/prompts";const API_BASE="https://server.fmode.cn/api/apig/aigc/gpt",agentPrompt=new AgentPrompt,PromptTplTalkSSMLOutputCode="talk-ssml-output-tpl",PromptTplTalkTextSSMLCode="talk-text-ssml-tpl";export function getMessageContentText(t){let e="";return"string"==typeof t&&(e=t),"object"==typeof t&&(e=t?.find((t=>t?.text))?.text||""),e}export function getMessageImageUrl(t){return"object"==typeof t?t?.find((t=>t?.image_url))?.image_url?.url||"":null}export class FmodeChat{async loadModelList(t){if(this.modelList?.length)return;let e=new Parse.Query("ChatModel");e.notEqualTo("isDeleted",!0),e.equalTo("isEnabled",!0),e.addAscending("index"),this.modelList=await e.find(),this.currentModel=t||this.modelList?.find((t=>"fmode-4.5-128k"==t.get("code")))}showAvatar(){this.avatarConfig=this.role?.get("avatarConfig"),this.avatarConfig&&(this.isAvatarShow=!0,this.avatarConfig?.image&&(this.avatarConfig.image.waiting=this.avatarConfig.image.waiting||this.role?.get("thumb")||this.role?.get("avatar"),this.avatarMode="image"),this.avatarConfig?.video&&(this.avatarConfig.video.waiting=this.avatarConfig.video.waiting,this.avatarMode="video"))}scrollToBottom(t){t=t||this.scrollComp,t?.nativeElement?.scrollHeight&&(t.nativeElement.scrollTop=t.nativeElement.scrollHeight)}constructor(t,e,s,i,o,n,a){this.ChatSession=Parse.Object.extend("ChatSession"),this.messageList=[{role:"system",content:"系统提示：AI仅供参考"}],this.latestAIResponse="",this.userInput="",this.userImage="",this.isDirect=!1,this.mode="page",this.hideShare=!1,this.hideModalSelect=!1,this.hideInputPreview=!1,this.isAvatarShow=!1,this.avatarMode="",this.isPromptModalOpen=!1,this.isPromptMessageAreaShow=!0,this.promptList=[],this.focusUserInput=()=>{},this.leftButtons=[{title:"灵感",icon:"color-wand-outline",onClick:()=>{this.isPromptModalOpen=!0},show:()=>this?.promptList?.length},{title:"角色",icon:"people-outline",onClick:()=>{this.navCtrl?.navigateRoot("/chat/pro/mask")},show:()=>!0},{title:"呼叫",icon:"call-outline",onClick:()=>{this.chatServ?.callRole(this.role)},show:()=>this?.role?.get("voiceConfig")}],this.isVoiceInputMode=!1,this.isTexting=!1,this.isTalkMode=!1,this.SSMLRoleVoice="zh-CN-XiaoxiaoNeural",this.playAnimation=t=>{console.log(t)},this.welcome=async()=>{let t=this.messageList?.filter((t=>"assistant"==t?.role));if(t?.length)return;let e=Parse.User.current(),s=await this.loadSelf("Person","userVerify"),i=await this.loadSelf("Profile","user"),o=e?.get("nickname")||i?.get("name")||e?.get("realname")||e?.get("name"),n=s?.get("name")||s?.get("userVefiry")?.get("realname")||s?.get("userVefiry")?.get("nickname");s?.get("userVerify")?.id==e?.id&&(n="您");let a=this.role.get("voiceConfig")?.welcome?.prompt;if(this.role.get("voiceConfig")?.welcome?.promptList?.length){let t=this.role.get("voiceConfig")?.welcome?.promptList;a=t[Math.floor(Math.random()*t.length)]}if(!a)return;let r=await PromptTemplate.fromTemplate(a,{templateFormat:"mustache"}).format({name:o,userName:o,personName:n,timeOfDay:this.getTimeOfDay()}),l=await this.getVoiceByContentText(r),h={role:"assistant",voice:l,content:r,complete:!0};this.voiceMap[l?.id],this.playChatVoice(this.voiceMap[l?.id]),this.messageList.push(h)},this.self={},this.voiceMap={},this.VoiceTTSMap={},this.chatServ=i,this.role=e,this.sessionId=t,this.navCtrl=o,this.ncloud=n,this.uploadServ=a,s?.id&&(this.chatSession=s,this.messageList=this.chatSession.get("messageList"),this.sessionId=s?.id),this.role?.id&&(this.voiceConfig=this.role?.get("voiceConfig"),this.voiceConfig?.autoTalk&&(this.isTalkMode=!0,this.isDirect=!0))}getTimeOfDay(){const t=(new Date).getHours();return t>=5&&t<12?"早上":t>=12&&t<14?"中午":t>=14&&t<18?"下午":"晚上"}async loadSelf(t,e){if(this.self[t])return this.self[t];let s=Parse.User.current(),i=new Parse.Query(t);return i.include(e),i.equalTo(e,s?.id),this.self[t]=await i.first(),this.self[t]}async loadTalkSystemPrompt(t){if(!this.isTalkMode)return;if(!t)return;"男"==t?.get("gender")?this.SSMLRoleVoice="zh-CN-YunyeNeural":this.SSMLRoleVoice="zh-CN-XiaoxiaoNeural",this.SSMLRoleVoice=t?.get("voiceConfig")?.voice||this.SSMLRoleVoice;let e=await agentPrompt.getFormatTpl("talk-ssml-output-tpl",{SSMLRoleVoice:this.SSMLRoleVoice}),s=t.get("prompt")||"请你扮演飞码AI的人工智能专家。";s+=e;let i={role:"user",content:s,hidden:!0},o=this.messageList?.map((t=>t?.content)).join();if(o.indexOf(s)>-1)return;let n=this.messageList?.findIndex((t=>"system"==t?.role)),a=n+1;this.messageList.splice(a,0,i)}loadRolePrompt(){let t=this.role?.get("prompt"),e={role:"user",content:t,hidden:!0};if(!t)return;let s=this.messageList?.map((t=>t?.content)).join();if(s.indexOf(t)>-1)return;let i=this.messageList?.findIndex((t=>"system"==t?.role)),o=i+1;this.messageList.splice(o,0,e)}async sendMessage(t="FmodeAiTest测试问题",e,s,i,o){if(this.scrollToBottom&&this.scrollToBottom(),this.isPromptMessageAreaShow=!1,this.loadRolePrompt(),e){let s={role:"user",content:[{type:"image_url",image_url:{url:e}},{type:"text",text:t}],complete:!0,createdAt:new Date};o&&(s.voice={id:o?.id}),this.messageList.push({role:"user",content:[{type:"image_url",image_url:{url:e}},{type:"text",text:t}],complete:!0,createdAt:new Date})}else{let e={role:"user",content:t,complete:!0,createdAt:new Date};o&&(e.voice={id:o?.id,duration:o?.duration}),this.messageList.push(e)}let n=new FmodeChatCompletion(this.fixMessageList(this.messageList),{model:this.chatServ?.currentModel?.get("code")||"fmode-4.5-128k"});if(this.onUserSend){if(!await this.onUserSend(this,this.messageList[this.messageList?.length-1]))return}this.userInput="",this.userImage="";let a=this.isDirect||!1;this.isTalkMode&&(a=!0);let r=n.sendCompletion({isDirect:a,onComplete:s||null}).pipe(finalize((async()=>{if(this.isTalkMode){let t=this.messageList[n.indexOfList]?.content,e=await this.getVoiceByContentText(t,i);i?.onSSMLComplete&&i?.onSSMLComplete(e),this.messageList[n.indexOfList].voice=e,this.playChatVoice(this.voiceMap[e?.id])}this.messageList[n.indexOfList].complete=!0}))).subscribe((t=>{this.messageList[n.indexOfList]=t,this.latestAIResponse=this.getContentText(t?.content);let e=this.chatSession?.get("messageList")?.length;this.messageList?.length>e&&this.saveChatSession(),t?.complete&&(this.onMessage&&this.onMessage(this,this.messageList[this.messageList?.length-1]),this.saveChatSession(),r.unsubscribe()),this.scrollToBottom&&this.scrollToBottom()}))}getVoiceByContentText(t,e,s=!1){let i=this.getContentText(t),o=new(Parse.Object.extend("ChatVoice")),n="";return this.SSMLRoleVoice=this.role?.get("voiceConfig")?.voice||this.SSMLRoleVoice,new Promise((async(t,e)=>{let resolveChatVoice=async()=>{o.set("content",i),o.set("ssml",n),o.set("role","assistant");let e=localStorage.getItem("company");e&&o.set("company",{__type:"Pointer",className:"Company",objectId:e}),Parse.User.current()?.id&&o.set("user",Parse.User.current().toPointer()),this.chatSession?.id&&o.set("session",this.chatSession?.toPointer()),o=await o.save(),this.voiceMap[o?.id]=o,t({id:o?.id})};if(0==s&&(n=`<speak xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" version="1.0" xml:lang="zh-CN"><voice name="${this.SSMLRoleVoice}">${i}</voice></speak>`,resolveChatVoice()),1==s){let t=await agentPrompt.getFormatTpl("talk-text-ssml-tpl",{content:i,SSMLRoleVoice:this.SSMLRoleVoice});new FmodeChatCompletion(this.fixMessageList([{role:"user",content:t}]),{model:this.chatServ?.currentModel?.get("code")||"fmode-4.5-128k"}).sendCompletion({isDirect:!0}).subscribe((async t=>{t?.complete&&(n=this.getContentText(t?.content),resolveChatVoice())}))}}))}getContentText(t){return"string"==typeof t?t:t?.[0]?.text||""}async initTTS(){let t=await this.ncloud.apig("voice/tts/token",{company:localStorage.getItem("company")});if(console.log(t),t?.token){return new FmodeTTS(t,this.uploadServ)}return null}stopPlayingVoice(){Object.values(this.VoiceTTSMap).forEach((t=>{t?.isPlaying&&t?.stop()}))}async playChatVoice(t,e){let s=await this.initTTS();if(s){try{this.playAnimation("talking"),s.speakAsync(t?.get("ssml"),t,{onStart:t=>{e?.onStart&&e?.onStart(t)},onLoaded:t=>{e?.onLoaded&&e?.onLoaded(t)},onStop:()=>{e?.onStop&&e?.onStop(),this.playAnimation("waiting")}})}catch(t){console.error(t)}return this.VoiceTTSMap[t.id]=s,s}return null}async saveChatSession(){if("new"==this.sessionId&&(this.chatSession=new this.ChatSession),this.chatSession.set("title",this.genTitle()),this.chatSession.set("role",this.role?.toPointer()),this.chatSession.set("messageList",this.messageList),this.chatSession.set("user",Parse.User.current()?.toPointer()),this.chatSession=await this.chatSession.save(),this.onChatSaved&&this.onChatSaved(this),this.sessionId=this.chatSession?.id,this.sessionId){let t=`${window.location.origin}/chat/pro/chat/${this.sessionId}`;window.location?.pathname?.indexOf("chat/session")>-1&&(t=`${window.location.origin}/chat/session/chat/${this.sessionId}`),"modal"==this.mode&&(t=window.location.href),t=this.getInviteUrl(t),window.history.replaceState(null,null,t+window.location.search);let e={sid:this.chatSession?.id,rid:this.role?.id,name:this.role?.get("name"),message:this.chatSession?.get("messageList")?.[this.chatSession?.get("messageList")?.length-1]?.content?.slice(0,20),latest:this.chatSession?.createdAt};this.chatServ?.chatList?.length||(this.chatServ.chatList=[]);let s=this.chatServ?.chatList?.find((t=>t?.sid==e?.sid));s>-1?this.chatServ.chatList[s]=e:this.chatServ?.chatList.unshift(e)}}getInviteUrl(t){let e=new URL(t),s=Parse.User?.current()?.id;return e.searchParams.set("invite",s),e.href}genTitle(){if(this.title)return this.title;let t=this.messageList.find((t=>"user"==t.role))?.content;return"string"==typeof t&&(this.title=t?.slice(0,15)||""),"object"==typeof t&&(this.title=t?.find((t=>t?.text))?.text||""),this.title}fixMessageList(t){return t.map((t=>({role:t.role,content:t.content})))}nowStr(){let t=new Date;return`${t.getFullYear()}/${t.getMonth()+1}/${t.getDate()} ${t.getHours()}:${t.getMinutes()}:${t.getSeconds()}`}}export class FmodeChatCompletion{constructor(t,e){this.content="",this.contentBuffer=[],this.isCompleted=!1,this.indexOfList=Number(t.length),this.messages=t,this.model=e?.model||"fmode-4.5-128k"}sendCompletion(t={}){t.intTime=t?.intTime||50,t.isDirect=t?.isDirect||!1,t?.isDirect&&(t.intTime=1);let e={messages:this.messages,stream:!0,model:this.model,temperature:.5,presence_penalty:0,frequency_penalty:0};return new Observable((s=>{let i=RequestFmodeChatApi("/v1/chat/completions",e).subscribe((e=>{let o=String(e);if("data: [DONE]"==o&&(this.isCompleted=!0,t?.isDirect&&this.isCompleted&&(s.next({role:"assistant",content:this.content,complete:!0,createdAt:new Date}),i.unsubscribe(),t?.onComplete&&t.onComplete({role:"assistant",content:this.content,complete:!0,createdAt:new Date}),s.complete())),o.indexOf("data: {")>-1){let e=chunkToJson(o),n=e?.choices?.[0]?.delta?.content||"";this.contentBuffer.push(n),t?.isDirect&&(this.content+=n||"",this.isCompleted||s.next({role:"assistant",cid:e?.id,content:this.content,createdAt:new Date})),t?.isDirect||this.contentPusher||(this.contentPusher=setInterval((()=>{this.isCompleted&&0==this.contentBuffer?.length&&(s.next({role:"assistant",cid:e?.id,content:this.content,complete:!0,createdAt:new Date}),i.unsubscribe(),clearInterval(this.contentPusher),s.complete()),this.contentBuffer?.length>=0&&(this.contentBuffer?.length>0&&(this.content+=this.contentBuffer.shift()),s.next({role:"assistant",cid:e?.id,content:this.content,createdAt:new Date}))}),t?.intTime))}}))})).pipe(bufferTime(100),concatMap((t=>t)),delay(200))}}function chunkToJson(t){let e;try{e=JSON.parse(t.replaceAll("data: ",""))}catch(t){console.error(t)}return e||{}}function RequestFmodeChatApi(t,e,s="POST"){return new Observable((i=>{let o=API_BASE+t,n=`Bearer ${Parse.User.current()?.getSessionToken()||localStorage.getItem("FMODE_AI_TOKEN")}`;return e.token=n,e&&(e=JSON.stringify(e)),fetch(o,{headers:{"Content-Type":"text/plain; charset=utf-8","Cache-Control":"no-cache"},body:e||null,method:s,credentials:"omit",mode:"cors"}).then((t=>{let e="";{let s=t.body?.getReader();const o=new TextDecoder;let n=new ReadableStream({start(t){!function read(){s.read().then((({done:e,value:s})=>{if(e)return t.close(),void i.complete();t.enqueue(s),read()}))}()}}).getReader();n.read().then((function processStream({done:t,value:s}){if(t)return;!function processData(t){let s=(e+t).split("\n");if(s?.length>1){for(let t=0;t<s.length-1;t++){let e=s[t];i.next(e)}e=s[s.length-1]}}(o.decode(s)),n.read().then(processStream)}))}})).catch((t=>i.error(t))),()=>{}}))}function JsonToFormData(t){const e=new FormData;return function appendFormData(t,s=""){Array.isArray(t)?t.forEach(((t,e)=>{appendFormData(t,`${s}[${e}]`)})):"object"==typeof t&&null!==t?Object.keys(t).forEach((e=>{const i=s?`${s}.${e}`:e;appendFormData(t[e],i)})):e.append(s,t)}(t),e}
|
|
8
|
+
import{bufferTime,concatMap,Observable,delay,finalize}from"rxjs";import Parse from"parse";import{AgentPrompt}from"../../agent";import{FmodeTTS}from"../../voice/tts";import{PromptTemplate}from"@langchain/core/prompts";const API_BASE="https://server.fmode.cn/api/apig/aigc/gpt",agentPrompt=new AgentPrompt,PromptTplTalkSSMLOutputCode="talk-ssml-output-tpl",PromptTplTalkTextSSMLCode="talk-text-ssml-tpl";export function getMessageContentText(t){let e="";return"string"==typeof t&&(e=t),"object"==typeof t&&(e=t?.find((t=>t?.text))?.text||""),e}export function getMessageImageUrl(t){return"object"==typeof t?t?.find((t=>t?.image_url))?.image_url?.url||"":null}export class FmodeChat{async loadModelList(t){if(this.modelList?.length)return;let e=new Parse.Query("ChatModel");e.notEqualTo("isDeleted",!0),e.equalTo("isEnabled",!0),e.addAscending("index"),this.modelList=await e.find(),this.currentModel=t||this.modelList?.find((t=>"fmode-4.5-128k"==t.get("code")))}showAvatar(){this.avatarConfig=this.role?.get("avatarConfig"),this.avatarConfig&&(this.isAvatarShow=!0,this.avatarConfig?.image&&(this.avatarConfig.image.waiting=this.avatarConfig.image.waiting||this.role?.get("thumb")||this.role?.get("avatar"),this.avatarMode="image"),this.avatarConfig?.video&&(this.avatarConfig.video.waiting=this.avatarConfig.video.waiting,this.avatarMode="video"))}scrollToBottom(t){t=t||this.scrollComp,t?.nativeElement?.scrollHeight&&(t.nativeElement.scrollTop=t.nativeElement.scrollHeight)}constructor(t,e,s,i,o,n,a){this.ChatSession=Parse.Object.extend("ChatSession"),this.messageList=[{role:"system",content:"系统提示：AI仅供参考"}],this.latestAIResponse="",this.userInput="",this.userImage="",this.isDirect=!1,this.mode="page",this.hideShare=!1,this.hideModalSelect=!1,this.hideInputPreview=!1,this.isAvatarShow=!1,this.avatarMode="",this.isPromptModalOpen=!1,this.isPromptMessageAreaShow=!0,this.promptList=[],this.focusUserInput=()=>{},this.leftButtons=[{title:"灵感",icon:"color-wand-outline",onClick:()=>{this.isPromptModalOpen=!0},show:()=>this?.promptList?.length},{title:"角色",icon:"people-outline",onClick:()=>{this.navCtrl?.navigateRoot("/chat/pro/mask")},show:()=>!0},{title:"呼叫",icon:"call-outline",onClick:()=>{this.chatServ?.callRole(this.role)},show:()=>this?.role?.get("voiceConfig")}],this.isVoiceInputMode=!1,this.isTexting=!1,this.isTalkMode=!1,this.SSMLRoleVoice="zh-CN-XiaoxiaoNeural",this.playAnimation=t=>{console.log(t)},this.welcome=async()=>{let t=this.messageList?.filter((t=>"assistant"==t?.role));if(t?.length)return;let e=Parse.User.current(),s=await this.loadSelf("Person","userVerify"),i=await this.loadSelf("Profile","user"),o=e?.get("nickname")||i?.get("name")||e?.get("realname")||e?.get("name"),n=s?.get("name")||s?.get("userVefiry")?.get("realname")||s?.get("userVefiry")?.get("nickname");s?.get("userVerify")?.id==e?.id&&(n="您");let a=this.role.get("voiceConfig")?.welcome?.prompt;if(this.role.get("voiceConfig")?.welcome?.promptList?.length){let t=this.role.get("voiceConfig")?.welcome?.promptList;a=t[Math.floor(Math.random()*t.length)]}if(!a)return;let r=await PromptTemplate.fromTemplate(a,{templateFormat:"mustache"}).format({name:o,userName:o,personName:n,timeOfDay:this.getTimeOfDay()}),l=await this.getVoiceByContentText(r),h={role:"assistant",voice:l,content:r,complete:!0};this.voiceMap[l?.id],this.playChatVoice(this.voiceMap[l?.id]),this.messageList.push(h)},this.self={},this.voiceMap={},this.VoiceTTSMap={},this.chatServ=i,this.role=e,this.sessionId=t,this.navCtrl=o,this.ncloud=n,this.uploadServ=a,s?.id&&(this.chatSession=s,this.messageList=this.chatSession.get("messageList"),this.sessionId=s?.id),this.role?.id&&(this.voiceConfig=this.role?.get("voiceConfig"),this.voiceConfig?.autoTalk&&(this.isTalkMode=!0,this.isDirect=!0))}getTimeOfDay(){const t=(new Date).getHours();return t>=5&&t<12?"早上":t>=12&&t<14?"中午":t>=14&&t<18?"下午":"晚上"}async loadSelf(t,e){if(this.self[t])return this.self[t];let s=Parse.User.current(),i=new Parse.Query(t);return i.include(e),i.equalTo(e,s?.id),this.self[t]=await i.first(),this.self[t]}async loadTalkSystemPrompt(t){if(!this.isTalkMode)return;if(!t)return;"男"==t?.get("gender")?this.SSMLRoleVoice="zh-CN-YunyeNeural":this.SSMLRoleVoice="zh-CN-XiaoxiaoNeural",this.SSMLRoleVoice=t?.get("voiceConfig")?.voice||this.SSMLRoleVoice;let e=await agentPrompt.getFormatTpl("talk-ssml-output-tpl",{SSMLRoleVoice:this.SSMLRoleVoice}),s=t.get("prompt")||"请你扮演飞码AI的人工智能专家。";s+=e;let i={role:"user",content:s,hidden:!0},o=this.messageList?.map((t=>t?.content)).join();if(o.indexOf(s)>-1)return;let n=this.messageList?.findIndex((t=>"system"==t?.role)),a=n+1;this.messageList.splice(a,0,i)}loadRolePrompt(){let t=this.role?.get("prompt"),e={role:"user",content:t,hidden:!0};if(!t)return;let s=this.messageList?.map((t=>t?.content)).join();if(s.indexOf(t)>-1)return;let i=this.messageList?.findIndex((t=>"system"==t?.role)),o=i+1;this.messageList.splice(o,0,e)}async sendMessage(t="FmodeAiTest测试问题",e,s,i,o){if(this.scrollToBottom&&this.scrollToBottom(),this.isPromptMessageAreaShow=!1,this.loadRolePrompt(),e){let s={role:"user",content:[{type:"image_url",image_url:{url:e}},{type:"text",text:t}],complete:!0,createdAt:new Date};o&&(s.voice={id:o?.id}),this.messageList.push({role:"user",content:[{type:"image_url",image_url:{url:e}},{type:"text",text:t}],complete:!0,createdAt:new Date})}else{let e={role:"user",content:t,complete:!0,createdAt:new Date};o&&(e.voice={id:o?.id,duration:o?.duration}),this.messageList.push(e)}let n=new FmodeChatCompletion(this.fixMessageList(this.messageList),{model:this.currentModel?.get("code")||"fmode-4.5-128k"});if(this.onUserSend){if(!await this.onUserSend(this,this.messageList[this.messageList?.length-1]))return}this.userInput="",this.userImage="";let a=this.isDirect||!1;this.isTalkMode&&(a=!0);let r=n.sendCompletion({isDirect:a,onComplete:s||null}).pipe(finalize((async()=>{if(this.isTalkMode){let t=this.messageList[n.indexOfList]?.content,e=await this.getVoiceByContentText(t,i);i?.onSSMLComplete&&i?.onSSMLComplete(e),this.messageList[n.indexOfList].voice=e,this.playChatVoice(this.voiceMap[e?.id])}this.messageList[n.indexOfList].complete=!0}))).subscribe((t=>{this.messageList[n.indexOfList]=t,this.latestAIResponse=this.getContentText(t?.content);let e=this.chatSession?.get("messageList")?.length;this.messageList?.length>e&&this.saveChatSession(),t?.complete&&(this.onMessage&&this.onMessage(this,this.messageList[this.messageList?.length-1]),this.saveChatSession(),r.unsubscribe()),this.scrollToBottom&&this.scrollToBottom()}))}getVoiceByContentText(t,e,s=!1){let i=this.getContentText(t),o=new(Parse.Object.extend("ChatVoice")),n="";return this.SSMLRoleVoice=this.role?.get("voiceConfig")?.voice||this.SSMLRoleVoice,new Promise((async(t,e)=>{let resolveChatVoice=async()=>{o.set("content",i),o.set("ssml",n),o.set("role","assistant");let e=localStorage.getItem("company");e&&o.set("company",{__type:"Pointer",className:"Company",objectId:e}),Parse.User.current()?.id&&o.set("user",Parse.User.current().toPointer()),this.chatSession?.id&&o.set("session",this.chatSession?.toPointer()),o=await o.save(),this.voiceMap[o?.id]=o,t({id:o?.id})};if(0==s&&(n=`<speak xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" version="1.0" xml:lang="zh-CN"><voice name="${this.SSMLRoleVoice}">${i}</voice></speak>`,resolveChatVoice()),1==s){let t=await agentPrompt.getFormatTpl("talk-text-ssml-tpl",{content:i,SSMLRoleVoice:this.SSMLRoleVoice});new FmodeChatCompletion(this.fixMessageList([{role:"user",content:t}]),{model:this.currentModel?.get("code")||"fmode-4.5-128k"}).sendCompletion({isDirect:!0}).subscribe((async t=>{t?.complete&&(n=this.getContentText(t?.content),resolveChatVoice())}))}}))}getContentText(t){return"string"==typeof t?t:t?.[0]?.text||""}async initTTS(){let t=await this.ncloud.apig("voice/tts/token",{company:localStorage.getItem("company")});if(console.log(t),t?.token){return new FmodeTTS(t,this.uploadServ)}return null}stopPlayingVoice(){Object.values(this.VoiceTTSMap).forEach((t=>{t?.isPlaying&&t?.stop()}))}async playChatVoice(t,e){let s=await this.initTTS();if(s){try{this.playAnimation("talking"),s.speakAsync(t?.get("ssml"),t,{onStart:t=>{e?.onStart&&e?.onStart(t)},onLoaded:t=>{e?.onLoaded&&e?.onLoaded(t)},onStop:()=>{e?.onStop&&e?.onStop(),this.playAnimation("waiting")}})}catch(t){console.error(t)}return this.VoiceTTSMap[t.id]=s,s}return null}async saveChatSession(){if("new"==this.sessionId&&(this.chatSession=new this.ChatSession),this.chatSession.set("title",this.genTitle()),this.chatSession.set("role",this.role?.toPointer()),this.chatSession.set("messageList",this.messageList),this.chatSession.set("user",Parse.User.current()?.toPointer()),this.chatSession=await this.chatSession.save(),this.onChatSaved&&this.onChatSaved(this),this.sessionId=this.chatSession?.id,this.sessionId){let t=`${window.location.origin}/chat/pro/chat/${this.sessionId}`;window.location?.pathname?.indexOf("chat/session")>-1&&(t=`${window.location.origin}/chat/session/chat/${this.sessionId}`),"modal"==this.mode&&(t=window.location.href),t=this.getInviteUrl(t),window.history.replaceState(null,null,t+window.location.search);let e={sid:this.chatSession?.id,rid:this.role?.id,name:this.role?.get("name"),message:this.chatSession?.get("messageList")?.[this.chatSession?.get("messageList")?.length-1]?.content?.slice(0,20),latest:this.chatSession?.createdAt};this.chatServ&&!this.chatServ?.chatList?.length&&(this.chatServ.chatList=[]);let s=this.chatServ?.chatList?.find((t=>t?.sid==e?.sid));s>-1?this.chatServ.chatList[s]=e:this.chatServ?.chatList.unshift(e)}}getInviteUrl(t){let e=new URL(t),s=Parse.User?.current()?.id;return e.searchParams.set("invite",s),e.href}genTitle(){if(this.title)return this.title;let t=this.messageList.find((t=>"user"==t.role))?.content;return"string"==typeof t&&(this.title=t?.slice(0,15)||""),"object"==typeof t&&(this.title=t?.find((t=>t?.text))?.text||""),this.title}fixMessageList(t){return t.map((t=>({role:t.role,content:t.content})))}nowStr(){let t=new Date;return`${t.getFullYear()}/${t.getMonth()+1}/${t.getDate()} ${t.getHours()}:${t.getMinutes()}:${t.getSeconds()}`}}export class FmodeChatCompletion{constructor(t,e){this.content="",this.contentBuffer=[],this.isCompleted=!1,this.indexOfList=Number(t.length),this.messages=t,this.model=e?.model||"fmode-4.5-128k"}sendCompletion(t={}){t.intTime=t?.intTime||50,t.isDirect=t?.isDirect||!1,t?.isDirect&&(t.intTime=1);let e={messages:this.messages,stream:!0,model:this.model,temperature:.5,presence_penalty:0,frequency_penalty:0};return new Observable((s=>{let i=RequestFmodeChatApi("/v1/chat/completions",e).subscribe((e=>{let o=String(e);if("data: [DONE]"==o&&(this.isCompleted=!0,t?.isDirect&&this.isCompleted&&(s.next({role:"assistant",content:this.content,complete:!0,createdAt:new Date}),i.unsubscribe(),t?.onComplete&&t.onComplete({role:"assistant",content:this.content,complete:!0,createdAt:new Date}),s.complete())),o.indexOf("data: {")>-1){let e=chunkToJson(o),n=e?.choices?.[0]?.delta?.content||"";this.contentBuffer.push(n),t?.isDirect&&(this.content+=n||"",this.isCompleted||s.next({role:"assistant",cid:e?.id,content:this.content,createdAt:new Date})),t?.isDirect||this.contentPusher||(this.contentPusher=setInterval((()=>{this.isCompleted&&0==this.contentBuffer?.length&&(s.next({role:"assistant",cid:e?.id,content:this.content,complete:!0,createdAt:new Date}),i.unsubscribe(),clearInterval(this.contentPusher),s.complete()),this.contentBuffer?.length>=0&&(this.contentBuffer?.length>0&&(this.content+=this.contentBuffer.shift()),s.next({role:"assistant",cid:e?.id,content:this.content,createdAt:new Date}))}),t?.intTime))}}))})).pipe(bufferTime(100),concatMap((t=>t)),delay(200))}}function chunkToJson(t){let e;try{e=JSON.parse(t.replaceAll("data: ",""))}catch(t){console.error(t)}return e||{}}function RequestFmodeChatApi(t,e,s="POST"){return new Observable((i=>{let o=API_BASE+t,n=`Bearer ${Parse.User.current()?.getSessionToken()||localStorage.getItem("FMODE_AI_TOKEN")}`;return e.token=n,e&&(e=JSON.stringify(e)),fetch(o,{headers:{"Content-Type":"text/plain; charset=utf-8","Cache-Control":"no-cache"},body:e||null,method:s,credentials:"omit",mode:"cors"}).then((t=>{let e="";{let s=t.body?.getReader();const o=new TextDecoder;let n=new ReadableStream({start(t){!function read(){s.read().then((({done:e,value:s})=>{if(e)return t.close(),void i.complete();t.enqueue(s),read()}))}()}}).getReader();n.read().then((function processStream({done:t,value:s}){if(t)return;!function processData(t){let s=(e+t).split("\n");if(s?.length>1){for(let t=0;t<s.length-1;t++){let e=s[t];i.next(e)}e=s[s.length-1]}}(o.decode(s)),n.read().then(processStream)}))}})).catch((t=>i.error(t))),()=>{}}))}function JsonToFormData(t){const e=new FormData;return function appendFormData(t,s=""){Array.isArray(t)?t.forEach(((t,e)=>{appendFormData(t,`${s}[${e}]`)})):"object"==typeof t&&null!==t?Object.keys(t).forEach((e=>{const i=s?`${s}.${e}`:e;appendFormData(t[e],i)})):e.append(s,t)}(t),e}
|
|
9
9
|
var MODULE_PATH_NEED = `6K+l5paH5Lu25piv5pys6aG555uu55qE5LiA6YOo5YiGIFRoaXMgZmlsZSBpcyBwYXJ0IG9mIHRoZSBDb21wb25lbnRzIGluIEZtb2RlIEluYy4KICAgIOeJiOadg+aJgOaciSDCqSDmnKrmnaXpo57pqawgwqkg5rGf6KW/6ISR5o6n56eR5oqA5pyJ6ZmQ5YWs5Y+4IENvcHlyaWdodCDCqSBGbW9kZSBUZWNobm9sb2d5IENvLiwgTHRkLgogICAg5L+d55WZ5omA5pyJ5p2D5YipIEFsbCBSaWdodHMgUmVzZXJ2ZWQuCiAgICDkuKXnpoHlnKjmnKrnu4/mjojmnYPnmoTmg4XlhrXkuIvvvIzpgJrov4fku7vkvZXlqpLku4vlpI3liLbmraTmlofku7YgVW5hdXRob3JpemVkIGNvcHlpbmcgb2YgdGhpcyBmaWxlLCB2aWEgYW55IG1lZGl1bSBpcyBzdHJpY3RseSBwcm9oaWJpdGVkCiAgICDor6Xmlofku7bmmK/kuJPmnInnmoTmnLrlr4bmlofku7YgUHJvcHJpZXRhcnkgYW5kIGNvbmZpZGVudGlhbAogICAKICAgIENvcHlyaWdodCAyMDIxLW5vdyBGbW9kZSBJbmMuIHN1cHBvcnRAZm1vZGUuY24uIDE4NjA3MDA3MDczLgogICAg5L+d55WZ5omA5pyJ5p2D5YipIEFsbCByaWdodHMgcmVzZXJ2ZWQuCgogICAgUEFUSDovaG9tZS9yeWFuL3dvcmtzcGFjZS9ub3ZhL25vdmEtYWRtaW4vZGlzdC9mbW9kZS1uZy9lc20yMDIyL2xpYi9haWdjL3NlcnZpY2UtZm1haS9zZXJ2aWNlLWNoYXQvY2hhdC1jbGFzcy5tanM=`
|
|
10
10
|
|
|
@@ -5,6 +5,6 @@
|
|
|
5
5
|
* 保留所有权利 All Rights Reserved.
|
|
6
6
|
* /home/ryan/workspace/nova/nova-admin/dist/fmode-ng/esm2022/lib/aigc/voice/stream.player.mjs
|
|
7
7
|
*/
|
|
8
|
-
import{PushAudioOutputStreamCallback}from"microsoft-cognitiveservices-speech-sdk
|
|
8
|
+
import{PushAudioOutputStreamCallback}from"microsoft-cognitiveservices-speech-sdk";import{Subject}from"rxjs";import{bufferWhen,filter,takeUntil}from"rxjs/operators";export class FmPushAudioOutputStreamCallback extends PushAudioOutputStreamCallback{constructor(){super(),this.audioDataSubject=new Subject,this.playedSubject=new Subject,this.isPlaying=!1,this.isClosed=!1,this.audioBufferQueue=[],this.maxBufferSize=10,this.writeCount=0,this.audioContext=new(window.AudioContext||window.webkitAudioContext),this.audioDataSubject.pipe(bufferWhen((()=>this.playedSubject)),takeUntil(this.audioDataSubject.pipe(filter((()=>this.isClosed))))).subscribe((t=>{this.playAudio(t)}))}async write(t){this.writeCount++,10==this.writeCount&&this.playedSubject.next(!0),this.audioDataSubject.next(t)}async playAudio(t){this.isPlaying=!0;const e=this.mergeArrayBuffers(t);let i;try{i=await this.audioContext.decodeAudioData(e)}catch(t){}if(i){const t=this.audioContext.createBufferSource();t.buffer=i,t.connect(this.audioContext.destination),t.start(),await new Promise((e=>{t.onended=e})),this.playedSubject.next(!0)}this.isPlaying=!1}mergeArrayBuffers(t){const e=t.reduce(((t,e)=>t+e.byteLength),0),i=new Uint8Array(e);let s=0;for(const e of t){const t=new Uint8Array(e);i.set(t,s),s+=t.length}return i.buffer}async stop(){this.isClosed=!0,this.audioDataSubject&&this.audioDataSubject.unsubscribe(),this.playedSubject&&this.playedSubject.unsubscribe(),this.audioContext&&this.audioContext.close()}async close(){console.log("close 数据加载完成"),this.isClosed=!0}}
|
|
9
9
|
var MODULE_PATH_NEED = `6K+l5paH5Lu25piv5pys6aG555uu55qE5LiA6YOo5YiGIFRoaXMgZmlsZSBpcyBwYXJ0IG9mIHRoZSBDb21wb25lbnRzIGluIEZtb2RlIEluYy4KICAgIOeJiOadg+aJgOaciSDCqSDmnKrmnaXpo57pqawgwqkg5rGf6KW/6ISR5o6n56eR5oqA5pyJ6ZmQ5YWs5Y+4IENvcHlyaWdodCDCqSBGbW9kZSBUZWNobm9sb2d5IENvLiwgTHRkLgogICAg5L+d55WZ5omA5pyJ5p2D5YipIEFsbCBSaWdodHMgUmVzZXJ2ZWQuCiAgICDkuKXnpoHlnKjmnKrnu4/mjojmnYPnmoTmg4XlhrXkuIvvvIzpgJrov4fku7vkvZXlqpLku4vlpI3liLbmraTmlofku7YgVW5hdXRob3JpemVkIGNvcHlpbmcgb2YgdGhpcyBmaWxlLCB2aWEgYW55IG1lZGl1bSBpcyBzdHJpY3RseSBwcm9oaWJpdGVkCiAgICDor6Xmlofku7bmmK/kuJPmnInnmoTmnLrlr4bmlofku7YgUHJvcHJpZXRhcnkgYW5kIGNvbmZpZGVudGlhbAogICAKICAgIENvcHlyaWdodCAyMDIxLW5vdyBGbW9kZSBJbmMuIHN1cHBvcnRAZm1vZGUuY24uIDE4NjA3MDA3MDczLgogICAg5L+d55WZ5omA5pyJ5p2D5YipIEFsbCByaWdodHMgcmVzZXJ2ZWQuCgogICAgUEFUSDovaG9tZS9yeWFuL3dvcmtzcGFjZS9ub3ZhL25vdmEtYWRtaW4vZGlzdC9mbW9kZS1uZy9lc20yMDIyL2xpYi9haWdjL3ZvaWNlL3N0cmVhbS5wbGF5ZXIubWpz`
|
|
10
10
|
|
|
@@ -5,6 +5,6 @@
|
|
|
5
5
|
* 保留所有权利 All Rights Reserved.
|
|
6
6
|
* /home/ryan/workspace/nova/nova-admin/dist/fmode-ng/esm2022/lib/aigc/voice/tts/fmode-tts-class.mjs
|
|
7
7
|
*/
|
|
8
|
-
import{SpeechSynthesizer}from"microsoft-cognitiveservices-speech-sdk
|
|
8
|
+
import{SpeechSynthesizer}from"microsoft-cognitiveservices-speech-sdk";import{SpeechConfig}from"microsoft-cognitiveservices-speech-sdk";import{ResultReason}from"microsoft-cognitiveservices-speech-sdk";import{AudioConfig}from"microsoft-cognitiveservices-speech-sdk";import Parse from"parse";import{AudioPlayer}from"../audio.player";import{FmPushAudioOutputStreamCallback}from"../stream.player";export class FmodeTTS{stop(){this.audioPlayer&&this.audioPlayer?.stop(),this.audioStream&&this.audioStream?.stop()}constructor(e,t){this.isPlaying=!1,this.uploadServ=t,this.subscriptionKey=e?.subscriptionKey,this.authorizationToken=e?.token,this.region=e?.region,this.subscriptionKey&&(this.speechConfig=SpeechConfig.fromSubscription(this.subscriptionKey,this.region)),this.authorizationToken&&(this.speechConfig=SpeechConfig.fromAuthorizationToken(this.authorizationToken,this.region)),this.audioStream=new FmPushAudioOutputStreamCallback,this.audioConfig=AudioConfig.fromStreamOutput(this.audioStream),this.synthesizer=new SpeechSynthesizer(this.speechConfig,this.audioConfig),this.audioPlayer=AudioPlayer.getInstance()}extractTextFromXML(e){const t=/>([^<]+)</g;let o,i=[];for(;null!==(o=t.exec(e));){const e=o[1].trim();e&&i.push(e)}return i.join(" ")}extractSSMLContent(e){var t=e.match(/<speak.*?<\/speak>/s);return t?t[0]:e}async speakAsync(e,t,o){if(!t){let o=Parse.Object.extend("ChatVoice");(t=new o).set("ssml",e),t.set("content",this.extractTextFromXML(e));let i=localStorage.getItem("company");i&&t.set("company",{__type:"Pointer",className:"Company",objectId:i}),Parse.User.current()?.id&&t.set("user",Parse.User.current().toPointer())}if(!t?.get("voiceFile")){let o=[];t?.get("content")&&o.push({content:t?.get("content")}),e&&o.push({ssml:e});let i=Parse.Query.fromJSON("ChatVoice",{include:"voiceFile",where:{$or:o}}),s=await i.first();t.set("voiceFile",s?.get("voiceFile"))}return t?.get("voiceFile")?(this.playAudioData(t?.get("voiceFile")?.get("url"),t,o),t):(e=this.extractSSMLContent(e),this.isPlaying=!0,new Promise(((i,s)=>{const n=Date.now();let a="speakTextAsync";e?.indexOf("<")>-1&&(a="speakSsmlAsync"),e=this.fixMarkdownChar(e),this.eventMap?.onSpeakBefore&&this.eventMap?.onSpeakBefore(),this.synthesizer[a](e,(e=>{if(console.log(e),e.reason===ResultReason.SynthesizingAudioStarted&&(this.isPlaying=!0),e.reason===ResultReason.SynthesizingAudioCompleted){this.eventMap?.onAudioCompleted&&this.eventMap?.onAudioCompleted();const s=Date.now();let a=e?.audioData;t.set("duration",Number(e?.audioDuration)/1e4),this.playAudioData(a,t,o),console.log(`Audio synthesis finished. Duration: ${s-n} ms`),setTimeout((()=>{this.isPlaying=!1,o?.onStop()&&o?.onStop()}),2e3),i(t)}else s(`Speech synthesis failed. Reason: ${e.errorDetails}`)}),(e=>{s(`Error occurred during synthesis: ${e}`)}))})))}onBreak(e){}fixMarkdownChar(e){return e.replaceAll("*","")}async playAudioData(e,t,o){let i;if(console.log("audioData",e),!(e?.indexOf&&e?.indexOf("http")>=-1)){let o=new Blob([e],{type:"audio/wav"});return i=URL.createObjectURL(o),void this.uploadAndSaveVoice(o,t)}i=e;let s=this.audioPlayer;o?.onStart&&o?.onStart(t),this.isPlaying=!0,s.setAudioEvent("onloadeddata",(()=>{let e=1e3*s.duration;console.log("duration",e),t?.get("duration")||(t?.set("duration",e),t?.save()),o?.onLoaded&&o?.onLoaded(s)})),console.log("event setting"),s.setAudioEvent("onabort",(()=>{this.isPlaying=!1,o?.onStop&&o?.onStop()})),s.setAudioEvent("onerror",(()=>{this.isPlaying=!1,o?.onStop&&o?.onStop()})),s.setAudioEvent("onpause",(()=>{this.isPlaying=!1,o?.onStop&&o?.onStop()})),s.setAudioEvent("onended",(()=>{this.isPlaying=!1,o?.onStop&&o?.onStop()})),s.setAudioEvent("onclose",(()=>{this.isPlaying=!1,o?.onStop&&o?.onStop()})),console.log("playAudio",i);let playAudio=()=>{s.play(i).then((()=>{})).catch((()=>{setTimeout((()=>{console.log("playAudio"),playAudio()}),200)}))};playAudio()}async uploadAndSaveVoice(e,t){if(console.log("blob",e),this.uploadServ){let o=t?.id||this.uploadServ?.genMd5(t?.get("content")||t?.get("ssml")),i=new Date,s=o+i.getFullYear()+(i.getMonth()+1)+i.getDate()+i.getHours()+i.getMinutes()+i.getSeconds()+".wav",n=new File([e],s,{type:"audio/wav"}),a=await this.uploadServ.upload(n,(e=>{console.log(e)})),r={__type:"Pointer",className:"Attachment",objectId:a?.id};return r?.objectId&&(t.set("voiceFile",r),t=await t.save()),a?.url}return null}}
|
|
9
9
|
var MODULE_PATH_NEED = `6K+l5paH5Lu25piv5pys6aG555uu55qE5LiA6YOo5YiGIFRoaXMgZmlsZSBpcyBwYXJ0IG9mIHRoZSBDb21wb25lbnRzIGluIEZtb2RlIEluYy4KICAgIOeJiOadg+aJgOaciSDCqSDmnKrmnaXpo57pqawgwqkg5rGf6KW/6ISR5o6n56eR5oqA5pyJ6ZmQ5YWs5Y+4IENvcHlyaWdodCDCqSBGbW9kZSBUZWNobm9sb2d5IENvLiwgTHRkLgogICAg5L+d55WZ5omA5pyJ5p2D5YipIEFsbCBSaWdodHMgUmVzZXJ2ZWQuCiAgICDkuKXnpoHlnKjmnKrnu4/mjojmnYPnmoTmg4XlhrXkuIvvvIzpgJrov4fku7vkvZXlqpLku4vlpI3liLbmraTmlofku7YgVW5hdXRob3JpemVkIGNvcHlpbmcgb2YgdGhpcyBmaWxlLCB2aWEgYW55IG1lZGl1bSBpcyBzdHJpY3RseSBwcm9oaWJpdGVkCiAgICDor6Xmlofku7bmmK/kuJPmnInnmoTmnLrlr4bmlofku7YgUHJvcHJpZXRhcnkgYW5kIGNvbmZpZGVudGlhbAogICAKICAgIENvcHlyaWdodCAyMDIxLW5vdyBGbW9kZSBJbmMuIHN1cHBvcnRAZm1vZGUuY24uIDE4NjA3MDA3MDczLgogICAg5L+d55WZ5omA5pyJ5p2D5YipIEFsbCByaWdodHMgcmVzZXJ2ZWQuCgogICAgUEFUSDovaG9tZS9yeWFuL3dvcmtzcGFjZS9ub3ZhL25vdmEtYWRtaW4vZGlzdC9mbW9kZS1uZy9lc20yMDIyL2xpYi9haWdjL3ZvaWNlL3R0cy9mbW9kZS10dHMtY2xhc3MubWpz`
|
|
10
10
|
|