@bdky/aaas-pilot-kit 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +222 -0
- package/dist/http.cjs.js +2 -0
- package/dist/http.cjs.js.LICENSE.txt +1 -0
- package/dist/http.esm.js +2 -0
- package/dist/http.esm.js.LICENSE.txt +1 -0
- package/dist/http.umd.js +2 -0
- package/dist/http.umd.js.LICENSE.txt +1 -0
- package/dist/index.cjs.js +44 -0
- package/dist/index.cjs.js.LICENSE.txt +45 -0
- package/dist/index.esm.js +44 -0
- package/dist/index.esm.js.LICENSE.txt +45 -0
- package/dist/index.umd.js +44 -0
- package/dist/index.umd.js.LICENSE.txt +45 -0
- package/dist/ky-aaas-pilot-kit.umd.js +44 -0
- package/dist/ky-aaas-pilot-kit.umd.js.LICENSE.txt +45 -0
- package/dist/libs/aaas-pilot-kit/src/http.d.ts +4 -0
- package/dist/libs/aaas-pilot-kit/src/index.d.ts +5 -0
- package/dist/libs/aaas-pilot-kit/src/lib/DI/types.d.ts +12 -0
- package/dist/libs/aaas-pilot-kit/src/lib/aaas-pilot-kit.d.ts +5 -0
- package/dist/libs/aaas-pilot-kit/src/lib/api/request.d.ts +9 -0
- package/dist/libs/aaas-pilot-kit/src/lib/constants/env.d.ts +1 -0
- package/dist/libs/aaas-pilot-kit/src/lib/controller.d.ts +107 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/BaseError.d.ts +12 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/ErrorEmitter.d.ts +54 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/ErrorHandler.d.ts +17 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/ErrorManager.d.ts +34 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/codes.d.ts +57 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/index.d.ts +16 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/monitoring/MonitoringError.d.ts +46 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/reporters/ConsoleReporter.d.ts +4 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/reporters/HttpReporter.d.ts +16 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/reporters/index.d.ts +2 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/services/AgentError.d.ts +17 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/services/AsrError.d.ts +23 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/services/ConversationError.d.ts +23 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/services/DigitalHumanError.d.ts +32 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/services/index.d.ts +4 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/types.d.ts +41 -0
- package/dist/libs/aaas-pilot-kit/src/lib/error/utils/ErrorUtils.d.ts +7 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/agent/aiobAgentService.d.ts +69 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/agent/baseAgentService.d.ts +65 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/agent/cspAgentService.d.ts +90 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/aiWorkerConversationBean.d.ts +39 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/clientConversationBean.d.ts +25 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/content/ImageContent.d.ts +25 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/content/MultimodalContent.d.ts +42 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/content/TextContent.d.ts +41 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/content/VideoContent.d.ts +26 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/content/index.d.ts +8 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/conversationBean.d.ts +25 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/conversationService.d.ts +36 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/conversation/index.d.ts +4 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/digital-human/BaseDigitalHumanService.d.ts +62 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/digital-human/CloudDigitalHumanService.d.ts +102 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/digital-human/PictureClientDigitalHumanService.d.ts +85 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/digital-human/interface.d.ts +100 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/rtc-asr/asr/baseAsrService.d.ts +82 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/rtc-asr/asr/brtcAsrService.d.ts +38 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/rtc-asr/signal/base.d.ts +19 -0
- package/dist/libs/aaas-pilot-kit/src/lib/service/rtc-asr/signal/brtc.d.ts +69 -0
- package/dist/libs/aaas-pilot-kit/src/lib/utils/applyTextReplacements.d.ts +15 -0
- package/dist/libs/aaas-pilot-kit/src/lib/utils/customXmlParser.d.ts +46 -0
- package/dist/libs/aaas-pilot-kit/src/lib/utils/isEmptyObject.d.ts +14 -0
- package/dist/libs/aaas-pilot-kit/src/lib/utils/isPlainObject.d.ts +13 -0
- package/dist/libs/aaas-pilot-kit/src/lib/utils/noop.d.ts +1 -0
- package/dist/libs/aaas-pilot-kit/src/lib/utils/sleep.d.ts +1 -0
- package/dist/libs/aaas-pilot-kit/src/lib/utils/toQueryString.d.ts +1 -0
- package/dist/libs/aaas-pilot-kit/src/lib/utils/ua.d.ts +3 -0
- package/dist/libs/aaas-pilot-kit/src/samples/react/Route/index.d.ts +3 -0
- package/dist/libs/aaas-pilot-kit/src/samples/react/conversation.d.ts +2 -0
- package/dist/libs/aaas-pilot-kit/src/samples/react/index.d.ts +1 -0
- package/dist/libs/aaas-pilot-kit/src/test-setup.d.ts +1 -0
- package/dist/libs/aaas-pilot-kit/src/types/agent.d.ts +8 -0
- package/dist/libs/aaas-pilot-kit/src/types/common.d.ts +5 -0
- package/dist/libs/aaas-pilot-kit/src/types/config.d.ts +552 -0
- package/package.json +64 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { BaseAgentService } from './baseAgentService';
|
|
2
|
+
export declare namespace NAiobAgentService {
|
|
3
|
+
enum EAiobAction {
|
|
4
|
+
/**
|
|
5
|
+
* 静默
|
|
6
|
+
*/
|
|
7
|
+
silence = 1,
|
|
8
|
+
/**
|
|
9
|
+
* 打断
|
|
10
|
+
*/
|
|
11
|
+
Interrupt = 2,
|
|
12
|
+
/**
|
|
13
|
+
* 挂机
|
|
14
|
+
*/
|
|
15
|
+
Hangup = 3
|
|
16
|
+
}
|
|
17
|
+
interface IAiobQueryParams {
|
|
18
|
+
query: string;
|
|
19
|
+
robotId: string;
|
|
20
|
+
sessionId: string;
|
|
21
|
+
action?: EAiobAction;
|
|
22
|
+
promptVar?: Record<string, string>;
|
|
23
|
+
queryId?: string;
|
|
24
|
+
isComplete: 0 | 1;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* 响应类型枚举
|
|
28
|
+
*/
|
|
29
|
+
enum EResponseType {
|
|
30
|
+
/**
|
|
31
|
+
* 欢迎语
|
|
32
|
+
*/
|
|
33
|
+
Welcome = 1,
|
|
34
|
+
/**
|
|
35
|
+
* 正常人机对话
|
|
36
|
+
*/
|
|
37
|
+
Normal = 2,
|
|
38
|
+
/**
|
|
39
|
+
* 指令
|
|
40
|
+
*/
|
|
41
|
+
Command = 3,
|
|
42
|
+
/**
|
|
43
|
+
* 大模型接管
|
|
44
|
+
*/
|
|
45
|
+
TakeOver = 4
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* 响应参数
|
|
49
|
+
*/
|
|
50
|
+
interface IConversationSSEMessage {
|
|
51
|
+
answer: string;
|
|
52
|
+
type: EResponseType;
|
|
53
|
+
isEnd: boolean;
|
|
54
|
+
index: number;
|
|
55
|
+
sessionId: string;
|
|
56
|
+
action: EAiobAction;
|
|
57
|
+
queryId: string;
|
|
58
|
+
}
|
|
59
|
+
enum ETextType {
|
|
60
|
+
unit = 0,
|
|
61
|
+
llm = 1
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
export declare class AiobAgentService extends BaseAgentService {
|
|
65
|
+
static REQUEST_URI: string;
|
|
66
|
+
query: (text: string) => Promise<void>;
|
|
67
|
+
hangup: () => Promise<void>;
|
|
68
|
+
dispose: () => void;
|
|
69
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import Emittery from 'emittery';
|
|
2
|
+
import { CommonRequest } from '../../api/request';
|
|
3
|
+
import type { IOptions } from '../../../types/config';
|
|
4
|
+
import { ErrorManager } from '../../error';
|
|
5
|
+
export declare namespace NBaseAgentService {
|
|
6
|
+
interface IEventWithSessionIdPayload {
|
|
7
|
+
sessionId: string;
|
|
8
|
+
queryId?: string;
|
|
9
|
+
}
|
|
10
|
+
interface IDataEventPayload extends IEventWithSessionIdPayload {
|
|
11
|
+
answer: string;
|
|
12
|
+
}
|
|
13
|
+
interface IInstructionReceivedEventPayload extends IEventWithSessionIdPayload {
|
|
14
|
+
instruction: Record<string, Record<string, any>>;
|
|
15
|
+
}
|
|
16
|
+
type IHangupEventPayload = IEventWithSessionIdPayload;
|
|
17
|
+
type ICompletedEventPayload = IEventWithSessionIdPayload;
|
|
18
|
+
interface IEmitter {
|
|
19
|
+
data: IDataEventPayload;
|
|
20
|
+
completed: ICompletedEventPayload;
|
|
21
|
+
timeout: IEventWithSessionIdPayload;
|
|
22
|
+
hangup: IHangupEventPayload;
|
|
23
|
+
instruction_received: IInstructionReceivedEventPayload;
|
|
24
|
+
quota_expired: IEventWithSessionIdPayload;
|
|
25
|
+
usage_quota_reached: IEventWithSessionIdPayload;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export declare abstract class BaseAgentService<E extends NBaseAgentService.IEmitter = NBaseAgentService.IEmitter> {
|
|
29
|
+
readonly request: CommonRequest;
|
|
30
|
+
readonly errorManager: ErrorManager;
|
|
31
|
+
readonly opts: Required<IOptions>;
|
|
32
|
+
readonly emitter: Emittery<E, E & import("emittery").OmnipresentEventData, import("emittery").DatalessEventNames<E>>;
|
|
33
|
+
protected conversationSessionId: string;
|
|
34
|
+
protected abortController: AbortController | null;
|
|
35
|
+
/**
|
|
36
|
+
* 每一轮query的id,用于区分不同的query
|
|
37
|
+
*/
|
|
38
|
+
protected _queryId: string;
|
|
39
|
+
/**
|
|
40
|
+
* 上一轮的query id
|
|
41
|
+
*/
|
|
42
|
+
protected _lastQueryId: string;
|
|
43
|
+
constructor(request: CommonRequest, errorManager: ErrorManager, opts: Required<IOptions>);
|
|
44
|
+
get sessionId(): string;
|
|
45
|
+
get queryId(): string;
|
|
46
|
+
get lastQueryId(): string;
|
|
47
|
+
setSessionId: (sessionId: string) => void;
|
|
48
|
+
setQueryId: (queryId: string) => void;
|
|
49
|
+
setLastQueryId: (queryId: string) => void;
|
|
50
|
+
abort: (reason?: string) => void;
|
|
51
|
+
/**
|
|
52
|
+
* 判断是否是重复的query请求
|
|
53
|
+
*/
|
|
54
|
+
checkIsRepeat: () => boolean;
|
|
55
|
+
protected onData: (payload: E["data"]) => void;
|
|
56
|
+
protected onCompleted: (payload: E["completed"]) => void;
|
|
57
|
+
protected onHangup: (payload: E["hangup"]) => void;
|
|
58
|
+
protected onTimeout: (payload: E["timeout"]) => void;
|
|
59
|
+
protected onInstructionReceived: (payload: E["instruction_received"]) => void;
|
|
60
|
+
protected onUsageQuotaReached: (payload: E["usage_quota_reached"]) => void;
|
|
61
|
+
protected onQuotaExpired: (payload: E["quota_expired"]) => void;
|
|
62
|
+
protected _dispose: () => void;
|
|
63
|
+
abstract query(text: string): Promise<any>;
|
|
64
|
+
abstract dispose(): void;
|
|
65
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { BaseAgentService } from './baseAgentService';
|
|
2
|
+
export declare namespace NCspAgentService {
|
|
3
|
+
interface ICspQueryParams {
|
|
4
|
+
query: string;
|
|
5
|
+
sessionId: string;
|
|
6
|
+
uid?: string;
|
|
7
|
+
username?: string;
|
|
8
|
+
}
|
|
9
|
+
enum EReplyType {
|
|
10
|
+
PlainText = 1,
|
|
11
|
+
RichText = 3,
|
|
12
|
+
Image = 4,
|
|
13
|
+
List = 8
|
|
14
|
+
}
|
|
15
|
+
type FormattedReplyType = 'plainText' | 'richText' | 'image' | 'list';
|
|
16
|
+
const REPLY_TYPE_MAPPING: {
|
|
17
|
+
[key in EReplyType]: FormattedReplyType;
|
|
18
|
+
};
|
|
19
|
+
type ReplySource = 'FAQ' | 'DOC_QA' | 'TEXT_REPLY' | 'LARGE_MODEL_REPLY' | 'COLLECT_SLOT_ENTITY' | 'CHITCHAT' | 'UNMATCHED' | 'SENSITIVE' | 'SYSTEM' | 'INTENT_CLARIFY' | 'INSTRUCTION' | 'TASK_BASE_BACK' | 'WELCOME' | 'SILENT_REPLY' | 'PLUGIN';
|
|
20
|
+
/**
|
|
21
|
+
* 流式对话中的一条SSE消息
|
|
22
|
+
*/
|
|
23
|
+
interface IConversationSSEMessage<TOutputs = unknown> {
|
|
24
|
+
/** 返回状态 200 -- 成功;4007270 -- token不足 */
|
|
25
|
+
replyStatus: '200' | '4007270';
|
|
26
|
+
/** 会话ID,主要用于打断 */
|
|
27
|
+
sessionId: string;
|
|
28
|
+
/** 推荐意图 猜你想问 */
|
|
29
|
+
recommendList?: string[];
|
|
30
|
+
/** 变量 */
|
|
31
|
+
variables?: Record<string, any>;
|
|
32
|
+
/** queryId */
|
|
33
|
+
queryId: string;
|
|
34
|
+
/** 命中的意图 */
|
|
35
|
+
/** 复杂对话消息的消息体, 流式对话中一条SSE content的长度必定为1 */
|
|
36
|
+
answer: Array<IConversationSSEMessageContent<TOutputs>>;
|
|
37
|
+
}
|
|
38
|
+
interface DocumentInfo {
|
|
39
|
+
document_id: string;
|
|
40
|
+
document_name: string;
|
|
41
|
+
knowledge_type: 'doc' | 'web';
|
|
42
|
+
web_url: string;
|
|
43
|
+
data: Array<{
|
|
44
|
+
segment_id: string;
|
|
45
|
+
sentence_id: string;
|
|
46
|
+
segment_coord: string;
|
|
47
|
+
}>;
|
|
48
|
+
}
|
|
49
|
+
interface IConversationSSEReplyMessageContent {
|
|
50
|
+
type: EReplyType;
|
|
51
|
+
text: string;
|
|
52
|
+
textList?: string[];
|
|
53
|
+
replySource: ReplySource;
|
|
54
|
+
plugin?: string;
|
|
55
|
+
clarifyGuide?: string;
|
|
56
|
+
showDocumentSource?: boolean;
|
|
57
|
+
documents?: DocumentInfo[];
|
|
58
|
+
}
|
|
59
|
+
type StatusType = 'init' | 'querying' | 'running' | 'done';
|
|
60
|
+
/** 复杂对话消息的消息体 */
|
|
61
|
+
interface IConversationSSEMessageContent<TOutputs = unknown> {
|
|
62
|
+
/** 节点推送的消息的id, 和content_type是 1:1的关系, 同一个chunkId的消息体可能会有多个 */
|
|
63
|
+
chunkId: number;
|
|
64
|
+
/** 当前会话的主题ID */
|
|
65
|
+
topicId: string | null;
|
|
66
|
+
/** 当前消息对应的 blockId,主要作用是用于前端高亮节点 */
|
|
67
|
+
blockId: string | null;
|
|
68
|
+
/** 当前消息对应的节点或者步骤id,也是用于高亮步骤或者节点 */
|
|
69
|
+
nodeId: string;
|
|
70
|
+
/** 执行状态,目前还没有,得督促后端加上 */
|
|
71
|
+
status: StatusType;
|
|
72
|
+
/** 回复消息内容(包含消息类型、) */
|
|
73
|
+
reply: IConversationSSEReplyMessageContent | null;
|
|
74
|
+
/** 触发的指令数据 */
|
|
75
|
+
instruction?: Record<string, Record<string, any>>;
|
|
76
|
+
/** 消息实际内容, 具体值跟随 reply.type 变动而变动, 与nodeId 1:1关联,前端定义用于渲染 */
|
|
77
|
+
outputs: TOutputs;
|
|
78
|
+
/** 点赞/踩状态,用于对话记录回显 0: 初始状态,1:点赞,2:点踩 */
|
|
79
|
+
thumbStatus?: 0 | 1 | 2;
|
|
80
|
+
/** 点踩反馈信息(反馈类别/选项) */
|
|
81
|
+
thumbReason?: string;
|
|
82
|
+
/** 点踩反馈信息(反馈详情) */
|
|
83
|
+
thumbDetail?: string;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
export declare class CspAgentService extends BaseAgentService {
|
|
87
|
+
static REQUEST_URI: string;
|
|
88
|
+
query: (text: string) => Promise<void>;
|
|
89
|
+
dispose: () => void;
|
|
90
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { IOptions } from '../../../types/config';
|
|
2
|
+
import { ConversationBean, type IConversationBeanSnapshot, NConversationBean } from './conversationBean';
|
|
3
|
+
import { NMultimodalContent, ImageContent, VideoContent, TextContent } from './content';
|
|
4
|
+
export declare namespace NAIWorkerConversationBean {
|
|
5
|
+
interface IContentChangePayload {
|
|
6
|
+
mod: NMultimodalContent.MultimodalMod;
|
|
7
|
+
/**
|
|
8
|
+
* 文本内容
|
|
9
|
+
* 图片/视频的url
|
|
10
|
+
*/
|
|
11
|
+
content: string;
|
|
12
|
+
completed: boolean;
|
|
13
|
+
}
|
|
14
|
+
interface IEmitter extends NConversationBean.IEmitter {
|
|
15
|
+
content_change: IContentChangePayload;
|
|
16
|
+
queue_empty: never;
|
|
17
|
+
interrupt: never;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export declare class AIWorkerConversationBean extends ConversationBean<'aiWorker', NAIWorkerConversationBean.IEmitter> {
|
|
21
|
+
readonly opts: Required<IOptions>;
|
|
22
|
+
contents: Array<ImageContent | VideoContent | TextContent>;
|
|
23
|
+
private readonly queue;
|
|
24
|
+
private readonly customXmlParser;
|
|
25
|
+
constructor(d: Partial<IConversationBeanSnapshot<'aiWorker'>>, opts: Required<IOptions>);
|
|
26
|
+
get fulltext(): string;
|
|
27
|
+
static create: (opts: Required<IOptions>) => AIWorkerConversationBean;
|
|
28
|
+
static isAllTextContents: (contents: Array<ImageContent | VideoContent | TextContent>) => contents is TextContent[];
|
|
29
|
+
static isTextContent: (content: TextContent | ImageContent | VideoContent) => content is TextContent;
|
|
30
|
+
static isImageContent: (content: TextContent | ImageContent | VideoContent) => content is ImageContent;
|
|
31
|
+
static isVideoContent: (content: TextContent | ImageContent | VideoContent) => content is VideoContent;
|
|
32
|
+
addContentRenderTask: (c: ImageContent | VideoContent | TextContent) => void;
|
|
33
|
+
addAllContents: (contents: Array<ImageContent | VideoContent | TextContent>) => void;
|
|
34
|
+
createContentsFromXmlString: (xmlString: string) => Array<TextContent | ImageContent | VideoContent>;
|
|
35
|
+
appendText: (text: string) => void;
|
|
36
|
+
dispose: () => void;
|
|
37
|
+
interrupt: () => void;
|
|
38
|
+
private readonly bindEvents;
|
|
39
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { ConversationBean, NConversationBean, type IConversationBeanSnapshot } from './conversationBean';
|
|
2
|
+
import { IOptions } from '../../../types/config';
|
|
3
|
+
import { NMultimodalContent } from './content';
|
|
4
|
+
export declare namespace NClientConversationBean {
|
|
5
|
+
interface IContentChangePayload {
|
|
6
|
+
mod: NMultimodalContent.MultimodalMod;
|
|
7
|
+
/**
|
|
8
|
+
* 文本内容
|
|
9
|
+
* 图片/视频的url
|
|
10
|
+
*/
|
|
11
|
+
content: string;
|
|
12
|
+
completed: boolean;
|
|
13
|
+
}
|
|
14
|
+
interface IEmitter extends NConversationBean.IEmitter {
|
|
15
|
+
content_change: IContentChangePayload;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export declare class ClientConversationBean extends ConversationBean<'client', NClientConversationBean.IEmitter> {
|
|
19
|
+
readonly opts: Required<IOptions>;
|
|
20
|
+
constructor(d: Partial<IConversationBeanSnapshot<'client'>>, opts: Required<IOptions>);
|
|
21
|
+
static create(opts: Required<IOptions>): ClientConversationBean;
|
|
22
|
+
appendText: (text: string) => void;
|
|
23
|
+
dispose: () => void;
|
|
24
|
+
interrupt: () => void;
|
|
25
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { MultimodalContent, NMultimodalContent } from './MultimodalContent';
|
|
2
|
+
export declare namespace NImageContent {
|
|
3
|
+
interface IMultimodalImageConfig extends NMultimodalContent.IMultimodalBaseConfig {
|
|
4
|
+
src: string;
|
|
5
|
+
alt?: string;
|
|
6
|
+
width?: number;
|
|
7
|
+
height?: number;
|
|
8
|
+
}
|
|
9
|
+
interface ITaskResult extends Omit<IMultimodalImageConfig, 'metadata' | 'timestamp'> {
|
|
10
|
+
mod: 'image';
|
|
11
|
+
}
|
|
12
|
+
type IEmitter = NMultimodalContent.IEmitter;
|
|
13
|
+
}
|
|
14
|
+
export declare class ImageContent extends MultimodalContent<NImageContent.IMultimodalImageConfig, NImageContent.ITaskResult, NImageContent.IEmitter> {
|
|
15
|
+
private imgElement;
|
|
16
|
+
get mod(): NMultimodalContent.MultimodalMod;
|
|
17
|
+
static fromSnapshot(snapshot: NMultimodalContent.ISnapshot<NImageContent.IMultimodalImageConfig>): ImageContent;
|
|
18
|
+
static fromSrc(src: string, options?: Partial<Omit<NImageContent.IMultimodalImageConfig, 'src'>>): ImageContent;
|
|
19
|
+
setTargetElement: (img: HTMLImageElement) => void;
|
|
20
|
+
isEmpty(): boolean;
|
|
21
|
+
runTask: () => Promise<NImageContent.ITaskResult>;
|
|
22
|
+
dispose: () => void;
|
|
23
|
+
interrupt: () => void;
|
|
24
|
+
private ensureImageElement;
|
|
25
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import Emittery from 'emittery';
|
|
2
|
+
export declare namespace NMultimodalContent {
|
|
3
|
+
type MultimodalMod = 'text' | 'image' | 'video';
|
|
4
|
+
interface IMultimodalBaseConfig {
|
|
5
|
+
metadata?: Record<string, any>;
|
|
6
|
+
timestamp?: number;
|
|
7
|
+
}
|
|
8
|
+
interface IMultimodalVideoConfig extends IMultimodalBaseConfig {
|
|
9
|
+
src: string;
|
|
10
|
+
poster?: string;
|
|
11
|
+
width?: number;
|
|
12
|
+
height?: number;
|
|
13
|
+
}
|
|
14
|
+
interface ISnapshot<T extends IMultimodalBaseConfig> {
|
|
15
|
+
id: string;
|
|
16
|
+
mod: MultimodalMod;
|
|
17
|
+
config: T;
|
|
18
|
+
createdAt: number;
|
|
19
|
+
updatedAt: number;
|
|
20
|
+
}
|
|
21
|
+
interface IEmitter {
|
|
22
|
+
complete: never;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
export declare abstract class MultimodalContent<T extends NMultimodalContent.IMultimodalBaseConfig, R extends Record<string, any>, E = NMultimodalContent.IEmitter> {
|
|
26
|
+
readonly id: string;
|
|
27
|
+
readonly createdAt: number;
|
|
28
|
+
updatedAt: number;
|
|
29
|
+
emitter: Emittery<E, E & import("emittery").OmnipresentEventData, import("emittery").DatalessEventNames<E>>;
|
|
30
|
+
protected config: T;
|
|
31
|
+
protected constructor(config: T);
|
|
32
|
+
abstract get mod(): NMultimodalContent.MultimodalMod;
|
|
33
|
+
getConfig: () => Readonly<T>;
|
|
34
|
+
updateConfig: (partialConfig: Partial<T>) => void;
|
|
35
|
+
toJSON: () => NMultimodalContent.ISnapshot<T>;
|
|
36
|
+
protected updateTimestamp: () => void;
|
|
37
|
+
protected readonly _dispose: () => void;
|
|
38
|
+
abstract isEmpty(): boolean;
|
|
39
|
+
abstract runTask(): Promise<R>;
|
|
40
|
+
abstract dispose(): void;
|
|
41
|
+
abstract interrupt(): void;
|
|
42
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { MultimodalContent, NMultimodalContent } from './MultimodalContent';
|
|
2
|
+
export declare namespace NTextContent {
|
|
3
|
+
interface IMultimodalTextConfig extends NMultimodalContent.IMultimodalBaseConfig {
|
|
4
|
+
content: string;
|
|
5
|
+
typewriterOptions?: {
|
|
6
|
+
delay?: number;
|
|
7
|
+
enDelay?: number;
|
|
8
|
+
loop?: boolean;
|
|
9
|
+
cursor?: string;
|
|
10
|
+
autoStart?: boolean;
|
|
11
|
+
onCreateTextNode?: (character: string, textNode: Text) => Text | null;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
interface IDataEventPayload {
|
|
15
|
+
char: string;
|
|
16
|
+
index: number;
|
|
17
|
+
done: boolean;
|
|
18
|
+
}
|
|
19
|
+
interface ITaskResult {
|
|
20
|
+
mod: 'text';
|
|
21
|
+
content: string;
|
|
22
|
+
}
|
|
23
|
+
interface IEmitter extends NMultimodalContent.IEmitter {
|
|
24
|
+
data: IDataEventPayload;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export declare class TextContent extends MultimodalContent<NTextContent.IMultimodalTextConfig, NTextContent.ITaskResult, NTextContent.IEmitter> {
|
|
28
|
+
private typewriter;
|
|
29
|
+
private targetElement;
|
|
30
|
+
private index;
|
|
31
|
+
get mod(): NMultimodalContent.MultimodalMod;
|
|
32
|
+
static fromSnapshot(snapshot: NMultimodalContent.ISnapshot<NTextContent.IMultimodalTextConfig>): TextContent;
|
|
33
|
+
static fromString(content: string, options?: Partial<Omit<NTextContent.IMultimodalTextConfig, 'content'>>): TextContent;
|
|
34
|
+
setTargetElement(element: HTMLElement): void;
|
|
35
|
+
stopTypewriter: () => void;
|
|
36
|
+
isEmpty(): boolean;
|
|
37
|
+
runTask: () => Promise<NTextContent.ITaskResult>;
|
|
38
|
+
dispose: () => void;
|
|
39
|
+
interrupt: () => void;
|
|
40
|
+
private readonly ensureTarget;
|
|
41
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { MultimodalContent, NMultimodalContent } from './MultimodalContent';
|
|
2
|
+
export declare namespace NVideoContent {
|
|
3
|
+
interface IMultimodalVideoConfig extends NMultimodalContent.IMultimodalBaseConfig {
|
|
4
|
+
src: string;
|
|
5
|
+
poster?: string;
|
|
6
|
+
width?: number;
|
|
7
|
+
height?: number;
|
|
8
|
+
preload?: 'none' | 'metadata' | 'auto';
|
|
9
|
+
}
|
|
10
|
+
interface ITaskResult extends Omit<IMultimodalVideoConfig, 'metadata' | 'timestamp'> {
|
|
11
|
+
mod: 'video';
|
|
12
|
+
}
|
|
13
|
+
type IEmitter = NMultimodalContent.IEmitter;
|
|
14
|
+
}
|
|
15
|
+
export declare class VideoContent extends MultimodalContent<NVideoContent.IMultimodalVideoConfig, NVideoContent.ITaskResult, NVideoContent.IEmitter> {
|
|
16
|
+
private videoElement;
|
|
17
|
+
get mod(): NMultimodalContent.MultimodalMod;
|
|
18
|
+
static fromSnapshot(snapshot: NMultimodalContent.ISnapshot<NVideoContent.IMultimodalVideoConfig>): VideoContent;
|
|
19
|
+
static fromSrc(src: string, options?: Partial<Omit<NVideoContent.IMultimodalVideoConfig, 'src'>>): VideoContent;
|
|
20
|
+
setTargetElement: (video: HTMLVideoElement) => void;
|
|
21
|
+
isEmpty(): boolean;
|
|
22
|
+
runTask: () => Promise<NVideoContent.ITaskResult>;
|
|
23
|
+
dispose: () => void;
|
|
24
|
+
interrupt: () => void;
|
|
25
|
+
private ensureVideoElement;
|
|
26
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { MultimodalContent } from './MultimodalContent';
|
|
2
|
+
export type { NMultimodalContent } from './MultimodalContent';
|
|
3
|
+
export { TextContent } from './TextContent';
|
|
4
|
+
export type { NTextContent } from './TextContent';
|
|
5
|
+
export { ImageContent } from './ImageContent';
|
|
6
|
+
export type { NImageContent } from './ImageContent';
|
|
7
|
+
export { VideoContent } from './VideoContent';
|
|
8
|
+
export { type NVideoContent } from './VideoContent';
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import Emittery from 'emittery';
|
|
2
|
+
import { IOptions } from '../../../types/config';
|
|
3
|
+
export type ConversationKind = 'client' | 'aiWorker';
|
|
4
|
+
export interface IConversationBeanSnapshot<K extends ConversationKind = ConversationKind> {
|
|
5
|
+
id: string;
|
|
6
|
+
type: K;
|
|
7
|
+
text: string;
|
|
8
|
+
}
|
|
9
|
+
export declare namespace NConversationBean {
|
|
10
|
+
interface IEmitter {
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export declare abstract class ConversationBean<K extends ConversationKind, E = NConversationBean.IEmitter> implements IConversationBeanSnapshot<K> {
|
|
14
|
+
readonly opts: Required<IOptions>;
|
|
15
|
+
readonly emitter: Emittery<E, E & import("emittery").OmnipresentEventData, import("emittery").DatalessEventNames<E>>;
|
|
16
|
+
id: string;
|
|
17
|
+
type: K;
|
|
18
|
+
text: string;
|
|
19
|
+
protected constructor(d: Partial<IConversationBeanSnapshot<K>>, opts: Required<IOptions>);
|
|
20
|
+
update: (d: Partial<IConversationBeanSnapshot>) => void;
|
|
21
|
+
toJSON: () => Required<IConversationBeanSnapshot>;
|
|
22
|
+
abstract appendText(text: string): void;
|
|
23
|
+
abstract dispose(): void;
|
|
24
|
+
abstract interrupt(): void;
|
|
25
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type IConversationBeanSnapshot } from './conversationBean';
|
|
2
|
+
import { ClientConversationBean } from './clientConversationBean';
|
|
3
|
+
import { AIWorkerConversationBean } from './aiWorkerConversationBean';
|
|
4
|
+
import type { IOptions } from '../../../types/config';
|
|
5
|
+
export type AnyConversation = ClientConversationBean | AIWorkerConversationBean;
|
|
6
|
+
export type IConversationFormatter = (text: string) => string;
|
|
7
|
+
export declare class ConversationService {
|
|
8
|
+
readonly opts: Required<IOptions>;
|
|
9
|
+
beans: AnyConversation[];
|
|
10
|
+
currentConversation: AnyConversation | null;
|
|
11
|
+
private aiWorkerConversationFormatters;
|
|
12
|
+
constructor(opts: Required<IOptions>);
|
|
13
|
+
get last(): AnyConversation | null;
|
|
14
|
+
get currentClientConversation(): ClientConversationBean | null;
|
|
15
|
+
get currentAIWorkerConversation(): AIWorkerConversationBean | null;
|
|
16
|
+
static isClientConversation(conversation: AnyConversation | null): conversation is ClientConversationBean;
|
|
17
|
+
static isAIWorkerConversation(conversation: AnyConversation | null): conversation is AIWorkerConversationBean;
|
|
18
|
+
add: (bean: AnyConversation) => void;
|
|
19
|
+
addBySnapshot: (snapshot?: Partial<IConversationBeanSnapshot>) => AnyConversation;
|
|
20
|
+
setCurrentConversation: (conversation: AnyConversation | null) => void;
|
|
21
|
+
clear: () => void;
|
|
22
|
+
toJSON: () => IConversationBeanSnapshot[];
|
|
23
|
+
dispose: () => void;
|
|
24
|
+
/**
|
|
25
|
+
* 注册aiWorkerConversation formatter
|
|
26
|
+
*/
|
|
27
|
+
registerAIWorkerConversationFormatter: (formatter: IConversationFormatter) => void;
|
|
28
|
+
/**
|
|
29
|
+
* 运行aiWorkerConversation formatters
|
|
30
|
+
*/
|
|
31
|
+
runAIWorkerConversationFormatters: (text: string) => string;
|
|
32
|
+
/**
|
|
33
|
+
* 打断所有conversations
|
|
34
|
+
*/
|
|
35
|
+
interruptAll: () => void;
|
|
36
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import Emittery from 'emittery';
|
|
2
|
+
import { type IOptions } from '../../../types/config';
|
|
3
|
+
import { ErrorManager } from '../../error';
|
|
4
|
+
import { CommonRequest } from '../../api/request';
|
|
5
|
+
export declare namespace NAbstractDigitalHumanService {
|
|
6
|
+
interface ISubtitle {
|
|
7
|
+
subtitleId: string;
|
|
8
|
+
content: string;
|
|
9
|
+
completed: boolean;
|
|
10
|
+
}
|
|
11
|
+
interface IBaseEmitterPayload {
|
|
12
|
+
requestId: string;
|
|
13
|
+
}
|
|
14
|
+
interface IRenderSubtitleEventPayload extends IBaseEmitterPayload {
|
|
15
|
+
subtitle: ISubtitle;
|
|
16
|
+
}
|
|
17
|
+
interface IEmitter {
|
|
18
|
+
disconnect_alert: IBaseEmitterPayload;
|
|
19
|
+
render_subtitle: IRenderSubtitleEventPayload;
|
|
20
|
+
render_finished: IBaseEmitterPayload;
|
|
21
|
+
timeout: IBaseEmitterPayload;
|
|
22
|
+
ready: never;
|
|
23
|
+
reach_session_limit: IBaseEmitterPayload;
|
|
24
|
+
reconnect_failed: never;
|
|
25
|
+
send_silence_message: IBaseEmitterPayload;
|
|
26
|
+
is_rendering_change: boolean;
|
|
27
|
+
}
|
|
28
|
+
type ISsmlHandler = (text: string) => string;
|
|
29
|
+
}
|
|
30
|
+
export declare abstract class BaseDigitalHumanService {
|
|
31
|
+
protected readonly opts: IOptions;
|
|
32
|
+
protected readonly request: CommonRequest;
|
|
33
|
+
protected readonly errorManager: ErrorManager;
|
|
34
|
+
protected static readonly GET_ACCESS_TOKEN_REQUEST_URI = "deepsight/employee/v1/get_digital_token";
|
|
35
|
+
emitter: Emittery<NAbstractDigitalHumanService.IEmitter>;
|
|
36
|
+
protected _isRendering: boolean;
|
|
37
|
+
protected _isMuted: boolean;
|
|
38
|
+
protected _canInterrupt: boolean;
|
|
39
|
+
protected ssmlHandlers: NAbstractDigitalHumanService.ISsmlHandler[];
|
|
40
|
+
protected token: string;
|
|
41
|
+
constructor(opts: IOptions, request: CommonRequest, errorManager: ErrorManager);
|
|
42
|
+
get isRendering(): boolean;
|
|
43
|
+
get isMuted(): boolean;
|
|
44
|
+
get canInterrupt(): boolean;
|
|
45
|
+
abstract get ready(): boolean;
|
|
46
|
+
setIsRendering(isRendering: boolean): void;
|
|
47
|
+
setIsMuted(isMuted: boolean): void;
|
|
48
|
+
setCanInterrupt(canInterrupt: boolean): void;
|
|
49
|
+
registerSSMLHandler(handler: NAbstractDigitalHumanService.ISsmlHandler): this;
|
|
50
|
+
protected getAccessToken: () => Promise<string>;
|
|
51
|
+
protected runSsmlHandlers(text: string): string;
|
|
52
|
+
abstract mount(container?: HTMLElement): Promise<void>;
|
|
53
|
+
abstract bindEvents(): this;
|
|
54
|
+
abstract unbindEvents(): this;
|
|
55
|
+
abstract renderPlainText(text: string): this;
|
|
56
|
+
abstract renderStreamText(text: string, hasEnded?: boolean): this;
|
|
57
|
+
abstract interrupt(): this;
|
|
58
|
+
abstract sendSilenceMessage(): this;
|
|
59
|
+
abstract dispose(): this;
|
|
60
|
+
abstract GetRequestId(): string;
|
|
61
|
+
protected abstract checkIsReady(): boolean;
|
|
62
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { BaseDigitalHumanService, NAbstractDigitalHumanService } from './BaseDigitalHumanService';
|
|
2
|
+
export declare namespace NDigitalHumanService {
|
|
3
|
+
type ISubtitle = NAbstractDigitalHumanService.ISubtitle;
|
|
4
|
+
type IBaseEmitterPayload = NAbstractDigitalHumanService.IBaseEmitterPayload;
|
|
5
|
+
type IRenderSubtitleEventPayload = NAbstractDigitalHumanService.IRenderSubtitleEventPayload;
|
|
6
|
+
type IEmitter = NAbstractDigitalHumanService.IEmitter;
|
|
7
|
+
type ISsmlHandler = NAbstractDigitalHumanService.ISsmlHandler;
|
|
8
|
+
interface IPosition {
|
|
9
|
+
crop?: {
|
|
10
|
+
top: number;
|
|
11
|
+
left: number;
|
|
12
|
+
width: number;
|
|
13
|
+
height: number;
|
|
14
|
+
};
|
|
15
|
+
location?: {
|
|
16
|
+
top: number;
|
|
17
|
+
left: number;
|
|
18
|
+
width: number;
|
|
19
|
+
height: number;
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export declare class CloudDigitalHumanService extends BaseDigitalHumanService {
|
|
24
|
+
/**
|
|
25
|
+
* iframe src的origin
|
|
26
|
+
*/
|
|
27
|
+
private readonly origin;
|
|
28
|
+
/**
|
|
29
|
+
* iframe实例id
|
|
30
|
+
*/
|
|
31
|
+
private readonly iframeId;
|
|
32
|
+
/**
|
|
33
|
+
* iframe控制器实例
|
|
34
|
+
*/
|
|
35
|
+
private iframeController;
|
|
36
|
+
/**
|
|
37
|
+
* iframe实例
|
|
38
|
+
*/
|
|
39
|
+
private iframe;
|
|
40
|
+
/**
|
|
41
|
+
* iframe容器实例
|
|
42
|
+
*/
|
|
43
|
+
private $iframeContainer;
|
|
44
|
+
/**
|
|
45
|
+
* 与数字员工iframe交互时使用的requestId
|
|
46
|
+
*/
|
|
47
|
+
private requestId;
|
|
48
|
+
/**
|
|
49
|
+
* ws是否链接
|
|
50
|
+
*/
|
|
51
|
+
private isWSConnected;
|
|
52
|
+
/**
|
|
53
|
+
* rtc实时视频流是否ready
|
|
54
|
+
*/
|
|
55
|
+
private isRtcReady;
|
|
56
|
+
/**
|
|
57
|
+
* 静默消息requestId列表
|
|
58
|
+
*
|
|
59
|
+
* (静默消息你报的字幕需要忽略)
|
|
60
|
+
*/
|
|
61
|
+
private silenceMessageRequestIds;
|
|
62
|
+
/**
|
|
63
|
+
* 是否正在流式文本渲染
|
|
64
|
+
*/
|
|
65
|
+
private isStreamTextRendering;
|
|
66
|
+
/**
|
|
67
|
+
* 连接关闭次数计数器
|
|
68
|
+
*/
|
|
69
|
+
private wsClosedCount;
|
|
70
|
+
get ready(): boolean;
|
|
71
|
+
mount: ($container?: HTMLElement) => Promise<void>;
|
|
72
|
+
bindEvents: () => this;
|
|
73
|
+
unbindEvents: () => this;
|
|
74
|
+
cancelMute: () => void;
|
|
75
|
+
renderPlainText: (text: string) => this;
|
|
76
|
+
renderStreamText: (text: string, hasEnded?: boolean) => this;
|
|
77
|
+
interrupt: () => this;
|
|
78
|
+
sendSilenceMessage: () => this;
|
|
79
|
+
dispose: () => this;
|
|
80
|
+
GetRequestId: () => string;
|
|
81
|
+
checkIsReady: () => boolean;
|
|
82
|
+
private readonly onIFrameErrorHandler;
|
|
83
|
+
private readonly onMessageHandler;
|
|
84
|
+
/**
|
|
85
|
+
* 处理msg类型的消息
|
|
86
|
+
*/
|
|
87
|
+
private readonly onReceiveMsgTypedMessageHandler;
|
|
88
|
+
/**
|
|
89
|
+
* 处理rtcState类型的消息
|
|
90
|
+
*/
|
|
91
|
+
private readonly onReceiveRtcStateTypedMessageHandler;
|
|
92
|
+
/**
|
|
93
|
+
* 处理wsState类型的消息
|
|
94
|
+
*/
|
|
95
|
+
private readonly onReceiveWsStateTypedMessageHandler;
|
|
96
|
+
/**
|
|
97
|
+
* 应用配置项到iframe元素上
|
|
98
|
+
*/
|
|
99
|
+
private readonly applyOptions;
|
|
100
|
+
private readonly refreshRequestId;
|
|
101
|
+
private readonly onRenderFinished;
|
|
102
|
+
}
|