@aj-archipelago/cortex 1.3.5 → 1.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/helper-apps/cortex-autogen/agents.py +31 -2
- package/helper-apps/cortex-realtime-voice-server/.env.sample +6 -0
- package/helper-apps/cortex-realtime-voice-server/README.md +22 -0
- package/helper-apps/cortex-realtime-voice-server/bun.lockb +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/bun.lockb +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/index.html +12 -0
- package/helper-apps/cortex-realtime-voice-server/client/package.json +65 -0
- package/helper-apps/cortex-realtime-voice-server/client/postcss.config.js +6 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/favicon.ico +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/index.html +43 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/logo192.png +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/logo512.png +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/manifest.json +25 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/robots.txt +3 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/sounds/connect.mp3 +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/sounds/disconnect.mp3 +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/App.test.tsx +9 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/App.tsx +126 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/SettingsModal.tsx +207 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/Chat.tsx +553 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubble.tsx +22 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubbleLeft.tsx +22 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubbleRight.tsx +21 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatMessage.tsx +27 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatMessageInput.tsx +74 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatTile.tsx +211 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/SoundEffects.ts +56 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavPacker.ts +112 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavRecorder.ts +571 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavStreamPlayer.ts +290 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/analysis/AudioAnalysis.ts +186 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/analysis/constants.ts +59 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/worklets/AudioProcessor.ts +214 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/worklets/StreamProcessor.ts +183 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/AudioVisualizer.tsx +151 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/CopyButton.tsx +32 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/ImageOverlay.tsx +166 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/MicrophoneVisualizer.tsx +95 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/ScreenshotCapture.tsx +116 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/hooks/useWindowResize.ts +27 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/utils/audio.ts +33 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/index.css +20 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/index.tsx +19 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/logo.svg +1 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/react-app-env.d.ts +1 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/reportWebVitals.ts +15 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/setupTests.ts +5 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/utils/logger.ts +45 -0
- package/helper-apps/cortex-realtime-voice-server/client/tailwind.config.js +14 -0
- package/helper-apps/cortex-realtime-voice-server/client/tsconfig.json +30 -0
- package/helper-apps/cortex-realtime-voice-server/client/vite.config.ts +22 -0
- package/helper-apps/cortex-realtime-voice-server/index.ts +19 -0
- package/helper-apps/cortex-realtime-voice-server/package.json +28 -0
- package/helper-apps/cortex-realtime-voice-server/src/ApiServer.ts +35 -0
- package/helper-apps/cortex-realtime-voice-server/src/SocketServer.ts +737 -0
- package/helper-apps/cortex-realtime-voice-server/src/Tools.ts +520 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/expert.ts +29 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/image.ts +29 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/memory.ts +91 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/reason.ts +29 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/search.ts +30 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/style.ts +31 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/utils.ts +95 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/vision.ts +34 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/client.ts +499 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/realtimeTypes.ts +279 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/socket.ts +27 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/transcription.ts +75 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/utils.ts +33 -0
- package/helper-apps/cortex-realtime-voice-server/src/utils/logger.ts +45 -0
- package/helper-apps/cortex-realtime-voice-server/src/utils/prompt.ts +81 -0
- package/helper-apps/cortex-realtime-voice-server/tsconfig.json +28 -0
- package/package.json +1 -1
- package/pathways/basePathway.js +3 -1
- package/pathways/system/entity/memory/sys_memory_manager.js +3 -0
- package/pathways/system/entity/memory/sys_memory_update.js +44 -45
- package/pathways/system/entity/memory/sys_read_memory.js +86 -6
- package/pathways/system/entity/memory/sys_search_memory.js +66 -0
- package/pathways/system/entity/shared/sys_entity_constants.js +2 -2
- package/pathways/system/entity/sys_entity_continue.js +2 -1
- package/pathways/system/entity/sys_entity_start.js +10 -0
- package/pathways/system/entity/sys_generator_expert.js +0 -2
- package/pathways/system/entity/sys_generator_memory.js +31 -0
- package/pathways/system/entity/sys_generator_voice_sample.js +36 -0
- package/pathways/system/entity/sys_router_tool.js +13 -10
- package/pathways/system/sys_parse_numbered_object_list.js +1 -1
- package/server/pathwayResolver.js +41 -31
- package/server/plugins/azureVideoTranslatePlugin.js +28 -16
- package/server/plugins/claude3VertexPlugin.js +0 -9
- package/server/plugins/gemini15ChatPlugin.js +18 -5
- package/server/plugins/modelPlugin.js +27 -6
- package/server/plugins/openAiChatPlugin.js +10 -8
- package/server/plugins/openAiVisionPlugin.js +56 -0
- package/tests/memoryfunction.test.js +73 -1
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import {type ChatMessage, type CortexVariables, getCortexResponse} from "./utils";
|
|
2
|
+
|
|
3
|
+
const WRITE_QUERY = `
|
|
4
|
+
query Reason($text: String, $contextId: String, $chatHistory: [MultiMessage], $aiName: String) {
|
|
5
|
+
sys_entity_continue(text: $text, contextId: $contextId, chatHistory: $chatHistory, aiName: $aiName, generatorPathway: "sys_generator_reasoning", voiceResponse: true) {
|
|
6
|
+
result
|
|
7
|
+
tool
|
|
8
|
+
errors
|
|
9
|
+
warnings
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
`
|
|
13
|
+
|
|
14
|
+
export async function reason(contextId: string,
|
|
15
|
+
aiName: string,
|
|
16
|
+
chatHistory: ChatMessage[],
|
|
17
|
+
text: string) {
|
|
18
|
+
|
|
19
|
+
const variables: CortexVariables = {
|
|
20
|
+
chatHistory,
|
|
21
|
+
contextId,
|
|
22
|
+
aiName,
|
|
23
|
+
text
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const res = await getCortexResponse(variables, WRITE_QUERY);
|
|
27
|
+
|
|
28
|
+
return res.sys_entity_continue;
|
|
29
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import {type ChatMessage, type CortexVariables, type DataSource, getCortexResponse} from "./utils";
|
|
2
|
+
|
|
3
|
+
const SEARCH_QUERY = `
|
|
4
|
+
query Search($text: String, $contextId: String, $chatHistory: [MultiMessage], $aiName: String, $dataSources: [String]) {
|
|
5
|
+
sys_entity_continue(text: $text, contextId: $contextId, chatHistory: $chatHistory, aiName: $aiName, dataSources: $dataSources, generatorPathway: "sys_generator_results", voiceResponse: true) {
|
|
6
|
+
result
|
|
7
|
+
tool
|
|
8
|
+
errors
|
|
9
|
+
warnings
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
`
|
|
13
|
+
|
|
14
|
+
export async function search(contextId: string,
|
|
15
|
+
aiName: string,
|
|
16
|
+
chatHistory: ChatMessage[],
|
|
17
|
+
dataSources: DataSource[],
|
|
18
|
+
text: string) {
|
|
19
|
+
const variables: CortexVariables = {
|
|
20
|
+
chatHistory,
|
|
21
|
+
contextId,
|
|
22
|
+
aiName,
|
|
23
|
+
dataSources,
|
|
24
|
+
text
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const res = await getCortexResponse(variables, SEARCH_QUERY);
|
|
28
|
+
|
|
29
|
+
return res.sys_entity_continue;
|
|
30
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import {type ChatMessage, type CortexVariables, getCortexResponse} from "./utils";
|
|
2
|
+
|
|
3
|
+
const STYLE_QUERY = `
|
|
4
|
+
query Style($text: String, $contextId: String, $chatHistory: [MultiMessage], $aiName: String, $aiStyle: String) {
|
|
5
|
+
sys_generator_voice_sample(text: $text, contextId: $contextId, chatHistory: $chatHistory, aiName: $aiName, aiStyle: $aiStyle) {
|
|
6
|
+
result
|
|
7
|
+
tool
|
|
8
|
+
errors
|
|
9
|
+
warnings
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
`
|
|
13
|
+
|
|
14
|
+
export async function style(contextId: string,
|
|
15
|
+
aiName: string,
|
|
16
|
+
aiStyle: string,
|
|
17
|
+
chatHistory: ChatMessage[],
|
|
18
|
+
text: string) {
|
|
19
|
+
|
|
20
|
+
const variables: CortexVariables = {
|
|
21
|
+
chatHistory,
|
|
22
|
+
contextId,
|
|
23
|
+
aiName,
|
|
24
|
+
aiStyle,
|
|
25
|
+
text
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const res = await getCortexResponse(variables, STYLE_QUERY);
|
|
29
|
+
|
|
30
|
+
return res.sys_generator_voice_sample;
|
|
31
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { logger } from '../utils/logger';
|
|
2
|
+
|
|
3
|
+
function getCortexApiKey() {
|
|
4
|
+
if (process.env.NODE_ENV === 'production') {
|
|
5
|
+
return process.env.CORTEX_API_KEY || ''
|
|
6
|
+
} else if (process.env.NODE_ENV === 'test') {
|
|
7
|
+
return process.env.CORTEX_DEV_API_KEY || ''
|
|
8
|
+
}
|
|
9
|
+
return '';
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function getCortexUrl() {
|
|
13
|
+
if (process.env.NODE_ENV === 'production') {
|
|
14
|
+
return 'https://cortex.aljazeera.com/graphql'
|
|
15
|
+
} else if (process.env.NODE_ENV === 'test') {
|
|
16
|
+
return 'https://cortex.aljazeera.com/dev/graphql';
|
|
17
|
+
}
|
|
18
|
+
return 'http://localhost:4000/graphql';
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function getHeaders() {
|
|
22
|
+
const headers: HeadersInit = new Headers();
|
|
23
|
+
headers.set('accept', 'application/json');
|
|
24
|
+
headers.set('Content-Type', 'application/json');
|
|
25
|
+
headers.set('ocp-apim-subscription-key', getCortexApiKey());
|
|
26
|
+
return headers;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export type ChatMessage = { role: string, content: string }
|
|
30
|
+
export type DataSource = "mydata" | "aja" | "aje" | "wires" | "bing"
|
|
31
|
+
|
|
32
|
+
export const MemorySection = {
|
|
33
|
+
memorySelf: "memorySelf",
|
|
34
|
+
memoryUser: "memoryUser",
|
|
35
|
+
memoryTopics: "memoryTopics",
|
|
36
|
+
memoryDirectives: "memoryDirectives",
|
|
37
|
+
memoryAll: "memoryAll"
|
|
38
|
+
} as const;
|
|
39
|
+
|
|
40
|
+
export type MemorySection = typeof MemorySection[keyof typeof MemorySection];
|
|
41
|
+
|
|
42
|
+
export type CortexVariables = {
|
|
43
|
+
contextId?: string,
|
|
44
|
+
aiName?: string,
|
|
45
|
+
aiStyle?: string,
|
|
46
|
+
chatHistory?: ChatMessage[],
|
|
47
|
+
text?: string,
|
|
48
|
+
useMemory?: boolean,
|
|
49
|
+
language?: string,
|
|
50
|
+
dataSources?: DataSource[];
|
|
51
|
+
section?: MemorySection;
|
|
52
|
+
width?: number;
|
|
53
|
+
height?: number;
|
|
54
|
+
size?: string;
|
|
55
|
+
style?: string;
|
|
56
|
+
priority?: number;
|
|
57
|
+
recentHours?: number;
|
|
58
|
+
numResults?: number;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function truncateBody(body: any): string {
|
|
62
|
+
const str = JSON.stringify(body);
|
|
63
|
+
if (str.length <= 5000) return str;
|
|
64
|
+
|
|
65
|
+
const halfLength = 2500;
|
|
66
|
+
return str.substring(0, halfLength) + '...' + str.substring(str.length - halfLength);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export async function getCortexResponse(
|
|
70
|
+
variables: CortexVariables,
|
|
71
|
+
query: string) {
|
|
72
|
+
const headers = getHeaders();
|
|
73
|
+
const body = {
|
|
74
|
+
query,
|
|
75
|
+
variables
|
|
76
|
+
}
|
|
77
|
+
logger.log(`Cortex URL: ${getCortexUrl()}`);
|
|
78
|
+
// logger.log(`Cortex Body: ${truncateBody(body)}`);
|
|
79
|
+
// logger.log(`Cortex Headers: ${JSON.stringify(headers)}`);
|
|
80
|
+
const res = await fetch(getCortexUrl(), {
|
|
81
|
+
method: 'POST',
|
|
82
|
+
headers,
|
|
83
|
+
body: JSON.stringify(body),
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
if (!res.ok) {
|
|
87
|
+
logger.error('Failed to fetch data:', res);
|
|
88
|
+
throw new Error('Failed to fetch data')
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const responseObject = await res.json();
|
|
92
|
+
// Debug logging can be enabled/disabled via logger's environment control
|
|
93
|
+
logger.debug('cortex response', responseObject);
|
|
94
|
+
return responseObject.data;
|
|
95
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import {type ChatMessage, type CortexVariables, getCortexResponse} from "./utils";
|
|
2
|
+
|
|
3
|
+
export type MultiMessage = {
|
|
4
|
+
role: string;
|
|
5
|
+
content: string | string[];
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
const VISION_QUERY = `
|
|
9
|
+
query Vision($text: String, $contextId: String, $chatHistory: [MultiMessage], $aiName: String) {
|
|
10
|
+
sys_generator_video_vision(text: $text, contextId: $contextId, chatHistory: $chatHistory, aiName: $aiName) {
|
|
11
|
+
result
|
|
12
|
+
tool
|
|
13
|
+
errors
|
|
14
|
+
warnings
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
`
|
|
18
|
+
|
|
19
|
+
export async function vision(contextId: string,
|
|
20
|
+
aiName: string,
|
|
21
|
+
chatHistory: (ChatMessage | MultiMessage)[],
|
|
22
|
+
text: string) {
|
|
23
|
+
|
|
24
|
+
const variables: Omit<CortexVariables, 'chatHistory'> & { chatHistory: (ChatMessage | MultiMessage)[] } = {
|
|
25
|
+
chatHistory,
|
|
26
|
+
contextId,
|
|
27
|
+
aiName,
|
|
28
|
+
text
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const res = await getCortexResponse(variables as CortexVariables, VISION_QUERY);
|
|
32
|
+
|
|
33
|
+
return res.sys_generator_video_vision;
|
|
34
|
+
}
|