@aj-archipelago/cortex 1.3.46 → 1.3.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/config.js +1 -1
  2. package/lib/pathwayTools.js +2 -2
  3. package/package.json +1 -1
  4. package/pathways/system/entity/sys_entity_agent.js +157 -133
  5. package/pathways/system/entity/tools/sys_tool_coding.js +3 -0
  6. package/pathways/system/entity/tools/sys_tool_cognitive_search.js +4 -4
  7. package/pathways/system/entity/tools/sys_tool_readfile.js +2 -2
  8. package/pathways/system/entity/tools/sys_tool_remember.js +23 -1
  9. package/server/pathwayResolver.js +15 -2
  10. package/server/plugins/openAiVisionPlugin.js +8 -1
package/config.js CHANGED
@@ -138,7 +138,7 @@ var config = convict({
138
138
 
139
139
  AI_EXPERTISE: "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.",
140
140
 
141
- AI_GROUNDING_INSTRUCTIONS: "Grounding your response: If you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the sourcewhen using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual:cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
141
+ AI_GROUNDING_INSTRUCTIONS: "Grounding your response: If you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the sourcewhen using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual :cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
142
142
 
143
143
  AI_STYLE_OPENAI: "oai-gpt41",
144
144
  AI_STYLE_ANTHROPIC: "claude-35-sonnet-vertex",
package/lib/pathwayTools.js CHANGED
@@ -178,12 +178,12 @@ const gpt3Decode = (text) => {
178
178
  return decode(text);
179
179
  }
180
180
 
181
- const say = async (requestId, message, maxMessageLength = Infinity, voiceResponse = true) => {
181
+ const say = async (requestId, message, maxMessageLength = Infinity, voiceResponse = true, isEphemeral = true) => {
182
182
  try {
183
183
  const chunks = getSemanticChunks(message, maxMessageLength);
184
184
 
185
185
  const info = JSON.stringify({
186
- ephemeral: true,
186
+ ephemeral: isEphemeral,
187
187
  });
188
188
 
189
189
  for (let chunk of chunks) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aj-archipelago/cortex",
3
- "version": "1.3.46",
3
+ "version": "1.3.48",
4
4
  "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
5
5
  "private": false,
6
6
  "repository": {
package/pathways/system/entity/sys_entity_agent.js CHANGED
@@ -1,10 +1,11 @@
1
1
  // sys_entity_agent.js
2
2
  // Agentic extension of the entity system that uses OpenAI's tool calling API
3
+ const MAX_TOOL_CALLS = 50;
4
+
3
5
  import { callPathway, callTool, say } from '../../../lib/pathwayTools.js';
4
6
  import logger from '../../../lib/logger.js';
5
7
  import { config } from '../../../config.js';
6
8
  import { chatArgsHasImageUrl, removeOldImageAndFileContent } from '../../../lib/util.js';
7
- import { insertToolCallAndResults } from './memory/shared/sys_memory_helpers.js';
8
9
  import { Prompt } from '../../../server/prompt.js';
9
10
  import { getToolsForEntity, loadEntityConfig } from './tools/shared/sys_entity_tools.js';
10
11
 
@@ -41,153 +42,176 @@ export default {
41
42
  const { tool_calls } = message;
42
43
  const pathwayResolver = resolver;
43
44
  const { entityTools, entityToolsOpenAiFormat } = args;
45
+
46
+ pathwayResolver.toolCallCount = (pathwayResolver.toolCallCount || 0);
44
47
 
45
- // Make a deep copy of the initial chat history
46
- const initialMessages = JSON.parse(JSON.stringify(args.chatHistory || []));
48
+ const preToolCallMessages = JSON.parse(JSON.stringify(args.chatHistory || []));
49
+ const finalMessages = JSON.parse(JSON.stringify(preToolCallMessages));
47
50
 
48
51
  if (tool_calls) {
49
- // Execute tool calls in parallel but with isolated message histories
50
- const toolResults = await Promise.all(tool_calls.map(async (toolCall) => {
51
- try {
52
- if (!toolCall?.function?.arguments) {
53
- throw new Error('Invalid tool call structure: missing function arguments');
54
- }
52
+ if (pathwayResolver.toolCallCount < MAX_TOOL_CALLS) {
53
+ // Execute tool calls in parallel but with isolated message histories
54
+ const toolResults = await Promise.all(tool_calls.map(async (toolCall) => {
55
+ try {
56
+ if (!toolCall?.function?.arguments) {
57
+ throw new Error('Invalid tool call structure: missing function arguments');
58
+ }
59
+
60
+ const toolArgs = JSON.parse(toolCall.function.arguments);
61
+ const toolFunction = toolCall.function.name.toLowerCase();
62
+
63
+ // Create an isolated copy of messages for this tool
64
+ const toolMessages = JSON.parse(JSON.stringify(preToolCallMessages));
65
+
66
+ // Get the tool definition to check for icon
67
+ const toolDefinition = entityTools[toolFunction]?.definition;
68
+ const toolIcon = toolDefinition?.icon || '🛠️';
69
+
70
+ // Report status to the user
71
+ const toolUserMessage = toolArgs.userMessage || `Executing tool: ${toolCall.function.name} - ${JSON.stringify(toolArgs)}`;
72
+ const messageWithIcon = toolIcon ? `${toolIcon}&nbsp;&nbsp;${toolUserMessage}` : toolUserMessage;
73
+ await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
74
+
75
+ const toolResult = await callTool(toolFunction, {
76
+ ...args,
77
+ ...toolArgs,
78
+ toolFunction,
79
+ chatHistory: toolMessages,
80
+ stream: false
81
+ }, entityTools, pathwayResolver);
82
+
83
+ // Tool calls and results need to be paired together in the message history
84
+ // Add the tool call to the isolated message history
85
+ toolMessages.push({
86
+ role: "assistant",
87
+ content: "",
88
+ tool_calls: [{
89
+ id: toolCall.id,
90
+ type: "function",
91
+ function: {
92
+ name: toolCall.function.name,
93
+ arguments: JSON.stringify(toolArgs)
94
+ }
95
+ }]
96
+ });
97
+
98
+ // Add the tool result to the isolated message history
99
+ const toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult?.result || toolResult);
55
100
 
56
- const toolArgs = JSON.parse(toolCall.function.arguments);
57
- const toolFunction = toolCall.function.name.toLowerCase();
58
-
59
- // Create an isolated copy of messages for this tool
60
- const toolMessages = JSON.parse(JSON.stringify(initialMessages));
61
-
62
- // Get the tool definition to check for icon
63
- const toolDefinition = entityTools[toolFunction]?.definition;
64
- const toolIcon = toolDefinition?.icon || '🛠️';
65
-
66
- // Report status to the user
67
- const toolUserMessage = toolArgs.userMessage || `Executing tool: ${toolCall.function.name} - ${JSON.stringify(toolArgs)}`;
68
- const messageWithIcon = toolIcon ? `${toolIcon}&nbsp;&nbsp;${toolUserMessage}` : toolUserMessage;
69
- await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
70
-
71
- const toolResult = await callTool(toolFunction, {
72
- ...args,
73
- ...toolArgs,
74
- toolFunction,
75
- chatHistory: toolMessages,
76
- stream: false
77
- }, entityTools, pathwayResolver);
78
-
79
- // Tool calls and results need to be paired together in the message history
80
- // Add the tool call to the isolated message history
81
- toolMessages.push({
82
- role: "assistant",
83
- content: "",
84
- tool_calls: [{
85
- id: toolCall.id,
86
- type: "function",
87
- function: {
88
- name: toolCall.function.name,
89
- arguments: JSON.stringify(toolArgs)
90
- }
91
- }]
92
- });
93
-
94
- // Add the tool result to the isolated message history
95
- const toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult?.result || toolResult);
96
-
97
- toolMessages.push({
98
- role: "tool",
99
- tool_call_id: toolCall.id,
100
- name: toolCall.function.name,
101
- content: toolResultContent
102
- });
103
-
104
- // Add the screenshots using OpenAI image format
105
- if (toolResult?.toolImages && toolResult.toolImages.length > 0) {
106
101
  toolMessages.push({
107
- role: "user",
108
- content: [
109
- {
110
- type: "text",
111
- text: "The tool with id " + toolCall.id + " has also supplied you with these images."
112
- },
113
- ...toolResult.toolImages.map(toolImage => ({
114
- type: "image_url",
115
- image_url: {
116
- url: `data:image/png;base64,${toolImage}`
117
- }
118
- }))
119
- ]
102
+ role: "tool",
103
+ tool_call_id: toolCall.id,
104
+ name: toolCall.function.name,
105
+ content: toolResultContent
120
106
  });
121
- }
122
107
 
123
- return {
124
- success: true,
125
- result: toolResult,
126
- toolCall,
127
- toolArgs,
128
- toolFunction,
129
- messages: toolMessages
130
- };
131
- } catch (error) {
132
- logger.error(`Error executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
133
-
134
- // Create error message history
135
- const errorMessages = JSON.parse(JSON.stringify(initialMessages));
136
- errorMessages.push({
137
- role: "assistant",
138
- content: "",
139
- tool_calls: [{
140
- id: toolCall.id,
141
- type: "function",
142
- function: {
143
- name: toolCall.function.name,
144
- arguments: JSON.stringify(toolCall.function.arguments)
145
- }
146
- }]
147
- });
148
- errorMessages.push({
149
- role: "tool",
150
- tool_call_id: toolCall.id,
151
- name: toolCall.function.name,
152
- content: `Error: ${error.message}`
153
- });
154
-
155
- return {
156
- success: false,
157
- error: error.message,
158
- toolCall,
159
- toolArgs: toolCall?.function?.arguments ? JSON.parse(toolCall.function.arguments) : {},
160
- toolFunction: toolCall?.function?.name?.toLowerCase() || 'unknown',
161
- messages: errorMessages
162
- };
163
- }
164
- }));
165
-
166
- // Merge all message histories in order
167
- let finalMessages = JSON.parse(JSON.stringify(initialMessages));
168
- for (const result of toolResults) {
169
- try {
170
- if (!result?.messages) {
171
- logger.error('Invalid tool result structure, skipping message history update');
172
- continue;
108
+ // Add the screenshots using OpenAI image format
109
+ if (toolResult?.toolImages && toolResult.toolImages.length > 0) {
110
+ toolMessages.push({
111
+ role: "user",
112
+ content: [
113
+ {
114
+ type: "text",
115
+ text: "The tool with id " + toolCall.id + " has also supplied you with these images."
116
+ },
117
+ ...toolResult.toolImages.map(toolImage => ({
118
+ type: "image_url",
119
+ image_url: {
120
+ url: `data:image/png;base64,${toolImage}`
121
+ }
122
+ }))
123
+ ]
124
+ });
125
+ }
126
+
127
+ return {
128
+ success: true,
129
+ result: toolResult,
130
+ toolCall,
131
+ toolArgs,
132
+ toolFunction,
133
+ messages: toolMessages
134
+ };
135
+ } catch (error) {
136
+ logger.error(`Error executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
137
+
138
+ // Create error message history
139
+ const errorMessages = JSON.parse(JSON.stringify(preToolCallMessages));
140
+ errorMessages.push({
141
+ role: "assistant",
142
+ content: "",
143
+ tool_calls: [{
144
+ id: toolCall.id,
145
+ type: "function",
146
+ function: {
147
+ name: toolCall.function.name,
148
+ arguments: JSON.stringify(toolCall.function.arguments)
149
+ }
150
+ }]
151
+ });
152
+ errorMessages.push({
153
+ role: "tool",
154
+ tool_call_id: toolCall.id,
155
+ name: toolCall.function.name,
156
+ content: `Error: ${error.message}`
157
+ });
158
+
159
+ return {
160
+ success: false,
161
+ error: error.message,
162
+ toolCall,
163
+ toolArgs: toolCall?.function?.arguments ? JSON.parse(toolCall.function.arguments) : {},
164
+ toolFunction: toolCall?.function?.name?.toLowerCase() || 'unknown',
165
+ messages: errorMessages
166
+ };
167
+ }
168
+ }));
169
+
170
+ // Merge all message histories in order
171
+ for (const result of toolResults) {
172
+ try {
173
+ if (!result?.messages) {
174
+ logger.error('Invalid tool result structure, skipping message history update');
175
+ continue;
176
+ }
177
+
178
+ // Add only the new messages from this tool's history
179
+ const newMessages = result.messages.slice(preToolCallMessages.length);
180
+ finalMessages.push(...newMessages);
181
+ } catch (error) {
182
+ logger.error(`Error merging message history for tool result: ${error.message}`);
173
183
  }
184
+ }
174
185
 
175
- // Add only the new messages from this tool's history
176
- const newMessages = result.messages.slice(initialMessages.length);
177
- finalMessages.push(...newMessages);
178
- } catch (error) {
179
- logger.error(`Error merging message history for tool result: ${error.message}`);
186
+ // Check if any tool calls failed
187
+ const failedTools = toolResults.filter(result => !result.success);
188
+ if (failedTools.length > 0) {
189
+ logger.warn(`Some tool calls failed: ${failedTools.map(t => t.error).join(', ')}`);
180
190
  }
181
- }
182
191
 
183
- // Check if any tool calls failed
184
- const failedTools = toolResults.filter(result => !result.success);
185
- if (failedTools.length > 0) {
186
- logger.warn(`Some tool calls failed: ${failedTools.map(t => t.error).join(', ')}`);
192
+ pathwayResolver.toolCallCount = (pathwayResolver.toolCallCount || 0) + toolResults.length;
193
+
194
+ } else {
195
+ finalMessages.push({
196
+ role: "user",
197
+ content: [
198
+ {
199
+ type: "text",
200
+ text: "This agent has reached the maximum number of tool calls - no more tool calls will be executed."
201
+ }
202
+ ]
203
+ });
204
+
187
205
  }
188
206
 
189
207
  args.chatHistory = finalMessages;
190
208
 
209
+ // clear any accumulated pathwayResolver errors from the tools
210
+ pathwayResolver.errors = [];
211
+
212
+ // Add a line break to avoid running output together
213
+ await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `\n`, 1000, false, false);
214
+
191
215
  return await pathwayResolver.promptAndParse({
192
216
  ...args,
193
217
  tools: entityToolsOpenAiFormat,
package/pathways/system/entity/tools/sys_tool_coding.js CHANGED
@@ -22,6 +22,8 @@ export default {
22
22
  useInputChunking: false,
23
23
  enableDuplicateRequests: false,
24
24
  timeout: 600,
25
+ // Tool disabled for now
26
+ /*
25
27
  toolDefinition: [{
26
28
  type: "function",
27
29
  icon: "💻",
@@ -44,6 +46,7 @@ export default {
44
46
  }
45
47
  }
46
48
  }],
49
+ */
47
50
 
48
51
  executePathway: async ({args, runAllPrompts, resolver}) => {
49
52
  if (args.detailedInstructions) {
package/pathways/system/entity/tools/sys_tool_cognitive_search.js CHANGED
@@ -24,7 +24,7 @@ export default {
24
24
  icon: "📂",
25
25
  function: {
26
26
  name: "SearchPersonal",
27
- description: "Search through the user's index of personal documents and indexed uploaded files. Use this for finding information in user-provided content or if the user refers to a file or a document that you don't see elsewhere in your context.",
27
+ description: "Search through the user's index of personal documents and indexed uploaded files and retrieve the content of the files. Use this tool if the user refers to a file or a document that you don't see uploaded elsewhere in your context. Some file types (e.g. Word documents, Excel documents, very large files, etc.) cannot be attached to a message and will be chunked and indexed and stored in the personal index.",
28
28
  parameters: {
29
29
  type: "object",
30
30
  properties: {
@@ -58,7 +58,7 @@ export default {
58
58
  icon: "📰",
59
59
  function: {
60
60
  name: "SearchAJA",
61
- description: "Search through Al Jazeera Arabic news articles. Use this for finding Arabic news content.",
61
+ description: "Search Al Jazeera Arabic news articles. Use this for finding Arabic news content including the latest news and articles.",
62
62
  parameters: {
63
63
  type: "object",
64
64
  properties: {
@@ -92,7 +92,7 @@ export default {
92
92
  icon: "📰",
93
93
  function: {
94
94
  name: "SearchAJE",
95
- description: "Search through Al Jazeera English news articles. Use this for finding English news content.",
95
+ description: "Search Al Jazeera English news articles. Use this for finding English news content including the latest news and articles.",
96
96
  parameters: {
97
97
  type: "object",
98
98
  properties: {
@@ -126,7 +126,7 @@ export default {
126
126
  icon: "⚡️",
127
127
  function: {
128
128
  name: "SearchWires",
129
- description: "Search through news wires from all sources. Use this for finding the latest news and articles.",
129
+ description: "Search through news wires from Reuters, AFP, AP, and other news agencies. Use this for finding the latest news and articles from the wires.",
130
130
  parameters: {
131
131
  type: "object",
132
132
  properties: {
package/pathways/system/entity/tools/sys_tool_readfile.js CHANGED
@@ -27,7 +27,7 @@ export default {
27
27
  icon: "📄",
28
28
  function: {
29
29
  name: "PDF",
30
- description: "Use specifically for analyzing and answering questions about PDF file content.",
30
+ description: "Use specifically for analyzing and answering questions about PDF file content. Do not use this tool for analyzing and answering questions about other file types.",
31
31
  parameters: {
32
32
  type: "object",
33
33
  properties: {
@@ -49,7 +49,7 @@ export default {
49
49
  icon: "📝",
50
50
  function: {
51
51
  name: "Text",
52
- description: "Use specifically for analyzing and answering questions about text files.",
52
+ description: "Use specifically for analyzing and answering questions about text and csv files.",
53
53
  parameters: {
54
54
  type: "object",
55
55
  properties: {
package/pathways/system/entity/tools/sys_tool_remember.js CHANGED
@@ -8,11 +8,33 @@ export default {
8
8
  model: 'oai-gpt41-mini',
9
9
 
10
10
  toolDefinition: [{
11
+ type: "function",
12
+ icon: "🧩",
13
+ function: {
14
+ name: "SearchMemory",
15
+ description: "Use this tool to search your memory and remember information or details that may not be present in your short term or contextual memory. You should always use this tool before you answer questions about remembered information. It's critical that you never fabricate memories.",
16
+ parameters: {
17
+ type: "object",
18
+ properties: {
19
+ detailedInstructions: {
20
+ type: "string",
21
+ description: "Detailed description of what you want to see if you remember"
22
+ },
23
+ userMessage: {
24
+ type: "string",
25
+ description: "A user-friendly message that describes what you're doing with this tool"
26
+ }
27
+ },
28
+ required: ["detailedInstructions", "userMessage"]
29
+ }
30
+ }
31
+ },
32
+ {
11
33
  type: "function",
12
34
  icon: "🧩",
13
35
  function: {
14
36
  name: "Remember",
15
- description: "Use specifically to search your long term memory for information or details that may not be present in your short term memory. You should always use this tool before you tell the user you don't remember something. If the user asks you a question (like what's your favorite color) and you don't remember the answer, use this tool to search your long term memory for the answer before you tell the user you don't have one.",
37
+ description: "When the user asks you if you remember something, you must use this tool before you answer. It's critical that you never fabricate memories.",
16
38
  parameters: {
17
39
  type: "object",
18
40
  properties: {
package/server/pathwayResolver.js CHANGED
@@ -89,12 +89,24 @@ class PathwayResolver {
89
89
  responseData = await this.executePathway(args);
90
90
  }
91
91
  catch (error) {
92
+ this.errors.push(error.message || error.toString());
92
93
  publishRequestProgress({
93
94
  requestId: this.rootRequestId || this.requestId,
94
95
  progress: 1,
95
96
  data: '',
96
97
  info: '',
97
- error: error.message || error.toString()
98
+ error: this.errors.join(', ')
99
+ });
100
+ return;
101
+ }
102
+
103
+ if (!responseData) {
104
+ publishRequestProgress({
105
+ requestId: this.rootRequestId || this.requestId,
106
+ progress: 1,
107
+ data: '',
108
+ info: '',
109
+ error: this.errors.join(', ')
98
110
  });
99
111
  return;
100
112
  }
@@ -113,7 +125,8 @@ class PathwayResolver {
113
125
  progress: Math.min(completedCount, totalCount) / totalCount,
114
126
  // Clients expect these to be strings
115
127
  data: JSON.stringify(responseData || ''),
116
- info: this.tool || ''
128
+ info: this.tool || '',
129
+ error: this.errors.join(', ') || ''
117
130
  });
118
131
  }
119
132
  }
package/server/plugins/openAiVisionPlugin.js CHANGED
@@ -84,7 +84,14 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
84
84
  const { length, units } = this.getLength(content);
85
85
  const displayContent = this.shortenContent(content);
86
86
 
87
- logger.verbose(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`);
87
+ let logMessage = `message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`;
88
+
89
+ // Add tool calls to log if they exist
90
+ if (message.role === 'assistant' && message.tool_calls) {
91
+ logMessage += `, tool_calls: ${JSON.stringify(message.tool_calls)}`;
92
+ }
93
+
94
+ logger.verbose(logMessage);
88
95
  totalLength += length;
89
96
  totalUnits = units;
90
97
  });