@aj-archipelago/cortex 1.3.46 → 1.3.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/package.json +1 -1
  2. package/pathways/system/entity/sys_entity_agent.js +154 -133
  3. package/server/pathwayResolver.js +15 -2
  4. package/server/plugins/openAiVisionPlugin.js +8 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aj-archipelago/cortex",
3
- "version": "1.3.46",
3
+ "version": "1.3.47",
4
4
  "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
5
5
  "private": false,
6
6
  "repository": {
package/pathways/system/entity/sys_entity_agent.js CHANGED
@@ -1,10 +1,11 @@
1
1
  // sys_entity_agent.js
2
2
  // Agentic extension of the entity system that uses OpenAI's tool calling API
3
+ const MAX_TOOL_CALLS = 50;
4
+
3
5
  import { callPathway, callTool, say } from '../../../lib/pathwayTools.js';
4
6
  import logger from '../../../lib/logger.js';
5
7
  import { config } from '../../../config.js';
6
8
  import { chatArgsHasImageUrl, removeOldImageAndFileContent } from '../../../lib/util.js';
7
- import { insertToolCallAndResults } from './memory/shared/sys_memory_helpers.js';
8
9
  import { Prompt } from '../../../server/prompt.js';
9
10
  import { getToolsForEntity, loadEntityConfig } from './tools/shared/sys_entity_tools.js';
10
11
 
@@ -41,153 +42,173 @@ export default {
41
42
  const { tool_calls } = message;
42
43
  const pathwayResolver = resolver;
43
44
  const { entityTools, entityToolsOpenAiFormat } = args;
45
+
46
+ pathwayResolver.toolCallCount = (pathwayResolver.toolCallCount || 0);
44
47
 
45
- // Make a deep copy of the initial chat history
46
- const initialMessages = JSON.parse(JSON.stringify(args.chatHistory || []));
48
+ const preToolCallMessages = JSON.parse(JSON.stringify(args.chatHistory || []));
49
+ const finalMessages = JSON.parse(JSON.stringify(preToolCallMessages));
47
50
 
48
51
  if (tool_calls) {
49
- // Execute tool calls in parallel but with isolated message histories
50
- const toolResults = await Promise.all(tool_calls.map(async (toolCall) => {
51
- try {
52
- if (!toolCall?.function?.arguments) {
53
- throw new Error('Invalid tool call structure: missing function arguments');
54
- }
52
+ if (pathwayResolver.toolCallCount < MAX_TOOL_CALLS) {
53
+ // Execute tool calls in parallel but with isolated message histories
54
+ const toolResults = await Promise.all(tool_calls.map(async (toolCall) => {
55
+ try {
56
+ if (!toolCall?.function?.arguments) {
57
+ throw new Error('Invalid tool call structure: missing function arguments');
58
+ }
59
+
60
+ const toolArgs = JSON.parse(toolCall.function.arguments);
61
+ const toolFunction = toolCall.function.name.toLowerCase();
62
+
63
+ // Create an isolated copy of messages for this tool
64
+ const toolMessages = JSON.parse(JSON.stringify(preToolCallMessages));
65
+
66
+ // Get the tool definition to check for icon
67
+ const toolDefinition = entityTools[toolFunction]?.definition;
68
+ const toolIcon = toolDefinition?.icon || '🛠️';
69
+
70
+ // Report status to the user
71
+ const toolUserMessage = toolArgs.userMessage || `Executing tool: ${toolCall.function.name} - ${JSON.stringify(toolArgs)}`;
72
+ const messageWithIcon = toolIcon ? `${toolIcon}&nbsp;&nbsp;${toolUserMessage}` : toolUserMessage;
73
+ await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
74
+
75
+ const toolResult = await callTool(toolFunction, {
76
+ ...args,
77
+ ...toolArgs,
78
+ toolFunction,
79
+ chatHistory: toolMessages,
80
+ stream: false
81
+ }, entityTools, pathwayResolver);
82
+
83
+ // Tool calls and results need to be paired together in the message history
84
+ // Add the tool call to the isolated message history
85
+ toolMessages.push({
86
+ role: "assistant",
87
+ content: "",
88
+ tool_calls: [{
89
+ id: toolCall.id,
90
+ type: "function",
91
+ function: {
92
+ name: toolCall.function.name,
93
+ arguments: JSON.stringify(toolArgs)
94
+ }
95
+ }]
96
+ });
97
+
98
+ // Add the tool result to the isolated message history
99
+ const toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult?.result || toolResult);
55
100
 
56
- const toolArgs = JSON.parse(toolCall.function.arguments);
57
- const toolFunction = toolCall.function.name.toLowerCase();
58
-
59
- // Create an isolated copy of messages for this tool
60
- const toolMessages = JSON.parse(JSON.stringify(initialMessages));
61
-
62
- // Get the tool definition to check for icon
63
- const toolDefinition = entityTools[toolFunction]?.definition;
64
- const toolIcon = toolDefinition?.icon || '🛠️';
65
-
66
- // Report status to the user
67
- const toolUserMessage = toolArgs.userMessage || `Executing tool: ${toolCall.function.name} - ${JSON.stringify(toolArgs)}`;
68
- const messageWithIcon = toolIcon ? `${toolIcon}&nbsp;&nbsp;${toolUserMessage}` : toolUserMessage;
69
- await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
70
-
71
- const toolResult = await callTool(toolFunction, {
72
- ...args,
73
- ...toolArgs,
74
- toolFunction,
75
- chatHistory: toolMessages,
76
- stream: false
77
- }, entityTools, pathwayResolver);
78
-
79
- // Tool calls and results need to be paired together in the message history
80
- // Add the tool call to the isolated message history
81
- toolMessages.push({
82
- role: "assistant",
83
- content: "",
84
- tool_calls: [{
85
- id: toolCall.id,
86
- type: "function",
87
- function: {
88
- name: toolCall.function.name,
89
- arguments: JSON.stringify(toolArgs)
90
- }
91
- }]
92
- });
93
-
94
- // Add the tool result to the isolated message history
95
- const toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult?.result || toolResult);
96
-
97
- toolMessages.push({
98
- role: "tool",
99
- tool_call_id: toolCall.id,
100
- name: toolCall.function.name,
101
- content: toolResultContent
102
- });
103
-
104
- // Add the screenshots using OpenAI image format
105
- if (toolResult?.toolImages && toolResult.toolImages.length > 0) {
106
101
  toolMessages.push({
107
- role: "user",
108
- content: [
109
- {
110
- type: "text",
111
- text: "The tool with id " + toolCall.id + " has also supplied you with these images."
112
- },
113
- ...toolResult.toolImages.map(toolImage => ({
114
- type: "image_url",
115
- image_url: {
116
- url: `data:image/png;base64,${toolImage}`
117
- }
118
- }))
119
- ]
102
+ role: "tool",
103
+ tool_call_id: toolCall.id,
104
+ name: toolCall.function.name,
105
+ content: toolResultContent
120
106
  });
121
- }
122
107
 
123
- return {
124
- success: true,
125
- result: toolResult,
126
- toolCall,
127
- toolArgs,
128
- toolFunction,
129
- messages: toolMessages
130
- };
131
- } catch (error) {
132
- logger.error(`Error executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
133
-
134
- // Create error message history
135
- const errorMessages = JSON.parse(JSON.stringify(initialMessages));
136
- errorMessages.push({
137
- role: "assistant",
138
- content: "",
139
- tool_calls: [{
140
- id: toolCall.id,
141
- type: "function",
142
- function: {
143
- name: toolCall.function.name,
144
- arguments: JSON.stringify(toolCall.function.arguments)
145
- }
146
- }]
147
- });
148
- errorMessages.push({
149
- role: "tool",
150
- tool_call_id: toolCall.id,
151
- name: toolCall.function.name,
152
- content: `Error: ${error.message}`
153
- });
154
-
155
- return {
156
- success: false,
157
- error: error.message,
158
- toolCall,
159
- toolArgs: toolCall?.function?.arguments ? JSON.parse(toolCall.function.arguments) : {},
160
- toolFunction: toolCall?.function?.name?.toLowerCase() || 'unknown',
161
- messages: errorMessages
162
- };
163
- }
164
- }));
165
-
166
- // Merge all message histories in order
167
- let finalMessages = JSON.parse(JSON.stringify(initialMessages));
168
- for (const result of toolResults) {
169
- try {
170
- if (!result?.messages) {
171
- logger.error('Invalid tool result structure, skipping message history update');
172
- continue;
108
+ // Add the screenshots using OpenAI image format
109
+ if (toolResult?.toolImages && toolResult.toolImages.length > 0) {
110
+ toolMessages.push({
111
+ role: "user",
112
+ content: [
113
+ {
114
+ type: "text",
115
+ text: "The tool with id " + toolCall.id + " has also supplied you with these images."
116
+ },
117
+ ...toolResult.toolImages.map(toolImage => ({
118
+ type: "image_url",
119
+ image_url: {
120
+ url: `data:image/png;base64,${toolImage}`
121
+ }
122
+ }))
123
+ ]
124
+ });
125
+ }
126
+
127
+ return {
128
+ success: true,
129
+ result: toolResult,
130
+ toolCall,
131
+ toolArgs,
132
+ toolFunction,
133
+ messages: toolMessages
134
+ };
135
+ } catch (error) {
136
+ logger.error(`Error executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
137
+
138
+ // Create error message history
139
+ const errorMessages = JSON.parse(JSON.stringify(preToolCallMessages));
140
+ errorMessages.push({
141
+ role: "assistant",
142
+ content: "",
143
+ tool_calls: [{
144
+ id: toolCall.id,
145
+ type: "function",
146
+ function: {
147
+ name: toolCall.function.name,
148
+ arguments: JSON.stringify(toolCall.function.arguments)
149
+ }
150
+ }]
151
+ });
152
+ errorMessages.push({
153
+ role: "tool",
154
+ tool_call_id: toolCall.id,
155
+ name: toolCall.function.name,
156
+ content: `Error: ${error.message}`
157
+ });
158
+
159
+ return {
160
+ success: false,
161
+ error: error.message,
162
+ toolCall,
163
+ toolArgs: toolCall?.function?.arguments ? JSON.parse(toolCall.function.arguments) : {},
164
+ toolFunction: toolCall?.function?.name?.toLowerCase() || 'unknown',
165
+ messages: errorMessages
166
+ };
167
+ }
168
+ }));
169
+
170
+ // Merge all message histories in order
171
+ for (const result of toolResults) {
172
+ try {
173
+ if (!result?.messages) {
174
+ logger.error('Invalid tool result structure, skipping message history update');
175
+ continue;
176
+ }
177
+
178
+ // Add only the new messages from this tool's history
179
+ const newMessages = result.messages.slice(preToolCallMessages.length);
180
+ finalMessages.push(...newMessages);
181
+ } catch (error) {
182
+ logger.error(`Error merging message history for tool result: ${error.message}`);
173
183
  }
184
+ }
174
185
 
175
- // Add only the new messages from this tool's history
176
- const newMessages = result.messages.slice(initialMessages.length);
177
- finalMessages.push(...newMessages);
178
- } catch (error) {
179
- logger.error(`Error merging message history for tool result: ${error.message}`);
186
+ // Check if any tool calls failed
187
+ const failedTools = toolResults.filter(result => !result.success);
188
+ if (failedTools.length > 0) {
189
+ logger.warn(`Some tool calls failed: ${failedTools.map(t => t.error).join(', ')}`);
180
190
  }
181
- }
182
191
 
183
- // Check if any tool calls failed
184
- const failedTools = toolResults.filter(result => !result.success);
185
- if (failedTools.length > 0) {
186
- logger.warn(`Some tool calls failed: ${failedTools.map(t => t.error).join(', ')}`);
192
+ pathwayResolver.toolCallCount = (pathwayResolver.toolCallCount || 0) + toolResults.length;
193
+
194
+ } else {
195
+ finalMessages.push({
196
+ role: "user",
197
+ content: [
198
+ {
199
+ type: "text",
200
+ text: "This agent has reached the maximum number of tool calls - no more tool calls will be executed."
201
+ }
202
+ ]
203
+ });
204
+
187
205
  }
188
206
 
189
207
  args.chatHistory = finalMessages;
190
208
 
209
+ // clear any accumulated pathwayResolver errors from the tools
210
+ pathwayResolver.errors = [];
211
+
191
212
  return await pathwayResolver.promptAndParse({
192
213
  ...args,
193
214
  tools: entityToolsOpenAiFormat,
package/server/pathwayResolver.js CHANGED
@@ -89,12 +89,24 @@ class PathwayResolver {
89
89
  responseData = await this.executePathway(args);
90
90
  }
91
91
  catch (error) {
92
+ this.errors.push(error.message || error.toString());
92
93
  publishRequestProgress({
93
94
  requestId: this.rootRequestId || this.requestId,
94
95
  progress: 1,
95
96
  data: '',
96
97
  info: '',
97
- error: error.message || error.toString()
98
+ error: this.errors.join(', ')
99
+ });
100
+ return;
101
+ }
102
+
103
+ if (!responseData) {
104
+ publishRequestProgress({
105
+ requestId: this.rootRequestId || this.requestId,
106
+ progress: 1,
107
+ data: '',
108
+ info: '',
109
+ error: this.errors.join(', ')
98
110
  });
99
111
  return;
100
112
  }
@@ -113,7 +125,8 @@ class PathwayResolver {
113
125
  progress: Math.min(completedCount, totalCount) / totalCount,
114
126
  // Clients expect these to be strings
115
127
  data: JSON.stringify(responseData || ''),
116
- info: this.tool || ''
128
+ info: this.tool || '',
129
+ error: this.errors.join(', ') || ''
117
130
  });
118
131
  }
119
132
  }
package/server/plugins/openAiVisionPlugin.js CHANGED
@@ -84,7 +84,14 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
84
84
  const { length, units } = this.getLength(content);
85
85
  const displayContent = this.shortenContent(content);
86
86
 
87
- logger.verbose(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`);
87
+ let logMessage = `message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`;
88
+
89
+ // Add tool calls to log if they exist
90
+ if (message.role === 'assistant' && message.tool_calls) {
91
+ logMessage += `, tool_calls: ${JSON.stringify(message.tool_calls)}`;
92
+ }
93
+
94
+ logger.verbose(logMessage);
88
95
  totalLength += length;
89
96
  totalUnits = units;
90
97
  });