@minded-ai/mindedjs 2.0.25 → 2.0.26-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +7 -0
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +80 -24
- package/dist/agent.js.map +1 -1
- package/dist/browserTask/cdp.d.ts +23 -0
- package/dist/browserTask/cdp.d.ts.map +1 -0
- package/dist/browserTask/cdp.js +162 -0
- package/dist/browserTask/cdp.js.map +1 -0
- package/dist/browserTask/cookieStore.py +108 -0
- package/dist/browserTask/executeBrowserTask.d.ts.map +1 -1
- package/dist/browserTask/executeBrowserTask.js +9 -5
- package/dist/browserTask/executeBrowserTask.js.map +1 -1
- package/dist/browserTask/executeBrowserTask.py +19 -52
- package/dist/browserTask/localBrowserTask.d.ts.map +1 -1
- package/dist/browserTask/localBrowserTask.js +45 -8
- package/dist/browserTask/localBrowserTask.js.map +1 -1
- package/dist/browserTask/types.d.ts +4 -6
- package/dist/browserTask/types.d.ts.map +1 -1
- package/dist/cli/index.js +0 -0
- package/dist/guidelines/guidelinesManager.d.ts +37 -0
- package/dist/guidelines/guidelinesManager.d.ts.map +1 -0
- package/dist/guidelines/guidelinesManager.js +172 -0
- package/dist/guidelines/guidelinesManager.js.map +1 -0
- package/dist/internalTools/retell.d.ts +12 -0
- package/dist/internalTools/retell.d.ts.map +1 -0
- package/dist/internalTools/retell.js +54 -0
- package/dist/internalTools/retell.js.map +1 -0
- package/dist/internalTools/sendPlaceholderMessage.d.ts +14 -0
- package/dist/internalTools/sendPlaceholderMessage.d.ts.map +1 -0
- package/dist/internalTools/sendPlaceholderMessage.js +61 -0
- package/dist/internalTools/sendPlaceholderMessage.js.map +1 -0
- package/dist/nodes/addBrowserTaskRunNode.d.ts.map +1 -1
- package/dist/nodes/addBrowserTaskRunNode.js +11 -35
- package/dist/nodes/addBrowserTaskRunNode.js.map +1 -1
- package/dist/platform/mindedChatOpenAI.d.ts +5 -0
- package/dist/platform/mindedChatOpenAI.d.ts.map +1 -0
- package/dist/platform/mindedChatOpenAI.js +23 -0
- package/dist/platform/mindedChatOpenAI.js.map +1 -0
- package/dist/platform/mindedConnection.d.ts +1 -0
- package/dist/platform/mindedConnection.d.ts.map +1 -1
- package/dist/platform/mindedConnection.js +6 -1
- package/dist/platform/mindedConnection.js.map +1 -1
- package/dist/platform/mindedConnectionTypes.d.ts +14 -1
- package/dist/platform/mindedConnectionTypes.d.ts.map +1 -1
- package/dist/platform/mindedConnectionTypes.js +4 -0
- package/dist/platform/mindedConnectionTypes.js.map +1 -1
- package/dist/utils/extractStateMemoryResponse.d.ts +5 -0
- package/dist/utils/extractStateMemoryResponse.d.ts.map +1 -0
- package/dist/utils/extractStateMemoryResponse.js +91 -0
- package/dist/utils/extractStateMemoryResponse.js.map +1 -0
- package/dist/utils/extractToolMemoryResponse.d.ts +4 -0
- package/dist/utils/extractToolMemoryResponse.d.ts.map +1 -0
- package/dist/utils/extractToolMemoryResponse.js +16 -0
- package/dist/utils/extractToolMemoryResponse.js.map +1 -0
- package/package.json +2 -2
- package/src/agent.ts +94 -28
- package/src/browserTask/executeBrowserTask.py +19 -52
- package/src/browserTask/executeBrowserTask.ts +16 -9
- package/src/browserTask/localBrowserTask.ts +51 -7
- package/src/browserTask/types.ts +4 -6
- package/src/nodes/addBrowserTaskRunNode.ts +13 -41
- package/src/platform/mindedConnection.ts +5 -0
- package/src/platform/mindedConnectionTypes.ts +18 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractStateMemoryResponse.js","sourceRoot":"","sources":["../../src/utils/extractStateMemoryResponse.ts"],"names":[],"mappings":";;AAAA,uDAA4G;AAC5G,qCAAkC;AAGlC,MAAM,wBAAwB,GAAG,CAAC,WAAwB,EAAkB,EAAE;IAC5E,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,OAAiB,CAAC,CAAC;QACzD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAC1B,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,gCAAgC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAClF,CAAC;YACD,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,mCAAmC,EAAE,GAAG,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,gCAAgC,GAAG,CAAC,QAAe,EAAiB,EAAE;IAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,kCAAkC,EAAE,QAAQ,EAAE,CAAC,CAAC;QACpE,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;QAClB,IAAI,CAAC;YACH,8DAA8D;YAC9D,IAAI,UAAU,YAAY,sBAAW,EAAE,CAAC;gBACtC,OAAO,UAAU,CAAC;YACpB,CAAC;YAED,wEAAwE;YACxE,MAAM,WAAW,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YAChD,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,MAAM,gBAAgB,GAAG,UAAU,CAAC,iBAAiB,IAAI,EAAE,CAAC;YAE5D,QAAQ,WAAW,EAAE,CAAC;gBACpB,KAAK,WAAW;oBACd,OAAO,IAAI,oBAAS,CAAC;wBACnB,OAAO;wBACP,EAAE;wBACF,iBAAiB,EAAE,gBAAgB;wBACnC,UAAU,EAAE,UAAU,CAAC,UAAU,IAAI,EAAE;wBACvC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB,IAAI,EAAE;qBACxD,CAAC,CAAC;gBACL,KAAK,cAAc;oBACjB,OAAO,IAAI,uBAAY,CAAC;wBACtB,OAAO;wBACP,EAAE;wBACF,iBAAiB,EAAE,gBAAgB;qBACpC,CAAC,CAAC;gBACL,KAAK,eAAe;oBAClB,OAAO,IAAI,wBAAa,CAAC;wBACvB,OAAO;wBACP,EAAE;wBACF,iBAAiB,EAAE,gBAAgB;qBACpC,CAAC,CAAC;gBACL,KAAK,aAAa;oBAChB,OAAO,IAAI,sBAAW,CAAC;wBACrB,OAAO;wBACP,EAAE;wBACF,iBAAiB,EAAE,gBAAgB;wBACnC,YAAY,EAAE,UAAU,CAAC,YAAY,IAAI,EAAE;qBAC5C,CAAC,CAAC;gBAEL;oBACE,4CAA4C;oBAC5C,eAAM,CAAC,IAAI,CAAC;wBACV,GAAG,EAAE,kDAAkD;wBACvD,WAAW;wBACX,UAAU;qBACX,CAAC,CAAC;oBACH,OAAO,IAAI,CAAC;YAChB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,eAAM,CAAC,KAAK,CAAC;gBACX,GAAG,EAAE,8BAA8B;gBACnC,GAAG;gBACH,UAAU;aACX,CAAC,CAAC;YACH,kDAAkD;YAClD,OAAO,IAAI,uBAAY,CAAC;gBACtB,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,8BAA8B;gBAC7D,EAAE,EAAE,UAAU,CAAC,EAAE;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAkB,CAAC;AACtC,CAAC,CAAC;AAEF,kBAAe,wBAAwB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractToolMemoryResponse.d.ts","sourceRoot":"","sources":["../../src/utils/extractToolMemoryResponse.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,QAAA,MAAM,yBAAyB,GAAI,MAAM,EAAE,aAAa,WAAW,KAAG,OAAO,CAAC,MAAM,CAUnF,CAAC;AAEF,eAAe,yBAAyB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const extractToolMemoryResponse = (toolMessage) => {
|
|
4
|
+
try {
|
|
5
|
+
const parsed = JSON.parse(toolMessage.content);
|
|
6
|
+
if (typeof parsed === 'object' && parsed !== null && 'memory' in parsed) {
|
|
7
|
+
return parsed.memory;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
catch (error) {
|
|
11
|
+
console.error('Error parsing tool memory response', error);
|
|
12
|
+
}
|
|
13
|
+
return {};
|
|
14
|
+
};
|
|
15
|
+
exports.default = extractToolMemoryResponse;
|
|
16
|
+
//# sourceMappingURL=extractToolMemoryResponse.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractToolMemoryResponse.js","sourceRoot":"","sources":["../../src/utils/extractToolMemoryResponse.ts"],"names":[],"mappings":";;AAEA,MAAM,yBAAyB,GAAG,CAAS,WAAwB,EAAmB,EAAE;IACtF,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,OAAiB,CAAC,CAAC;QACzD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;YACxE,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEF,kBAAe,yBAAyB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@minded-ai/mindedjs",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.26-beta.10",
|
|
4
4
|
"description": "MindedJS is a TypeScript library for building agents.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -72,4 +72,4 @@
|
|
|
72
72
|
"peerDependencies": {
|
|
73
73
|
"playwright": "^1.55.0"
|
|
74
74
|
}
|
|
75
|
-
}
|
|
75
|
+
}
|
package/src/agent.ts
CHANGED
|
@@ -88,6 +88,7 @@ export class Agent {
|
|
|
88
88
|
private stateAnnotation: ReturnType<typeof createStateAnnotation>;
|
|
89
89
|
public flows!: Flow[];
|
|
90
90
|
public tools!: Tool<any, any>[];
|
|
91
|
+
private createAgentParams: CreateAgentParams<z.infer<typeof this.memorySchema>>;
|
|
91
92
|
/**
|
|
92
93
|
* Internal tools
|
|
93
94
|
*/
|
|
@@ -130,6 +131,7 @@ export class Agent {
|
|
|
130
131
|
private initPromise: Promise<void> | null = null;
|
|
131
132
|
private startingNodeId: string | null = null;
|
|
132
133
|
public voiceSessions: Map<string, VoiceSession> = new Map();
|
|
134
|
+
private sigtermHandlerRegistered = false;
|
|
133
135
|
|
|
134
136
|
// Session storage
|
|
135
137
|
public parseSessionIdFromTrigger?: (trigger: any, triggerName: string) => string;
|
|
@@ -166,10 +168,10 @@ export class Agent {
|
|
|
166
168
|
* ```
|
|
167
169
|
*/
|
|
168
170
|
constructor(params: CreateAgentParams<z.infer<typeof this.memorySchema>>) {
|
|
169
|
-
|
|
170
|
-
this.
|
|
171
|
-
this.stateAnnotation = createStateAnnotation(memorySchema);
|
|
172
|
-
this.initPromise = this.init(
|
|
171
|
+
this.memorySchema = params.memorySchema;
|
|
172
|
+
this.createAgentParams = params;
|
|
173
|
+
this.stateAnnotation = createStateAnnotation(this.memorySchema);
|
|
174
|
+
this.initPromise = this.init();
|
|
173
175
|
this.config = params.config;
|
|
174
176
|
this.toolExecutor = new ToolExecutor(this);
|
|
175
177
|
this.parseSessionIdFromTrigger = params.parseSessionIdFromTrigger;
|
|
@@ -178,9 +180,8 @@ export class Agent {
|
|
|
178
180
|
this.registerDefaultAnalyticsHandler();
|
|
179
181
|
}
|
|
180
182
|
|
|
181
|
-
private async init(
|
|
183
|
+
private async init(): Promise<void> {
|
|
182
184
|
try {
|
|
183
|
-
const { config, tools, memorySaver, interruptSessionManager } = params;
|
|
184
185
|
const { runLocally } = getConfig();
|
|
185
186
|
if (!runLocally) {
|
|
186
187
|
await mindedConnection.start();
|
|
@@ -188,6 +189,9 @@ export class Agent {
|
|
|
188
189
|
// Initialize PII gateway
|
|
189
190
|
this._piiGateway = new PIIGateway();
|
|
190
191
|
|
|
192
|
+
// Register SIGTERM handler
|
|
193
|
+
this.registerSigtermHandler();
|
|
194
|
+
|
|
191
195
|
mindedConnection.on(mindedConnectionSocketMessageType.INVOKE, async (message) => {
|
|
192
196
|
const invokeMessage = message as InvokeMessage;
|
|
193
197
|
try {
|
|
@@ -236,6 +240,16 @@ export class Agent {
|
|
|
236
240
|
this.updateState({ sessionId, state });
|
|
237
241
|
});
|
|
238
242
|
|
|
243
|
+
// Handle configuration updates
|
|
244
|
+
mindedConnection.on(mindedConnectionSocketMessageType.AGENT_CONFIGURATION_UPDATED, async () => {
|
|
245
|
+
logger.info({ msg: '[Agent] Received configuration update notification' });
|
|
246
|
+
try {
|
|
247
|
+
await this.loadAgentResources();
|
|
248
|
+
} catch (error) {
|
|
249
|
+
logger.error({ msg: '[Agent] Error reloading agent resources', error });
|
|
250
|
+
}
|
|
251
|
+
});
|
|
252
|
+
|
|
239
253
|
// Handle stop browser session requests
|
|
240
254
|
mindedConnection.on(mindedConnectionSocketMessageType.STOP_BROWSER_SESSION, async (message) => {
|
|
241
255
|
const { sessionId } = message as StopBrowserSessionRequest;
|
|
@@ -279,38 +293,44 @@ export class Agent {
|
|
|
279
293
|
});
|
|
280
294
|
}
|
|
281
295
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
this.playbooks = playbooks;
|
|
285
|
-
this.flows = flows;
|
|
286
|
-
this.validate();
|
|
287
|
-
const appActionsRunnerTools = this.initAppActionsRunnerTools();
|
|
288
|
-
const libraryActionsRunnerTools = initLibraryActionsRunnerTools(flows);
|
|
289
|
-
this.tools = [...tools, ...appActionsRunnerTools, ...libraryActionsRunnerTools];
|
|
296
|
+
this.interruptSessionManager = this.createAgentParams.interruptSessionManager || createInterruptSessionManager();
|
|
297
|
+
this.checkpointer = this.createAgentParams.memorySaver || createCheckpointSaver();
|
|
290
298
|
|
|
291
|
-
|
|
292
|
-
this.toolExecutor.registerTools(this.tools);
|
|
299
|
+
await this.loadAgentResources();
|
|
293
300
|
|
|
294
|
-
this.checkpointer = memorySaver || createCheckpointSaver();
|
|
295
|
-
this.interruptSessionManager = interruptSessionManager || createInterruptSessionManager();
|
|
296
|
-
|
|
297
|
-
// call here methods that needs environment variables to be loaded
|
|
298
|
-
this.llm = createLlmInstance(config.llm);
|
|
299
|
-
this.compiledGraph = this.initializeGraph();
|
|
300
301
|
this.initialized = true;
|
|
301
302
|
|
|
302
|
-
const flowHasVoiceTrigger = this.flows.some((flow) =>
|
|
303
|
-
flow.nodes.some((node) => node.type === NodeType.TRIGGER && node.triggerType === TriggerType.VOICE),
|
|
304
|
-
);
|
|
305
|
-
if (flowHasVoiceTrigger) {
|
|
306
|
-
this.setupVoice();
|
|
307
|
-
}
|
|
308
303
|
} catch (err) {
|
|
309
304
|
this.initialized = false;
|
|
310
305
|
throw err;
|
|
311
306
|
}
|
|
312
307
|
}
|
|
313
308
|
|
|
309
|
+
private async loadAgentResources(): Promise<void> {
|
|
310
|
+
const [, flows, playbooks] = await Promise.all([this.loadSecrets(), loadFlows(this.createAgentParams.config.flows), loadPlaybooks(this.createAgentParams.config.playbooks)]);
|
|
311
|
+
|
|
312
|
+
this.playbooks = playbooks;
|
|
313
|
+
this.flows = flows;
|
|
314
|
+
this.validate();
|
|
315
|
+
const appActionsRunnerTools = this.initAppActionsRunnerTools();
|
|
316
|
+
const libraryActionsRunnerTools = initLibraryActionsRunnerTools(flows);
|
|
317
|
+
this.tools = [...this.createAgentParams.tools, ...appActionsRunnerTools, ...libraryActionsRunnerTools];
|
|
318
|
+
|
|
319
|
+
// Register tools with the tool executor
|
|
320
|
+
this.toolExecutor.registerTools(this.tools);
|
|
321
|
+
|
|
322
|
+
// call here methods that needs environment variables to be loaded
|
|
323
|
+
this.llm = createLlmInstance(this.createAgentParams.config.llm);
|
|
324
|
+
this.compiledGraph = this.initializeGraph();
|
|
325
|
+
|
|
326
|
+
const flowHasVoiceTrigger = this.flows.some((flow) =>
|
|
327
|
+
flow.nodes.some((node) => node.type === NodeType.TRIGGER && node.triggerType === TriggerType.VOICE),
|
|
328
|
+
);
|
|
329
|
+
if (flowHasVoiceTrigger) {
|
|
330
|
+
this.setupVoice();
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
314
334
|
private validate() {
|
|
315
335
|
if (this.flows.length === 0) {
|
|
316
336
|
throw new Error('No flows provided');
|
|
@@ -1093,4 +1113,50 @@ export class Agent {
|
|
|
1093
1113
|
await this.interruptSessionManager.release(sessionId);
|
|
1094
1114
|
return undefined;
|
|
1095
1115
|
}
|
|
1116
|
+
|
|
1117
|
+
/**
|
|
1118
|
+
* Register SIGTERM handler to notify backend before shutdown
|
|
1119
|
+
*/
|
|
1120
|
+
private registerSigtermHandler(): void {
|
|
1121
|
+
if (this.sigtermHandlerRegistered) {
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
const handleSigterm = async () => {
|
|
1126
|
+
logger.info({ msg: '[Agent] Received SIGTERM, notifying backend' });
|
|
1127
|
+
try {
|
|
1128
|
+
// Send notification to backend
|
|
1129
|
+
await mindedConnection.emit(mindedConnectionSocketMessageType.SDK_SIGTERM, {
|
|
1130
|
+
type: mindedConnectionSocketMessageType.SDK_SIGTERM,
|
|
1131
|
+
});
|
|
1132
|
+
logger.info({ msg: '[Agent] SIGTERM notification sent to backend' });
|
|
1133
|
+
|
|
1134
|
+
// Set a hard deadline (e.g., 14 minutes) to ensure we exit before force kill
|
|
1135
|
+
setTimeout(() => {
|
|
1136
|
+
logger.warn({ msg: '[Agent] Grace period expired, forcing exit' });
|
|
1137
|
+
process.exit(0);
|
|
1138
|
+
}, 840000); // 14 minutes
|
|
1139
|
+
|
|
1140
|
+
// Sample messages in processing every 10 seconds and exit if no messages are in processing
|
|
1141
|
+
setInterval(() => {
|
|
1142
|
+
logger.info({ msg: '[Agent] Messages in processing', incomingMessagesInProcessing: mindedConnection.incomingMessagesInProcessing.size });
|
|
1143
|
+
if (mindedConnection.incomingMessagesInProcessing.size === 0) {
|
|
1144
|
+
logger.info({ msg: '[Agent] No messages in processing, exiting' });
|
|
1145
|
+
process.exit(0);
|
|
1146
|
+
}
|
|
1147
|
+
}, 10000);
|
|
1148
|
+
|
|
1149
|
+
} catch (err) {
|
|
1150
|
+
logger.error({ msg: '[Agent] Failed to send SIGTERM notification', err });
|
|
1151
|
+
}
|
|
1152
|
+
// Allow some time for the message to be sent before exiting
|
|
1153
|
+
setTimeout(() => {
|
|
1154
|
+
process.exit(0);
|
|
1155
|
+
}, 100);
|
|
1156
|
+
};
|
|
1157
|
+
|
|
1158
|
+
process.on('SIGTERM', handleSigterm);
|
|
1159
|
+
this.sigtermHandlerRegistered = true;
|
|
1160
|
+
logger.debug({ msg: '[Agent] SIGTERM handler registered' });
|
|
1161
|
+
}
|
|
1096
1162
|
}
|
|
@@ -5,6 +5,8 @@ This script runs browser automation tasks using browser-use and can capture
|
|
|
5
5
|
screenshots at the end of each step, uploading them to S3 when configured.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
+
import boto3
|
|
9
|
+
from botocore.exceptions import ClientError
|
|
8
10
|
import asyncio
|
|
9
11
|
import json
|
|
10
12
|
from typing import List, Optional, Dict, Any, TypedDict
|
|
@@ -42,24 +44,14 @@ class ScreenshotCapture:
|
|
|
42
44
|
self.config = config or {}
|
|
43
45
|
|
|
44
46
|
# Import boto3 only when screenshot capture is needed
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
from botocore.exceptions import ClientError
|
|
48
|
-
self.boto3 = boto3
|
|
49
|
-
self.ClientError = ClientError
|
|
50
|
-
except ImportError:
|
|
51
|
-
raise ImportError(
|
|
52
|
-
"boto3 is required for screenshot capture. "
|
|
53
|
-
"Please install it with: uv pip install boto3 "
|
|
54
|
-
"or run: npx minded setup-local-operator"
|
|
55
|
-
)
|
|
56
|
-
|
|
47
|
+
self.ClientError = ClientError
|
|
48
|
+
|
|
57
49
|
# S3 configuration with defaults
|
|
58
|
-
self.s3_bucket = self.config.get('s3_bucket'
|
|
59
|
-
self.s3_prefix = self.config.get('s3_prefix'
|
|
50
|
+
self.s3_bucket = self.config.get('s3_bucket')
|
|
51
|
+
self.s3_prefix = self.config.get('s3_prefix')
|
|
60
52
|
|
|
61
53
|
# AWS region configuration
|
|
62
|
-
self.aws_region = self.config.get('aws_region'
|
|
54
|
+
self.aws_region = self.config.get('aws_region')
|
|
63
55
|
|
|
64
56
|
# Ensure prefix ends with slash
|
|
65
57
|
if not self.s3_prefix.endswith('/'):
|
|
@@ -69,7 +61,7 @@ class ScreenshotCapture:
|
|
|
69
61
|
self.session_id = self.config.get('session_id', datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3])
|
|
70
62
|
|
|
71
63
|
# Initialize S3 client with region
|
|
72
|
-
self.s3_client =
|
|
64
|
+
self.s3_client = boto3.client('s3', region_name=self.aws_region)
|
|
73
65
|
|
|
74
66
|
# Track step counter
|
|
75
67
|
self.step_counter = 0
|
|
@@ -85,19 +77,14 @@ class ScreenshotCapture:
|
|
|
85
77
|
# Get current page
|
|
86
78
|
page = await agent.browser_session.get_current_page()
|
|
87
79
|
|
|
88
|
-
# Get current URL for logging (browser-use might use method instead of property)
|
|
89
|
-
try:
|
|
90
|
-
current_url = page.url if hasattr(page, 'url') else page.url()
|
|
91
|
-
except:
|
|
92
|
-
current_url = "unknown"
|
|
93
|
-
|
|
94
80
|
# Update step counter
|
|
95
81
|
step_number = self.step_counter
|
|
96
82
|
self.step_counter += 1
|
|
97
83
|
|
|
98
|
-
logger.info(f"📸 Capturing screenshot #{step_number}
|
|
84
|
+
logger.info(f"📸 Capturing screenshot #{step_number}")
|
|
99
85
|
|
|
100
86
|
# Take screenshot - browser-use takes viewport by default
|
|
87
|
+
# Set timeout to prevent hanging (30 seconds)
|
|
101
88
|
screenshot_data = await page.screenshot()
|
|
102
89
|
|
|
103
90
|
# Ensure we have bytes - browser-use might return base64 string or bytes
|
|
@@ -155,24 +142,14 @@ class LogsCapture:
|
|
|
155
142
|
self.config = config or {}
|
|
156
143
|
|
|
157
144
|
# Import boto3 only when logs capture is needed
|
|
158
|
-
|
|
159
|
-
import boto3
|
|
160
|
-
from botocore.exceptions import ClientError
|
|
161
|
-
self.boto3 = boto3
|
|
162
|
-
self.ClientError = ClientError
|
|
163
|
-
except ImportError:
|
|
164
|
-
raise ImportError(
|
|
165
|
-
"boto3 is required for logs capture. "
|
|
166
|
-
"Please install it with: uv pip install boto3 "
|
|
167
|
-
"or run: npx minded setup-local-operator"
|
|
168
|
-
)
|
|
145
|
+
self.ClientError = ClientError
|
|
169
146
|
|
|
170
|
-
# S3 configuration
|
|
171
|
-
self.s3_bucket = self.config.get('s3_bucket'
|
|
172
|
-
self.s3_prefix = self.config.get('s3_prefix'
|
|
147
|
+
# S3 configuration
|
|
148
|
+
self.s3_bucket = self.config.get('s3_bucket')
|
|
149
|
+
self.s3_prefix = self.config.get('s3_prefix')
|
|
173
150
|
|
|
174
151
|
# AWS region configuration
|
|
175
|
-
self.aws_region = self.config.get('aws_region'
|
|
152
|
+
self.aws_region = self.config.get('aws_region')
|
|
176
153
|
|
|
177
154
|
# Ensure prefix ends with slash
|
|
178
155
|
if not self.s3_prefix.endswith('/'):
|
|
@@ -183,7 +160,7 @@ class LogsCapture:
|
|
|
183
160
|
self.tool_call_id = self.config.get('tool_call_id', 'unknown')
|
|
184
161
|
|
|
185
162
|
# Initialize S3 client with region
|
|
186
|
-
self.s3_client =
|
|
163
|
+
self.s3_client = boto3.client('s3', region_name=self.aws_region)
|
|
187
164
|
|
|
188
165
|
# Track accumulated logs
|
|
189
166
|
self.log_entries: List[str] = []
|
|
@@ -208,20 +185,15 @@ class LogsCapture:
|
|
|
208
185
|
try:
|
|
209
186
|
model_outputs = history.model_outputs() if hasattr(history, 'model_outputs') else []
|
|
210
187
|
for model_output in model_outputs:
|
|
211
|
-
print(f"model_output")
|
|
212
188
|
if hasattr(model_output, 'current_state'):
|
|
213
|
-
print(f"model_output.current_state")
|
|
214
189
|
if model_output.current_state.thinking:
|
|
215
190
|
self.log_entries.append('💡 Thinking: ' + model_output.current_state.thinking)
|
|
216
191
|
if model_output.current_state.memory:
|
|
217
192
|
self.log_entries.append('🧠 Memory: ' + model_output.current_state.memory)
|
|
218
193
|
if model_output.current_state.next_goal:
|
|
219
194
|
self.log_entries.append('🎯 Next goal: ' + model_output.current_state.next_goal)
|
|
220
|
-
print("Finished adding log entries")
|
|
221
195
|
except Exception as e:
|
|
222
196
|
logger.warning(f"Could not extract model outputs: {e}")
|
|
223
|
-
|
|
224
|
-
print(f"self.log_entries count: {len(self.log_entries)}")
|
|
225
197
|
|
|
226
198
|
# Upload to S3
|
|
227
199
|
s3_key = f"{self.s3_prefix}{self.session_id}/{self.tool_call_id}/operator.log"
|
|
@@ -301,11 +273,8 @@ async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Opt
|
|
|
301
273
|
screenshot_capture = None
|
|
302
274
|
logs_capture = None
|
|
303
275
|
on_step_end_hook = None
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
enable_screenshot_capture_to_s3 = screenshot_config and screenshot_config.get('enableScreenshotCaptureToS3', False)
|
|
307
|
-
|
|
308
|
-
if enable_screenshot_capture_to_s3:
|
|
276
|
+
|
|
277
|
+
if screenshot_config:
|
|
309
278
|
logger.info("-" * 50)
|
|
310
279
|
logger.info("🎯 Initializing screenshot capture for browser task")
|
|
311
280
|
logger.info(f" Session: {session_id}")
|
|
@@ -318,9 +287,7 @@ async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Opt
|
|
|
318
287
|
logger.info("📷 Screenshot capture is DISABLED for this browser task")
|
|
319
288
|
|
|
320
289
|
# Initialize logs capture if enabled (independent from screenshots)
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
if enable_logs_capture_to_s3:
|
|
290
|
+
if screenshot_config:
|
|
324
291
|
logger.info("-" * 50)
|
|
325
292
|
logger.info("🎯 Initializing logs capture for browser task")
|
|
326
293
|
logger.info(f" Session: {session_id}")
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
import { kill, getOrStartLocalCDP, isLocalBrowserRunning } from './localBrowserTask';
|
|
16
16
|
import { BrowserTaskMode, InvokeBrowserTaskOptions } from './types';
|
|
17
17
|
import { isLocalOperatorSetup, validateLocalOperatorSetup } from '../cli/localOperatorSetup';
|
|
18
|
+
import { getConfig } from '../platform/config';
|
|
18
19
|
import CDP from 'chrome-remote-interface';
|
|
19
20
|
import { CookieStore } from './CookieStore';
|
|
20
21
|
|
|
@@ -206,9 +207,14 @@ export const destroyBrowserSession = async ({
|
|
|
206
207
|
export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Promise<InvokeBrowserTaskResponse> => {
|
|
207
208
|
try {
|
|
208
209
|
const { sessionId, cdpUrl, task, keepAlive, hooks, browserTaskMode, toolSchemas, outputSchema, screenshotConfig, toolCallId } = options;
|
|
210
|
+
const { isDeployed } = getConfig();
|
|
209
211
|
|
|
210
212
|
if (browserTaskMode === BrowserTaskMode.LOCAL) {
|
|
211
|
-
|
|
213
|
+
|
|
214
|
+
if (!isDeployed) {
|
|
215
|
+
validateLocalOperatorSetup();
|
|
216
|
+
}
|
|
217
|
+
|
|
212
218
|
const pythonScriptPath = path.resolve(__dirname, 'executeBrowserTask.py');
|
|
213
219
|
|
|
214
220
|
// Calculate folder path for downloads
|
|
@@ -218,18 +224,16 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
|
|
|
218
224
|
logger.info({
|
|
219
225
|
message: 'Spawning Python process',
|
|
220
226
|
args,
|
|
221
|
-
enableScreenshotCaptureToS3: screenshotConfig?.enableScreenshotCaptureToS3 || false,
|
|
222
|
-
enableLogsCaptureToS3: screenshotConfig?.enableLogsCaptureToS3 || false,
|
|
223
227
|
});
|
|
224
228
|
|
|
225
229
|
const child = spawn('uv', args, {
|
|
226
230
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
227
231
|
env: isLocalOperatorSetup()
|
|
228
232
|
? {
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
+
...process.env,
|
|
234
|
+
VIRTUAL_ENV: path.join(process.cwd(), '.venv'),
|
|
235
|
+
PATH: `${path.join(process.cwd(), '.venv', 'bin')}:${process.env.PATH}`,
|
|
236
|
+
}
|
|
233
237
|
: process.env,
|
|
234
238
|
});
|
|
235
239
|
|
|
@@ -245,6 +249,11 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
|
|
|
245
249
|
toolCallId,
|
|
246
250
|
};
|
|
247
251
|
|
|
252
|
+
logger.info({
|
|
253
|
+
msg: 'Browser task payload',
|
|
254
|
+
payload,
|
|
255
|
+
});
|
|
256
|
+
|
|
248
257
|
// Write JSON payload to stdin
|
|
249
258
|
child.stdin.write(JSON.stringify(payload));
|
|
250
259
|
child.stdin.end();
|
|
@@ -341,8 +350,6 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
|
|
|
341
350
|
hooksCount: hooks?.length || 0,
|
|
342
351
|
onPrem: browserTaskMode === BrowserTaskMode.ON_PREM,
|
|
343
352
|
outputSchemaFields: outputSchema?.length || 0,
|
|
344
|
-
enableScreenshotCaptureToS3: screenshotConfig?.enableScreenshotCaptureToS3 || false,
|
|
345
|
-
enableLogsCaptureToS3: screenshotConfig?.enableLogsCaptureToS3 || false,
|
|
346
353
|
});
|
|
347
354
|
|
|
348
355
|
const response = await mindedConnection.awaitEmit<InvokeBrowserTaskRequest, InvokeBrowserTaskResponse>(
|
|
@@ -6,6 +6,7 @@ import { chromium } from 'playwright';
|
|
|
6
6
|
import { logger } from '../utils/logger';
|
|
7
7
|
import { wait } from '../utils/wait';
|
|
8
8
|
import * as net from 'net';
|
|
9
|
+
import { getConfig } from '../platform/config';
|
|
9
10
|
|
|
10
11
|
interface BrowserInstance {
|
|
11
12
|
id: string;
|
|
@@ -86,6 +87,7 @@ export type StartChromiumOptions = {
|
|
|
86
87
|
* Works with Chrome/Chromium/Chrome for Testing.
|
|
87
88
|
*/
|
|
88
89
|
export async function getOrStartLocalCDP(opts: StartChromiumOptions): Promise<{ cdpUrl: string; instanceId: string }> {
|
|
90
|
+
const { isDeployed } = getConfig();
|
|
89
91
|
// If instanceId is provided, try to reuse existing instance
|
|
90
92
|
if (opts.instanceId && browserInstances.has(opts.instanceId)) {
|
|
91
93
|
const instance = browserInstances.get(opts.instanceId)!;
|
|
@@ -100,7 +102,9 @@ export async function getOrStartLocalCDP(opts: StartChromiumOptions): Promise<{
|
|
|
100
102
|
|
|
101
103
|
const { env } = opts;
|
|
102
104
|
|
|
103
|
-
const executablePath = chromium.executablePath();
|
|
105
|
+
const executablePath = process.env.CHROMIUM_PATH || chromium.executablePath();
|
|
106
|
+
|
|
107
|
+
logger.info({ message: 'Chromium executable path', executablePath });
|
|
104
108
|
|
|
105
109
|
// Find an available port
|
|
106
110
|
const port = opts.preferredPort
|
|
@@ -119,7 +123,11 @@ export async function getOrStartLocalCDP(opts: StartChromiumOptions): Promise<{
|
|
|
119
123
|
const args = [
|
|
120
124
|
'--new-window',
|
|
121
125
|
`--user-data-dir=${userDataDir}`,
|
|
122
|
-
...(
|
|
126
|
+
...(isDeployed ? [
|
|
127
|
+
'--headless',
|
|
128
|
+
'--no-sandbox', // Required for running in Docker/containerized environments
|
|
129
|
+
'--disable-setuid-sandbox', // Additional sandbox bypass for containers
|
|
130
|
+
] : []),
|
|
123
131
|
'--disable-component-extensions-with-background-pages',
|
|
124
132
|
'--disable-background-networking',
|
|
125
133
|
'--disable-back-forward-cache',
|
|
@@ -192,13 +200,49 @@ export async function getOrStartLocalCDP(opts: StartChromiumOptions): Promise<{
|
|
|
192
200
|
env: { ...process.env, ...env },
|
|
193
201
|
});
|
|
194
202
|
|
|
195
|
-
|
|
196
|
-
|
|
203
|
+
// Capture stderr output to help debug crashes
|
|
204
|
+
let stderrBuffer = '';
|
|
205
|
+
proc.stderr?.on('data', (data) => {
|
|
206
|
+
const output = data.toString();
|
|
207
|
+
stderrBuffer += output;
|
|
208
|
+
|
|
209
|
+
// Skip common non-critical errors in containerized environments
|
|
210
|
+
const ignoredPatterns = [
|
|
211
|
+
'Failed to connect to the bus',
|
|
212
|
+
'Failed to call method: org.freedesktop.DBus',
|
|
213
|
+
'DEPRECATED_ENDPOINT',
|
|
214
|
+
'Authentication Failed: wrong_secret', // Google GCM
|
|
215
|
+
];
|
|
216
|
+
|
|
217
|
+
const shouldIgnore = ignoredPatterns.some(pattern => output.includes(pattern));
|
|
218
|
+
if (shouldIgnore) {
|
|
219
|
+
return; // Silently ignore these expected container warnings
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// Log important error messages immediately
|
|
223
|
+
if (output.includes('ERROR') || output.includes('FATAL') || output.includes('Segmentation fault')) {
|
|
224
|
+
logger.error({ message: 'Chromium stderr', output: output.trim() });
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
proc.on("error", (error) => {
|
|
229
|
+
logger.error({ message: 'Chromium spawn error', error });
|
|
197
230
|
});
|
|
198
231
|
|
|
199
|
-
proc.on(
|
|
200
|
-
|
|
201
|
-
|
|
232
|
+
proc.on("exit", (code, signal) => {
|
|
233
|
+
const exitInfo: any = {
|
|
234
|
+
message: 'Chromium exited',
|
|
235
|
+
code,
|
|
236
|
+
signal,
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
// Include recent stderr output in exit log
|
|
240
|
+
if (stderrBuffer) {
|
|
241
|
+
const recentStderr = stderrBuffer.slice(-500); // Last 500 chars
|
|
242
|
+
exitInfo.recentStderr = recentStderr;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
logger.error(exitInfo);
|
|
202
246
|
browserInstances.delete(instanceId);
|
|
203
247
|
});
|
|
204
248
|
|
package/src/browserTask/types.ts
CHANGED
|
@@ -26,11 +26,9 @@ export type ToolSchema = {
|
|
|
26
26
|
};
|
|
27
27
|
|
|
28
28
|
export type ScreenshotConfig = {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
s3_prefix?: string;
|
|
33
|
-
aws_region?: string;
|
|
29
|
+
s3_bucket: string;
|
|
30
|
+
s3_prefix: string;
|
|
31
|
+
aws_region: string;
|
|
34
32
|
};
|
|
35
33
|
|
|
36
34
|
export interface InvokeBrowserTaskOptions {
|
|
@@ -42,6 +40,6 @@ export interface InvokeBrowserTaskOptions {
|
|
|
42
40
|
browserTaskMode: BrowserTaskMode;
|
|
43
41
|
toolSchemas?: ToolSchema[];
|
|
44
42
|
outputSchema?: OutputSchemaField[];
|
|
45
|
-
screenshotConfig
|
|
43
|
+
screenshotConfig: ScreenshotConfig | null;
|
|
46
44
|
toolCallId: string;
|
|
47
45
|
}
|
|
@@ -55,47 +55,19 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
|
|
|
55
55
|
// We compile the env variables to avoid having to pass them to the platform in the tool input
|
|
56
56
|
const promptCompiledWithEnv = compilePrompt(prompt, { env: process.env });
|
|
57
57
|
|
|
58
|
-
const { browserTaskMode } = getConfig();
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
// Build config with separate flags
|
|
72
|
-
const config: ScreenshotConfig = {
|
|
73
|
-
enableScreenshotCaptureToS3,
|
|
74
|
-
enableLogsCaptureToS3,
|
|
75
|
-
};
|
|
76
|
-
|
|
77
|
-
// Add optional S3 configuration if provided
|
|
78
|
-
if (process.env.SCREENSHOT_S3_BUCKET) {
|
|
79
|
-
config.s3_bucket = process.env.SCREENSHOT_S3_BUCKET;
|
|
80
|
-
}
|
|
81
|
-
if (process.env.SCREENSHOT_S3_PREFIX) {
|
|
82
|
-
config.s3_prefix = process.env.SCREENSHOT_S3_PREFIX;
|
|
83
|
-
}
|
|
84
|
-
if (process.env.AWS_REGION) {
|
|
85
|
-
config.aws_region = process.env.AWS_REGION;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
logger.debug({
|
|
89
|
-
msg: 'Browser task capture configuration',
|
|
90
|
-
screenshots: enableScreenshotCaptureToS3,
|
|
91
|
-
logs: enableLogsCaptureToS3,
|
|
92
|
-
bucket: config.s3_bucket || 'default',
|
|
93
|
-
prefix: config.s3_prefix || 'default',
|
|
94
|
-
region: config.aws_region || 'default',
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
return config;
|
|
98
|
-
})();
|
|
58
|
+
const { browserTaskMode, isDeployed } = getConfig();
|
|
59
|
+
|
|
60
|
+
const screenshotConfig: ScreenshotConfig | null = !isDeployed ? null : {
|
|
61
|
+
aws_region: process.env.AWS_REGION || 'us-east-1',
|
|
62
|
+
s3_bucket: process.env.AWS_BUCKET || 'minded-agents',
|
|
63
|
+
s3_prefix: 'browser-task/',
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
logger.info({
|
|
67
|
+
msg: 'Screenshot config',
|
|
68
|
+
screenshotConfig,
|
|
69
|
+
isDeployed,
|
|
70
|
+
})
|
|
99
71
|
|
|
100
72
|
// Invoke browser task via socket
|
|
101
73
|
const result = await invokeBrowserTask({
|
|
@@ -5,6 +5,7 @@ import { getConfig } from './config';
|
|
|
5
5
|
import { logger } from '../utils/logger';
|
|
6
6
|
import { wait } from '../utils/wait';
|
|
7
7
|
import * as packageJson from '../../package.json';
|
|
8
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
8
9
|
|
|
9
10
|
// Helper function to conditionally add colors to messages
|
|
10
11
|
const colorize = (message: string, colorCode: string): string => {
|
|
@@ -18,6 +19,7 @@ let isReconnecting = false; // Flag to prevent multiple simultaneous reconnectio
|
|
|
18
19
|
const listeners: {
|
|
19
20
|
[key: string]: ((message: any) => any)[];
|
|
20
21
|
} = {};
|
|
22
|
+
export const incomingMessagesInProcessing = new Set<string>();
|
|
21
23
|
|
|
22
24
|
export const isConnected = (): boolean => {
|
|
23
25
|
return socket?.connected ?? false;
|
|
@@ -276,7 +278,10 @@ const connect = async (token: string, retryCount: number = 0): Promise<void> =>
|
|
|
276
278
|
socket.onAny((event, message, callback) => {
|
|
277
279
|
if (listeners[event]) {
|
|
278
280
|
listeners[event].forEach(async (listener) => {
|
|
281
|
+
const key = uuidv4();
|
|
282
|
+
incomingMessagesInProcessing.add(key);
|
|
279
283
|
const res = await listener(message);
|
|
284
|
+
incomingMessagesInProcessing.delete(key);
|
|
280
285
|
if (callback) {
|
|
281
286
|
callback(res);
|
|
282
287
|
}
|