@minded-ai/mindedjs 1.0.0-ec2-beta-1 → 1.0.0-ec2-beta-2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browserTask/executeBrowserTask.d.ts.map +1 -1
- package/dist/browserTask/executeBrowserTask.js +8 -2
- package/dist/browserTask/executeBrowserTask.js.map +1 -1
- package/dist/browserTask/executeBrowserTask.py +124 -3
- package/dist/browserTask/types.d.ts +7 -0
- package/dist/browserTask/types.d.ts.map +1 -1
- package/dist/nodes/addBrowserTaskRunNode.d.ts.map +1 -1
- package/dist/nodes/addBrowserTaskRunNode.js +30 -0
- package/dist/nodes/addBrowserTaskRunNode.js.map +1 -1
- package/package.json +1 -1
- package/src/browserTask/executeBrowserTask.py +124 -3
- package/src/browserTask/executeBrowserTask.ts +8 -2
- package/src/browserTask/types.ts +9 -1
- package/src/nodes/addBrowserTaskRunNode.ts +38 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executeBrowserTask.d.ts","sourceRoot":"","sources":["../../src/browserTask/executeBrowserTask.ts"],"names":[],"mappings":"AAIA,OAAO,EAEL,4BAA4B,EAC5B,yBAAyB,EAIzB,6BAA6B,EAC9B,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAC;AAOpE,eAAO,MAAM,oBAAoB,GAAU,wCAIxC;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,eAAe,CAAC;CAClC,KAAG,OAAO,CAAC,4BAA4B,CAqCvC,CAAC;AAEF,eAAO,MAAM,qBAAqB,GAAU,kCAIzC;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,KAAG,OAAO,CAAC,6BAA6B,CAoCxC,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAU,SAAS,wBAAwB,KAAG,OAAO,CAAC,yBAAyB,
|
|
1
|
+
{"version":3,"file":"executeBrowserTask.d.ts","sourceRoot":"","sources":["../../src/browserTask/executeBrowserTask.ts"],"names":[],"mappings":"AAIA,OAAO,EAEL,4BAA4B,EAC5B,yBAAyB,EAIzB,6BAA6B,EAC9B,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAC;AAOpE,eAAO,MAAM,oBAAoB,GAAU,wCAIxC;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,eAAe,CAAC;CAClC,KAAG,OAAO,CAAC,4BAA4B,CAqCvC,CAAC;AAEF,eAAO,MAAM,qBAAqB,GAAU,kCAIzC;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,KAAG,OAAO,CAAC,6BAA6B,CAoCxC,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAU,SAAS,wBAAwB,KAAG,OAAO,CAAC,yBAAyB,CAgJ5G,CAAC"}
|
|
@@ -77,12 +77,16 @@ exports.destroyBrowserSession = destroyBrowserSession;
|
|
|
77
77
|
const invokeBrowserTask = async (options) => {
|
|
78
78
|
var _a, _b, _c;
|
|
79
79
|
try {
|
|
80
|
-
const { sessionId, cdpUrl, task, keepAlive, hooks, browserTaskMode, toolSchemas, outputSchema } = options;
|
|
80
|
+
const { sessionId, cdpUrl, task, keepAlive, hooks, browserTaskMode, toolSchemas, outputSchema, screenshotConfig } = options;
|
|
81
81
|
if (browserTaskMode === types_1.BrowserTaskMode.LOCAL) {
|
|
82
82
|
(0, localOperatorSetup_1.validateLocalOperatorSetup)();
|
|
83
83
|
const pythonScriptPath = path_1.default.resolve(__dirname, 'executeBrowserTask.py');
|
|
84
84
|
const args = ['run', pythonScriptPath];
|
|
85
|
-
logger_1.logger.info({
|
|
85
|
+
logger_1.logger.info({
|
|
86
|
+
message: 'Spawning Python process',
|
|
87
|
+
args,
|
|
88
|
+
screenshotEnabled: (screenshotConfig === null || screenshotConfig === void 0 ? void 0 : screenshotConfig.enabled) || false,
|
|
89
|
+
});
|
|
86
90
|
const child = (0, node_child_process_1.spawn)('uv', args, {
|
|
87
91
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
88
92
|
env: (0, localOperatorSetup_1.isLocalOperatorSetup)()
|
|
@@ -100,6 +104,7 @@ const invokeBrowserTask = async (options) => {
|
|
|
100
104
|
task,
|
|
101
105
|
outputSchema,
|
|
102
106
|
otpSecret: process.env.OTP_SECRET,
|
|
107
|
+
screenshotConfig,
|
|
103
108
|
};
|
|
104
109
|
// Write JSON payload to stdin
|
|
105
110
|
child.stdin.write(JSON.stringify(payload));
|
|
@@ -161,6 +166,7 @@ const invokeBrowserTask = async (options) => {
|
|
|
161
166
|
hooksCount: (hooks === null || hooks === void 0 ? void 0 : hooks.length) || 0,
|
|
162
167
|
onPrem: browserTaskMode === types_1.BrowserTaskMode.ON_PREM,
|
|
163
168
|
outputSchemaFields: (outputSchema === null || outputSchema === void 0 ? void 0 : outputSchema.length) || 0,
|
|
169
|
+
screenshotEnabled: (screenshotConfig === null || screenshotConfig === void 0 ? void 0 : screenshotConfig.enabled) || false,
|
|
164
170
|
});
|
|
165
171
|
const response = await mindedConnection_1.mindedConnection.awaitEmit(mindedConnectionTypes_1.mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK, {
|
|
166
172
|
type: mindedConnectionTypes_1.mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executeBrowserTask.js","sourceRoot":"","sources":["../../src/browserTask/executeBrowserTask.ts"],"names":[],"mappings":";;;;;;AAAA,2DAA2C;AAC3C,4CAAyC;AACzC,mEAAgE;AAChE,gDAAwB;AACxB,6EAQ2C;AAC3C,yDAAqF;AACrF,mCAAoE;AACpE,kEAA6F;AAE7F,kEAAkE;AAClE,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAkB,CAAC;AAExD,sCAAsC;AAC/B,MAAM,oBAAoB,GAAG,KAAK,EAAE,EACzC,SAAS,EACT,KAAK,EACL,eAAe,GAKhB,EAAyC,EAAE;IAC1C,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,qCAAqC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,IAAI,eAAe,KAAK,uBAAe,CAAC,KAAK,EAAE,CAAC;QAC9C,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAA,qCAAkB,EAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAE7E,sCAAsC;QACtC,qBAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO;YACL,SAAS;YACT,MAAM;SACP,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,mCAAgB,CAAC,SAAS,CAC/C,yDAAiC,CAAC,sBAAsB,EACxD;QACE,IAAI,EAAE,yDAAiC,CAAC,sBAAsB;QAC9D,KAAK;QACL,MAAM,EAAE,eAAe,KAAK,uBAAe,CAAC,OAAO;KACpD,EACD,KAAK,CACN,CAAC;IAEF,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,kCAAkC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,eAAM,CAAC,KAAK,CAAC;QACX,GAAG,EAAE,sCAAsC;QAC3C,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW;KACnC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AA7CW,QAAA,oBAAoB,wBA6C/B;AAEK,MAAM,qBAAqB,GAAG,KAAK,EAAE,EAC1C,SAAS,EACT,MAAM,EACN,QAAQ,GAKT,EAA0C,EAAE;IAC3C,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,uCAAuC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAElF,IAAI,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,uBAAe,CAAC,KAAK,EAAE,CAAC;QACxE,gDAAgD;QAChD,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACxD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAA,uBAAI,EAAC,UAAU,CAAC,CAAC;YACvB,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,eAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,yDAAyD,EAAE,SAAS,EAAE,CAAC,CAAC;YAC3F,MAAM,IAAA,uBAAI,GAAE,CAAC;QACf,CAAC;QACD,OAAO;YACL,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,mCAAgB,CAAC,SAAS,CAC/C,yDAAiC,CAAC,uBAAuB,EACzD;QACE,IAAI,EAAE,yDAAiC,CAAC,uBAAuB;QAC/D,SAAS;QACT,MAAM;KACP,EACD,KAAK,CACN,CAAC;IAEF,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,mCAAmC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAC7F,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,wCAAwC,EAAE,SAAS,EAAE,CAAC,CAAC;IAC3E,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AA5CW,QAAA,qBAAqB,yBA4ChC;AAEK,MAAM,iBAAiB,GAAG,KAAK,EAAE,OAAiC,EAAsC,EAAE;;IAC/G,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"executeBrowserTask.js","sourceRoot":"","sources":["../../src/browserTask/executeBrowserTask.ts"],"names":[],"mappings":";;;;;;AAAA,2DAA2C;AAC3C,4CAAyC;AACzC,mEAAgE;AAChE,gDAAwB;AACxB,6EAQ2C;AAC3C,yDAAqF;AACrF,mCAAoE;AACpE,kEAA6F;AAE7F,kEAAkE;AAClE,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAkB,CAAC;AAExD,sCAAsC;AAC/B,MAAM,oBAAoB,GAAG,KAAK,EAAE,EACzC,SAAS,EACT,KAAK,EACL,eAAe,GAKhB,EAAyC,EAAE;IAC1C,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,qCAAqC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,IAAI,eAAe,KAAK,uBAAe,CAAC,KAAK,EAAE,CAAC;QAC9C,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAA,qCAAkB,EAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAE7E,sCAAsC;QACtC,qBAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO;YACL,SAAS;YACT,MAAM;SACP,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,mCAAgB,CAAC,SAAS,CAC/C,yDAAiC,CAAC,sBAAsB,EACxD;QACE,IAAI,EAAE,yDAAiC,CAAC,sBAAsB;QAC9D,KAAK;QACL,MAAM,EAAE,eAAe,KAAK,uBAAe,CAAC,OAAO;KACpD,EACD,KAAK,CACN,CAAC;IAEF,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,kCAAkC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,eAAM,CAAC,KAAK,CAAC;QACX,GAAG,EAAE,sCAAsC;QAC3C,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW;KACnC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AA7CW,QAAA,oBAAoB,wBA6C/B;AAEK,MAAM,qBAAqB,GAAG,KAAK,EAAE,EAC1C,SAAS,EACT,MAAM,EACN,QAAQ,GAKT,EAA0C,EAAE;IAC3C,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,uCAAuC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAElF,IAAI,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,uBAAe,CAAC,KAAK,EAAE,CAAC;QACxE,gDAAgD;QAChD,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACxD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAA,uBAAI,EAAC,UAAU,CAAC,CAAC;YACvB,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,eAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,yDAAyD,EAAE,SAAS,EAAE,CAAC,CAAC;YAC3F,MAAM,IAAA,uBAAI,GAAE,CAAC;QACf,CAAC;QACD,OAAO;YACL,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,mCAAgB,CAAC,SAAS,CAC/C,yDAAiC,CAAC,uBAAuB,EACzD;QACE,IAAI,EAAE,yDAAiC,CAAC,uBAAuB;QAC/D,SAAS;QACT,MAAM;KACP,EACD,KAAK,CACN,CAAC;IAEF,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,mCAAmC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAC7F,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,wCAAwC,EAAE,SAAS,EAAE,CAAC,CAAC;IAC3E,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AA5CW,QAAA,qBAAqB,yBA4ChC;AAEK,MAAM,iBAAiB,GAAG,KAAK,EAAE,OAAiC,EAAsC,EAAE;;IAC/G,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,YAAY,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC;QAE5H,IAAI,eAAe,KAAK,uBAAe,CAAC,KAAK,EAAE,CAAC;YAC9C,IAAA,+CAA0B,GAAE,CAAC;YAC7B,MAAM,gBAAgB,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;YAE1E,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;YACvC,eAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,yBAAyB;gBAClC,IAAI;gBACJ,iBAAiB,EAAE,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,OAAO,KAAI,KAAK;aACtD,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAA,0BAAK,EAAC,IAAI,EAAE,IAAI,EAAE;gBAC9B,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;gBAC/B,GAAG,EAAE,IAAA,yCAAoB,GAAE;oBACzB,CAAC,CAAC;wBACE,GAAG,OAAO,CAAC,GAAG;wBACd,WAAW,EAAE,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC;wBAC9C,IAAI,EAAE,GAAG,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;qBACxE;oBACH,CAAC,CAAC,OAAO,CAAC,GAAG;aAChB,CAAC,CAAC;YAEH,mCAAmC;YACnC,MAAM,OAAO,GAAG;gBACd,SAAS;gBACT,MAAM;gBACN,IAAI;gBACJ,YAAY;gBACZ,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;gBACjC,gBAAgB;aACjB,CAAC;YAEF,8BAA8B;YAC9B,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3C,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAElB,IAAI,YAAY,GAAG,EAAE,CAAC;YACtB,IAAI,YAAY,GAAG,EAAE,CAAC;YAEtB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC7B,YAAY,IAAI,IAAI,CAAC;gBACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC7B,CAAC,CAAC,CAAC;YAEH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC7B,YAAY,IAAI,IAAI,CAAC;gBACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC7B,CAAC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,4CAA4C;gBAC5C,IAAI,CAAC,IAAA,wCAAqB,GAAE,EAAE,CAAC;oBAC7B,eAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,6DAA6D,EAAE,CAAC,CAAC;oBACzF,KAAK,CAAC,IAAI,EAAE,CAAC;oBACb,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,sEAAsE;YACtE,UAAU,CAAC,GAAG,EAAE;gBACd,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,aAAa;YAE1B,MAAM,QAAQ,GAAW,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC7D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,CAAC,CAAC,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC;YAEH,aAAa,CAAC,QAAQ,CAAC,CAAC;YAExB,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACnB,eAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;gBAC7E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;YAC1E,CAAC;YAED,eAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;YAE9C,IAAI,MAAM,GAAG,CAAA,MAAA,YAAY,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,0CAAE,IAAI,EAAE,KAAI,YAAY,CAAC;YAC3E,IAAI,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,MAAM,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,eAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,wBAAwB,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;YAED,OAAO;gBACL,MAAM;gBACN,KAAK,EAAE,EAAE;gBACT,UAAU,EAAE,EAAE;aACf,CAAC;QACJ,CAAC;QAED,eAAM,CAAC,KAAK,CAAC;YACX,GAAG,EAAE,kCAAkC;YACvC,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,SAAS;YACT,UAAU,EAAE,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,MAAM,KAAI,CAAC;YAC9B,MAAM,EAAE,eAAe,KAAK,uBAAe,CAAC,OAAO;YACnD,kBAAkB,EAAE,CAAA,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,MAAM,KAAI,CAAC;YAC7C,iBAAiB,EAAE,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,OAAO,KAAI,KAAK;SACtD,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,mCAAgB,CAAC,SAAS,CAC/C,yDAAiC,CAAC,mBAAmB,EACrD;YACE,IAAI,EAAE,yDAAiC,CAAC,mBAAmB;YAC3D,MAAM;YACN,IAAI;YACJ,SAAS;YACT,SAAS;YACT,KAAK;YACL,MAAM,EAAE,eAAe,KAAK,uBAAe,CAAC,OAAO;YACnD,WAAW;YACX,YAAY;SACb,EACD,MAAM,CACP,CAAC;QAEF,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnB,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,+BAA+B,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9E,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QAED,eAAM,CAAC,KAAK,CAAC;YACX,GAAG,EAAE,qCAAqC;YAC1C,SAAS;YACT,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM;YAC5B,SAAS,EAAE,CAAA,MAAA,QAAQ,CAAC,KAAK,0CAAE,MAAM,KAAI,CAAC;YACtC,cAAc,EAAE,CAAA,MAAA,QAAQ,CAAC,UAAU,0CAAE,MAAM,KAAI,CAAC;SACjD,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,6BAA6B,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC,CAAC;AAhJW,QAAA,iBAAiB,qBAgJ5B"}
|
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Browser Task Executor with Screenshot Capture Support
|
|
3
|
+
|
|
4
|
+
This script runs browser automation tasks using browser-use and automatically captures
|
|
5
|
+
screenshots at the end of each step, uploading them to S3.
|
|
6
|
+
|
|
7
|
+
Environment Variables for Screenshots:
|
|
8
|
+
- SCREENSHOT_S3_BUCKET: S3 bucket name (default: 'global-development-agentsforce')
|
|
9
|
+
- SCREENSHOT_S3_PREFIX: S3 key prefix (default: 'browser-use-runs-screenshots/')
|
|
10
|
+
- AWS_REGION: AWS region for S3 (default: 'us-east-1')
|
|
11
|
+
|
|
12
|
+
AWS credentials should be configured via standard AWS SDK methods.
|
|
13
|
+
"""
|
|
14
|
+
|
|
1
15
|
import asyncio
|
|
2
16
|
import json
|
|
3
17
|
from typing import List, Optional, Dict, Any, TypedDict
|
|
@@ -7,10 +21,19 @@ from browser_use.browser import BrowserProfile, BrowserSession
|
|
|
7
21
|
from browser_use.llm import ChatOpenAI
|
|
8
22
|
import os
|
|
9
23
|
import sys
|
|
24
|
+
import logging
|
|
25
|
+
from datetime import datetime
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
import boto3
|
|
28
|
+
from botocore.exceptions import ClientError
|
|
10
29
|
from dotenv import load_dotenv
|
|
11
30
|
|
|
12
31
|
load_dotenv()
|
|
13
32
|
|
|
33
|
+
# Configure logging
|
|
34
|
+
logging.basicConfig(level=logging.INFO)
|
|
35
|
+
logger = logging.getLogger(__name__)
|
|
36
|
+
|
|
14
37
|
class OutputSchemaItemRequired(TypedDict):
|
|
15
38
|
name: str
|
|
16
39
|
type: str # 'string' | 'number'
|
|
@@ -20,6 +43,82 @@ class OutputSchemaItem(OutputSchemaItemRequired, total=False):
|
|
|
20
43
|
required: bool
|
|
21
44
|
|
|
22
45
|
|
|
46
|
+
class ScreenshotCapture:
|
|
47
|
+
"""Simple screenshot capture class that takes screenshots at step end and uploads to S3"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
|
50
|
+
self.config = config or {}
|
|
51
|
+
|
|
52
|
+
# S3 configuration with defaults
|
|
53
|
+
self.s3_bucket = self.config.get('s3_bucket', os.getenv('SCREENSHOT_S3_BUCKET', 'global-development-agentsforce'))
|
|
54
|
+
self.s3_prefix = self.config.get('s3_prefix', os.getenv('SCREENSHOT_S3_PREFIX', 'browser-use-runs-screenshots/'))
|
|
55
|
+
|
|
56
|
+
# AWS region configuration
|
|
57
|
+
self.aws_region = self.config.get('aws_region', os.getenv('AWS_REGION', 'us-east-1'))
|
|
58
|
+
|
|
59
|
+
# Ensure prefix ends with slash
|
|
60
|
+
if not self.s3_prefix.endswith('/'):
|
|
61
|
+
self.s3_prefix += '/'
|
|
62
|
+
|
|
63
|
+
# Get session ID from config or generate one
|
|
64
|
+
self.session_id = self.config.get('session_id', datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3])
|
|
65
|
+
|
|
66
|
+
# Initialize S3 client with region
|
|
67
|
+
self.s3_client = boto3.client('s3', region_name=self.aws_region)
|
|
68
|
+
|
|
69
|
+
# Track step counter
|
|
70
|
+
self.step_counter = 0
|
|
71
|
+
|
|
72
|
+
logger.info(f"📸 Screenshot capture initialized - S3: s3://{self.s3_bucket}/{self.s3_prefix}")
|
|
73
|
+
logger.info(f" Session ID: {self.session_id}, Region: {self.aws_region}")
|
|
74
|
+
|
|
75
|
+
async def capture_screenshot(self, agent: Any) -> None:
|
|
76
|
+
"""Capture screenshot at step end and upload to S3"""
|
|
77
|
+
try:
|
|
78
|
+
# Get current page
|
|
79
|
+
page = await agent.browser_session.get_current_page()
|
|
80
|
+
|
|
81
|
+
# Get current URL for context
|
|
82
|
+
current_url = page.url
|
|
83
|
+
logger.info(f"📸 Taking step_end screenshot at: {current_url}")
|
|
84
|
+
|
|
85
|
+
# Update step counter
|
|
86
|
+
step_number = self.step_counter
|
|
87
|
+
self.step_counter += 1
|
|
88
|
+
|
|
89
|
+
# Take screenshot (viewport only)
|
|
90
|
+
screenshot_bytes = await page.screenshot(full_page=False)
|
|
91
|
+
|
|
92
|
+
# Generate filename with metadata
|
|
93
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]
|
|
94
|
+
filename = f"{self.session_id}/step_{step_number:04d}_{timestamp}.png"
|
|
95
|
+
|
|
96
|
+
# Upload to S3
|
|
97
|
+
s3_key = f"{self.s3_prefix}{filename}"
|
|
98
|
+
|
|
99
|
+
try:
|
|
100
|
+
# Upload with KMS encryption if required by bucket
|
|
101
|
+
self.s3_client.put_object(
|
|
102
|
+
Bucket=self.s3_bucket,
|
|
103
|
+
Key=s3_key,
|
|
104
|
+
Body=screenshot_bytes,
|
|
105
|
+
ContentType='image/png',
|
|
106
|
+
ServerSideEncryption='aws:kms',
|
|
107
|
+
Tagging='retention=30d'
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
s3_url = f"s3://{self.s3_bucket}/{s3_key}"
|
|
111
|
+
logger.info(f"✅ Screenshot uploaded: {s3_url}")
|
|
112
|
+
|
|
113
|
+
except ClientError as e:
|
|
114
|
+
logger.error(f"❌ Failed to upload screenshot to S3: {str(e)}")
|
|
115
|
+
# Continue execution even if screenshot upload fails
|
|
116
|
+
|
|
117
|
+
except Exception as e:
|
|
118
|
+
logger.error(f"❌ Error capturing screenshot: {str(e)}")
|
|
119
|
+
# Don't raise - continue execution
|
|
120
|
+
|
|
121
|
+
|
|
23
122
|
def create_pydantic_model_from_schema(output_schema: Optional[List[Dict[str, Any]]]) -> Optional[type[BaseModel]]:
|
|
24
123
|
if not output_schema:
|
|
25
124
|
return None
|
|
@@ -43,7 +142,8 @@ def create_pydantic_model_from_schema(output_schema: Optional[List[Dict[str, Any
|
|
|
43
142
|
return None
|
|
44
143
|
|
|
45
144
|
|
|
46
|
-
async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Optional[str] = None,
|
|
145
|
+
async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Optional[str] = None,
|
|
146
|
+
otp_secret: Optional[str] = None, screenshot_config: Optional[Dict[str, Any]] = None):
|
|
47
147
|
llm = ChatOpenAI(
|
|
48
148
|
model="gpt-4.1",
|
|
49
149
|
api_key=os.getenv("OPENAI_API_KEY"),
|
|
@@ -60,6 +160,22 @@ async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Opt
|
|
|
60
160
|
browser_profile=BrowserProfile(downloads_path=folder_path),
|
|
61
161
|
cdp_url=cdp_url
|
|
62
162
|
)
|
|
163
|
+
|
|
164
|
+
# Initialize screenshot capture if enabled
|
|
165
|
+
screenshot_capture = None
|
|
166
|
+
on_step_end_hook = None
|
|
167
|
+
|
|
168
|
+
if screenshot_config and screenshot_config.get('enabled', False):
|
|
169
|
+
logger.info("📸 Screenshot capture enabled (captures at step end)")
|
|
170
|
+
# Add session_id to config
|
|
171
|
+
screenshot_config['session_id'] = session_id
|
|
172
|
+
screenshot_capture = ScreenshotCapture(screenshot_config)
|
|
173
|
+
|
|
174
|
+
# Create hook function for step end only
|
|
175
|
+
async def on_step_end(agent: Any) -> None:
|
|
176
|
+
await screenshot_capture.capture_screenshot(agent)
|
|
177
|
+
|
|
178
|
+
on_step_end_hook = on_step_end
|
|
63
179
|
|
|
64
180
|
output_schema = None
|
|
65
181
|
if output_schema_json:
|
|
@@ -99,7 +215,11 @@ async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Opt
|
|
|
99
215
|
available_file_paths=available_files,
|
|
100
216
|
browser_session=browser_session,
|
|
101
217
|
)
|
|
102
|
-
|
|
218
|
+
|
|
219
|
+
# Run agent with screenshot hook if configured
|
|
220
|
+
history = await agent.run(
|
|
221
|
+
on_step_end=on_step_end_hook
|
|
222
|
+
)
|
|
103
223
|
final_result = history.final_result()
|
|
104
224
|
|
|
105
225
|
# Parse result with output model if provided
|
|
@@ -143,11 +263,12 @@ if __name__ == '__main__':
|
|
|
143
263
|
task = payload.get('task')
|
|
144
264
|
output_schema = payload.get('outputSchema')
|
|
145
265
|
otp_secret = payload.get('otpSecret')
|
|
266
|
+
screenshot_config = payload.get('screenshotConfig')
|
|
146
267
|
|
|
147
268
|
if not session_id or not cdp_url or not task:
|
|
148
269
|
raise SystemExit("Missing required fields in JSON payload: sessionId, cdpUrl, task")
|
|
149
270
|
|
|
150
271
|
output_schema_json = json.dumps(output_schema) if output_schema is not None else None
|
|
151
|
-
asyncio.run(main(session_id, cdp_url, task, output_schema_json, otp_secret))
|
|
272
|
+
asyncio.run(main(session_id, cdp_url, task, output_schema_json, otp_secret, screenshot_config))
|
|
152
273
|
else:
|
|
153
274
|
raise SystemExit("Usage: uv run executeBrowserTask.py; send a JSON payload via stdin")
|
|
@@ -22,6 +22,12 @@ export type ToolSchema = {
|
|
|
22
22
|
required?: boolean;
|
|
23
23
|
}>;
|
|
24
24
|
};
|
|
25
|
+
export type ScreenshotConfig = {
|
|
26
|
+
enabled: boolean;
|
|
27
|
+
s3_bucket?: string;
|
|
28
|
+
s3_prefix?: string;
|
|
29
|
+
aws_region?: string;
|
|
30
|
+
};
|
|
25
31
|
export interface InvokeBrowserTaskOptions {
|
|
26
32
|
sessionId: string;
|
|
27
33
|
cdpUrl: string;
|
|
@@ -31,5 +37,6 @@ export interface InvokeBrowserTaskOptions {
|
|
|
31
37
|
browserTaskMode: BrowserTaskMode;
|
|
32
38
|
toolSchemas?: ToolSchema[];
|
|
33
39
|
outputSchema?: OutputSchemaField[];
|
|
40
|
+
screenshotConfig?: ScreenshotConfig;
|
|
34
41
|
}
|
|
35
42
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/browserTask/types.ts"],"names":[],"mappings":"AAAA,oBAAY,eAAe;IACzB,OAAO,WAAW;IAClB,KAAK,aAAa;IAClB,KAAK,UAAU;CAChB;AAGD,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAE/C,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,KAAK,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;QAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC,CAAC;CACJ,CAAC;AAEF,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,eAAe,EAAE,eAAe,CAAC;IACjC,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,YAAY,CAAC,EAAE,iBAAiB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/browserTask/types.ts"],"names":[],"mappings":"AAAA,oBAAY,eAAe;IACzB,OAAO,WAAW;IAClB,KAAK,aAAa;IAClB,KAAK,UAAU;CAChB;AAGD,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAE/C,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,KAAK,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;QAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC,CAAC;CACJ,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,eAAe,EAAE,eAAe,CAAC;IACjC,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,YAAY,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACnC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"addBrowserTaskRunNode.d.ts","sourceRoot":"","sources":["../../src/nodes/addBrowserTaskRunNode.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAmB,MAAM,0BAA0B,CAAC;AAE7E,OAAO,EAAiC,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEtF,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"addBrowserTaskRunNode.d.ts","sourceRoot":"","sources":["../../src/nodes/addBrowserTaskRunNode.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAmB,MAAM,0BAA0B,CAAC;AAE7E,OAAO,EAAiC,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEtF,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAWjC,KAAK,2BAA2B,GAAG;IACjC,KAAK,EAAE,gBAAgB,CAAC;IACxB,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd,CAAC;AAEF,eAAO,MAAM,2BAA2B,GAAI,UAAU,MAAM,WAAyD,CAAC;AAEtH,eAAO,MAAM,qBAAqB,GAAU,uDAAuD,2BAA2B,kBA2M7H,CAAC"}
|
|
@@ -41,6 +41,35 @@ const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedToNodeNam
|
|
|
41
41
|
// We compile the env variables to avoid having to pass them to the platform in the tool input
|
|
42
42
|
const promptCompiledWithEnv = (0, compilePrompt_1.compilePrompt)(prompt, { env: process.env });
|
|
43
43
|
const { browserTaskMode } = (0, config_1.getConfig)();
|
|
44
|
+
// Build screenshot config conditionally based on environment variables
|
|
45
|
+
const screenshotConfig = (() => {
|
|
46
|
+
// Only enable screenshots if S3 bucket is configured or explicitly enabled
|
|
47
|
+
const screenshotEnabled = process.env.ENABLE_BROWSER_SCREENSHOTS === 'true' || !!process.env.SCREENSHOT_S3_BUCKET;
|
|
48
|
+
if (!screenshotEnabled) {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
// Screenshots enabled - build config
|
|
52
|
+
const config = {
|
|
53
|
+
enabled: true,
|
|
54
|
+
};
|
|
55
|
+
// Add optional S3 configuration if provided
|
|
56
|
+
if (process.env.SCREENSHOT_S3_BUCKET) {
|
|
57
|
+
config.s3_bucket = process.env.SCREENSHOT_S3_BUCKET;
|
|
58
|
+
}
|
|
59
|
+
if (process.env.SCREENSHOT_S3_PREFIX) {
|
|
60
|
+
config.s3_prefix = process.env.SCREENSHOT_S3_PREFIX;
|
|
61
|
+
}
|
|
62
|
+
if (process.env.AWS_REGION) {
|
|
63
|
+
config.aws_region = process.env.AWS_REGION;
|
|
64
|
+
}
|
|
65
|
+
logger_1.logger.debug({
|
|
66
|
+
msg: 'Screenshot configuration enabled',
|
|
67
|
+
bucket: config.s3_bucket || 'default',
|
|
68
|
+
prefix: config.s3_prefix || 'default',
|
|
69
|
+
region: config.aws_region || 'default',
|
|
70
|
+
});
|
|
71
|
+
return config;
|
|
72
|
+
})();
|
|
44
73
|
// Invoke browser task via socket
|
|
45
74
|
const result = await (0, executeBrowserTask_1.invokeBrowserTask)({
|
|
46
75
|
sessionId,
|
|
@@ -51,6 +80,7 @@ const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedToNodeNam
|
|
|
51
80
|
browserTaskMode,
|
|
52
81
|
toolSchemas,
|
|
53
82
|
outputSchema,
|
|
83
|
+
screenshotConfig,
|
|
54
84
|
});
|
|
55
85
|
logger_1.logger.debug({
|
|
56
86
|
msg: 'Browser task completed',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"addBrowserTaskRunNode.js","sourceRoot":"","sources":["../../src/nodes/addBrowserTaskRunNode.ts"],"names":[],"mappings":";;;AAEA,sDAAsF;AACtF,uDAAkE;AAElE,4CAAyC;AACzC,8CAAqD;AAErD,+BAAoC;AACpC,0EAAsE;
|
|
1
|
+
{"version":3,"file":"addBrowserTaskRunNode.js","sourceRoot":"","sources":["../../src/nodes/addBrowserTaskRunNode.ts"],"names":[],"mappings":";;;AAEA,sDAAsF;AACtF,uDAAkE;AAElE,4CAAyC;AACzC,8CAAqD;AAErD,+BAAoC;AACpC,0EAAsE;AAEtE,mDAA4D;AAC5D,+CAA+C;AAC/C,mDAAgD;AASzC,MAAM,2BAA2B,GAAG,CAAC,QAAgB,EAAE,EAAE,CAAC,GAAG,QAAQ,GAAG,iCAAmB,CAAC,gBAAgB,EAAE,CAAC;AAAzG,QAAA,2BAA2B,+BAA8E;AAE/G,MAAM,qBAAqB,GAAG,KAAK,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,KAAK,EAA+B,EAAE,EAAE;IAChI,MAAM,QAAQ,GAAiB,KAAK,EAAE,KAAmC,EAAE,EAAE;;QAC3E,eAAM,CAAC,IAAI,CAAC,uCAAuC,eAAe,CAAC,WAAW,EAAE,CAAC,CAAC;QAElF,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAQ,CAAC;QACrE,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,sBAAsB,GAAG,MAAA,WAAW,CAAC,iBAAiB,0CAAE,cAAc,CAAC;QAC7E,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,EAAE,MAAM,EAAE,GAAG,WAAW,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC;QACjD,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,sBAAsB,CAAC;QAErF,IAAI,CAAC;YACH,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACrC,CAAC;YAED,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAC/E,CAAC;YAED,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK;iBAC5B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC;iBACtD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACd,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAA,yBAAiB,EAAC,IAAI,CAAC,KAAK,CAAC;aAC3C,CAAC,CAAC,CAAC;YAEN,8FAA8F;YAC9F,MAAM,qBAAqB,GAAG,IAAA,6BAAa,EAAC,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAE1E,MAAM,EAAE,eAAe,EAAE,GAAG,IAAA,kBAAS,GAAE,CAAC;YAExC,uEAAuE;YACvE,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE;gBAC7B,2EAA2E;gBAC3E,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;gBAElH,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBACvB,OAAO,SAAS,CAAC;gBACnB,CAAC;gBAED,qCAAqC;gBACrC,MAAM,MAAM,GAAqB;oBAC/B,OAAO,EAAE,IAAI;iBACd,CAAC;gBAEF,4CAA4C;gBAC5C,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC;oBACrC,MAAM,CAAC,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;gBACtD,CAAC;gBACD,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC;oBACrC,MAAM,CAAC,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;gBACtD,CAAC;gBACD,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;oBAC3B,MAAM,CAAC,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;gBAC7C,CAAC;gBAED,eAAM,CAAC,KAAK,CAAC;oBACX,GAAG,EAAE,kCAAkC;oBACvC,MAAM,EAAE,MAAM,CAAC,SAAS,IAAI,SAAS;oBACrC,MAAM,EAAE,MAAM,CAAC,SAAS,IAAI,SAAS;oBACrC,MAAM,EAAE,MAAM,CAAC,UAAU,IAAI,SAAS;iBACvC,CAAC,CAAC;gBAEH,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC,EAAE,CAAC;YAEL,iCAAiC;YACjC,MAAM,MAAM,GAAG,MAAM,IAAA,sCAAiB,EAAC;gBACrC,SAAS;gBACT,MAAM;gBACN,IAAI,EAAE,qBAAqB;gBAC3B,SAAS;gBACT,KAAK;gBACL,eAAe;gBACf,WAAW;gBACX,YAAY;gBACZ,gBAAgB;aACjB,CAAC,CAAC;YAEH,eAAM,CAAC,KAAK,CAAC;gBACX,GAAG,EAAE,wBAAwB;gBAC7B,SAAS;gBACT,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM;gBAC1B,SAAS,EAAE,CAAA,MAAA,MAAM,CAAC,KAAK,0CAAE,MAAM,KAAI,CAAC;gBACpC,cAAc,EAAE,CAAA,MAAA,MAAM,CAAC,UAAU,0CAAE,MAAM,KAAI,CAAC;aAC/C,CAAC,CAAC;YAEH,sCAAsC;YACtC,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC;gBAClC,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;oBACtB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,gCAAgC;oBACzD,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;oBACzB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE;oBACnC,WAAW,EAAE,WAAW;iBACzB,CAAC;gBACF,IAAI,EAAE,cAAc;gBACpB,YAAY,EAAE,QAAQ,CAAC,EAAE;gBACzB,MAAM,EAAE,SAAS;gBACjB,iBAAiB,EAAE;oBACjB,cAAc,EAAE;wBACd,QAAQ,EAAE,sBAAQ,CAAC,YAAY;wBAC/B,eAAe,EAAE,eAAe,CAAC,WAAW;wBAC5C,SAAS;wBACT,KAAK,EAAE,MAAM,CAAC,KAAK;wBACnB,UAAU,EAAE,MAAM,CAAC,UAAU;wBAC7B,WAAW,EAAE,WAAW;qBACzB;iBACF;aACF,CAAC,CAAC;YAEH,qEAAqE;YACrE,MAAM,gBAAgB,GAAG,IAAI,oBAAS,CAAC;gBACrC,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,iBAAiB,EAAE;oBACjB,GAAG,WAAW,CAAC,iBAAiB;oBAChC,cAAc,EAAE;wBACd,GAAG,MAAA,WAAW,CAAC,iBAAiB,0CAAE,cAAc;wBAChD,eAAe,EAAE,eAAe,CAAC,WAAW;wBAC5C,KAAK,EAAE,MAAM,CAAC,KAAK;wBACnB,UAAU,EAAE,MAAM,CAAC,UAAU;wBAC7B,MAAM,EAAE,WAAW;qBACpB;oBACD,MAAM,EAAE,IAAI,EAAE,mEAAmE;iBAClF;aACF,CAAC,CAAC;YAEH,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YACrB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;YACnD,KAAK,CAAC,OAAO,CAAC,IAAI,CAChB,IAAA,2BAAiB,EAAc,KAAK,CAAC,OAAO,EAAE;gBAC5C,IAAI,EAAE,sBAAQ,CAAC,YAAY;gBAC3B,MAAM,EAAE,eAAe,CAAC,IAAI;gBAC5B,eAAe,EAAE,eAAe,CAAC,WAAW;gBAC5C,GAAG,EAAE,MAAM;gBACX,UAAU,EAAE,CAAC,WAAW,CAAC,EAAG,CAAC;aAC9B,CAAC,CACH,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,eAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,uCAAuC,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAExF,MAAM,gBAAgB,GAAG,IAAI,sBAAW,CAAC;gBACvC,EAAE,EAAE,IAAA,SAAM,GAAE;gBACZ,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;oBACtB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE;oBAChE,WAAW,EAAE,WAAW;oBACxB,OAAO,EAAE,gCAAgC;iBAC1C,CAAC;gBACF,MAAM,EAAE,OAAO;gBACf,IAAI,EAAE,cAAc;gBACpB,YAAY,EAAE,QAAQ,CAAC,EAAE;gBACzB,iBAAiB,EAAE;oBACjB,cAAc,EAAE;wBACd,QAAQ,EAAE,sBAAQ,CAAC,YAAY;wBAC/B,eAAe,EAAE,eAAe,CAAC,WAAW;wBAC5C,SAAS;wBACT,KAAK,EAAE,KAAK,CAAC,OAAO;wBACpB,WAAW,EAAE,WAAW;qBACzB;iBACF;aACF,CAAC,CAAC;YAEH,0CAA0C;YAC1C,MAAM,gBAAgB,GAAG,IAAI,oBAAS,CAAC;gBACrC,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,iBAAiB,EAAE;oBACjB,GAAG,WAAW,CAAC,iBAAiB;oBAChC,cAAc,EAAE;wBACd,GAAG,MAAA,WAAW,CAAC,iBAAiB,0CAAE,cAAc;wBAChD,eAAe,EAAE,eAAe,CAAC,WAAW;wBAC5C,MAAM,EAAE,QAAQ;wBAChB,KAAK,EAAE,KAAK,CAAC,OAAO;qBACrB;oBACD,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;YAEH,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YACrB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;YACxD,KAAK,CAAC,OAAO,CAAC,IAAI,CAChB,IAAA,2BAAiB,EAAc,KAAK,CAAC,OAAO,EAAE;gBAC5C,IAAI,EAAE,sBAAQ,CAAC,YAAY;gBAC3B,MAAM,EAAE,eAAe,CAAC,IAAI;gBAC5B,eAAe,EAAE,eAAe,CAAC,WAAW;gBAC5C,GAAG,EAAE,KAAK;gBACV,UAAU,EAAE,EAAE;aACf,CAAC,CACH,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,CAAC,OAAO,CAAC,IAAA,mCAA2B,EAAC,kBAAkB,CAAC,EAAE,QAAQ,CAAC,CAAC;IACzE,KAAK,CAAC,OAAO,CAAC,kBAAyB,EAAE,IAAA,mCAA2B,EAAC,kBAAkB,CAAQ,CAAC,CAAC;AACnG,CAAC,CAAC;AA3MW,QAAA,qBAAqB,yBA2MhC"}
|
package/package.json
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Browser Task Executor with Screenshot Capture Support
|
|
3
|
+
|
|
4
|
+
This script runs browser automation tasks using browser-use and automatically captures
|
|
5
|
+
screenshots at the end of each step, uploading them to S3.
|
|
6
|
+
|
|
7
|
+
Environment Variables for Screenshots:
|
|
8
|
+
- SCREENSHOT_S3_BUCKET: S3 bucket name (default: 'global-development-agentsforce')
|
|
9
|
+
- SCREENSHOT_S3_PREFIX: S3 key prefix (default: 'browser-use-runs-screenshots/')
|
|
10
|
+
- AWS_REGION: AWS region for S3 (default: 'us-east-1')
|
|
11
|
+
|
|
12
|
+
AWS credentials should be configured via standard AWS SDK methods.
|
|
13
|
+
"""
|
|
14
|
+
|
|
1
15
|
import asyncio
|
|
2
16
|
import json
|
|
3
17
|
from typing import List, Optional, Dict, Any, TypedDict
|
|
@@ -7,10 +21,19 @@ from browser_use.browser import BrowserProfile, BrowserSession
|
|
|
7
21
|
from browser_use.llm import ChatOpenAI
|
|
8
22
|
import os
|
|
9
23
|
import sys
|
|
24
|
+
import logging
|
|
25
|
+
from datetime import datetime
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
import boto3
|
|
28
|
+
from botocore.exceptions import ClientError
|
|
10
29
|
from dotenv import load_dotenv
|
|
11
30
|
|
|
12
31
|
load_dotenv()
|
|
13
32
|
|
|
33
|
+
# Configure logging
|
|
34
|
+
logging.basicConfig(level=logging.INFO)
|
|
35
|
+
logger = logging.getLogger(__name__)
|
|
36
|
+
|
|
14
37
|
class OutputSchemaItemRequired(TypedDict):
|
|
15
38
|
name: str
|
|
16
39
|
type: str # 'string' | 'number'
|
|
@@ -20,6 +43,82 @@ class OutputSchemaItem(OutputSchemaItemRequired, total=False):
|
|
|
20
43
|
required: bool
|
|
21
44
|
|
|
22
45
|
|
|
46
|
+
class ScreenshotCapture:
|
|
47
|
+
"""Simple screenshot capture class that takes screenshots at step end and uploads to S3"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
|
50
|
+
self.config = config or {}
|
|
51
|
+
|
|
52
|
+
# S3 configuration with defaults
|
|
53
|
+
self.s3_bucket = self.config.get('s3_bucket', os.getenv('SCREENSHOT_S3_BUCKET', 'global-development-agentsforce'))
|
|
54
|
+
self.s3_prefix = self.config.get('s3_prefix', os.getenv('SCREENSHOT_S3_PREFIX', 'browser-use-runs-screenshots/'))
|
|
55
|
+
|
|
56
|
+
# AWS region configuration
|
|
57
|
+
self.aws_region = self.config.get('aws_region', os.getenv('AWS_REGION', 'us-east-1'))
|
|
58
|
+
|
|
59
|
+
# Ensure prefix ends with slash
|
|
60
|
+
if not self.s3_prefix.endswith('/'):
|
|
61
|
+
self.s3_prefix += '/'
|
|
62
|
+
|
|
63
|
+
# Get session ID from config or generate one
|
|
64
|
+
self.session_id = self.config.get('session_id', datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3])
|
|
65
|
+
|
|
66
|
+
# Initialize S3 client with region
|
|
67
|
+
self.s3_client = boto3.client('s3', region_name=self.aws_region)
|
|
68
|
+
|
|
69
|
+
# Track step counter
|
|
70
|
+
self.step_counter = 0
|
|
71
|
+
|
|
72
|
+
logger.info(f"📸 Screenshot capture initialized - S3: s3://{self.s3_bucket}/{self.s3_prefix}")
|
|
73
|
+
logger.info(f" Session ID: {self.session_id}, Region: {self.aws_region}")
|
|
74
|
+
|
|
75
|
+
async def capture_screenshot(self, agent: Any) -> None:
|
|
76
|
+
"""Capture screenshot at step end and upload to S3"""
|
|
77
|
+
try:
|
|
78
|
+
# Get current page
|
|
79
|
+
page = await agent.browser_session.get_current_page()
|
|
80
|
+
|
|
81
|
+
# Get current URL for context
|
|
82
|
+
current_url = page.url
|
|
83
|
+
logger.info(f"📸 Taking step_end screenshot at: {current_url}")
|
|
84
|
+
|
|
85
|
+
# Update step counter
|
|
86
|
+
step_number = self.step_counter
|
|
87
|
+
self.step_counter += 1
|
|
88
|
+
|
|
89
|
+
# Take screenshot (viewport only)
|
|
90
|
+
screenshot_bytes = await page.screenshot(full_page=False)
|
|
91
|
+
|
|
92
|
+
# Generate filename with metadata
|
|
93
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]
|
|
94
|
+
filename = f"{self.session_id}/step_{step_number:04d}_{timestamp}.png"
|
|
95
|
+
|
|
96
|
+
# Upload to S3
|
|
97
|
+
s3_key = f"{self.s3_prefix}{filename}"
|
|
98
|
+
|
|
99
|
+
try:
|
|
100
|
+
# Upload with KMS encryption if required by bucket
|
|
101
|
+
self.s3_client.put_object(
|
|
102
|
+
Bucket=self.s3_bucket,
|
|
103
|
+
Key=s3_key,
|
|
104
|
+
Body=screenshot_bytes,
|
|
105
|
+
ContentType='image/png',
|
|
106
|
+
ServerSideEncryption='aws:kms',
|
|
107
|
+
Tagging='retention=30d'
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
s3_url = f"s3://{self.s3_bucket}/{s3_key}"
|
|
111
|
+
logger.info(f"✅ Screenshot uploaded: {s3_url}")
|
|
112
|
+
|
|
113
|
+
except ClientError as e:
|
|
114
|
+
logger.error(f"❌ Failed to upload screenshot to S3: {str(e)}")
|
|
115
|
+
# Continue execution even if screenshot upload fails
|
|
116
|
+
|
|
117
|
+
except Exception as e:
|
|
118
|
+
logger.error(f"❌ Error capturing screenshot: {str(e)}")
|
|
119
|
+
# Don't raise - continue execution
|
|
120
|
+
|
|
121
|
+
|
|
23
122
|
def create_pydantic_model_from_schema(output_schema: Optional[List[Dict[str, Any]]]) -> Optional[type[BaseModel]]:
|
|
24
123
|
if not output_schema:
|
|
25
124
|
return None
|
|
@@ -43,7 +142,8 @@ def create_pydantic_model_from_schema(output_schema: Optional[List[Dict[str, Any
|
|
|
43
142
|
return None
|
|
44
143
|
|
|
45
144
|
|
|
46
|
-
async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Optional[str] = None,
|
|
145
|
+
async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Optional[str] = None,
|
|
146
|
+
otp_secret: Optional[str] = None, screenshot_config: Optional[Dict[str, Any]] = None):
|
|
47
147
|
llm = ChatOpenAI(
|
|
48
148
|
model="gpt-4.1",
|
|
49
149
|
api_key=os.getenv("OPENAI_API_KEY"),
|
|
@@ -60,6 +160,22 @@ async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Opt
|
|
|
60
160
|
browser_profile=BrowserProfile(downloads_path=folder_path),
|
|
61
161
|
cdp_url=cdp_url
|
|
62
162
|
)
|
|
163
|
+
|
|
164
|
+
# Initialize screenshot capture if enabled
|
|
165
|
+
screenshot_capture = None
|
|
166
|
+
on_step_end_hook = None
|
|
167
|
+
|
|
168
|
+
if screenshot_config and screenshot_config.get('enabled', False):
|
|
169
|
+
logger.info("📸 Screenshot capture enabled (captures at step end)")
|
|
170
|
+
# Add session_id to config
|
|
171
|
+
screenshot_config['session_id'] = session_id
|
|
172
|
+
screenshot_capture = ScreenshotCapture(screenshot_config)
|
|
173
|
+
|
|
174
|
+
# Create hook function for step end only
|
|
175
|
+
async def on_step_end(agent: Any) -> None:
|
|
176
|
+
await screenshot_capture.capture_screenshot(agent)
|
|
177
|
+
|
|
178
|
+
on_step_end_hook = on_step_end
|
|
63
179
|
|
|
64
180
|
output_schema = None
|
|
65
181
|
if output_schema_json:
|
|
@@ -99,7 +215,11 @@ async def main(session_id: str, cdp_url: str, task: str, output_schema_json: Opt
|
|
|
99
215
|
available_file_paths=available_files,
|
|
100
216
|
browser_session=browser_session,
|
|
101
217
|
)
|
|
102
|
-
|
|
218
|
+
|
|
219
|
+
# Run agent with screenshot hook if configured
|
|
220
|
+
history = await agent.run(
|
|
221
|
+
on_step_end=on_step_end_hook
|
|
222
|
+
)
|
|
103
223
|
final_result = history.final_result()
|
|
104
224
|
|
|
105
225
|
# Parse result with output model if provided
|
|
@@ -143,11 +263,12 @@ if __name__ == '__main__':
|
|
|
143
263
|
task = payload.get('task')
|
|
144
264
|
output_schema = payload.get('outputSchema')
|
|
145
265
|
otp_secret = payload.get('otpSecret')
|
|
266
|
+
screenshot_config = payload.get('screenshotConfig')
|
|
146
267
|
|
|
147
268
|
if not session_id or not cdp_url or not task:
|
|
148
269
|
raise SystemExit("Missing required fields in JSON payload: sessionId, cdpUrl, task")
|
|
149
270
|
|
|
150
271
|
output_schema_json = json.dumps(output_schema) if output_schema is not None else None
|
|
151
|
-
asyncio.run(main(session_id, cdp_url, task, output_schema_json, otp_secret))
|
|
272
|
+
asyncio.run(main(session_id, cdp_url, task, output_schema_json, otp_secret, screenshot_config))
|
|
152
273
|
else:
|
|
153
274
|
raise SystemExit("Usage: uv run executeBrowserTask.py; send a JSON payload via stdin")
|
|
@@ -114,14 +114,18 @@ export const destroyBrowserSession = async ({
|
|
|
114
114
|
|
|
115
115
|
export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Promise<InvokeBrowserTaskResponse> => {
|
|
116
116
|
try {
|
|
117
|
-
const { sessionId, cdpUrl, task, keepAlive, hooks, browserTaskMode, toolSchemas, outputSchema } = options;
|
|
117
|
+
const { sessionId, cdpUrl, task, keepAlive, hooks, browserTaskMode, toolSchemas, outputSchema, screenshotConfig } = options;
|
|
118
118
|
|
|
119
119
|
if (browserTaskMode === BrowserTaskMode.LOCAL) {
|
|
120
120
|
validateLocalOperatorSetup();
|
|
121
121
|
const pythonScriptPath = path.resolve(__dirname, 'executeBrowserTask.py');
|
|
122
122
|
|
|
123
123
|
const args = ['run', pythonScriptPath];
|
|
124
|
-
logger.info({
|
|
124
|
+
logger.info({
|
|
125
|
+
message: 'Spawning Python process',
|
|
126
|
+
args,
|
|
127
|
+
screenshotEnabled: screenshotConfig?.enabled || false,
|
|
128
|
+
});
|
|
125
129
|
|
|
126
130
|
const child = spawn('uv', args, {
|
|
127
131
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
@@ -141,6 +145,7 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
|
|
|
141
145
|
task,
|
|
142
146
|
outputSchema,
|
|
143
147
|
otpSecret: process.env.OTP_SECRET,
|
|
148
|
+
screenshotConfig,
|
|
144
149
|
};
|
|
145
150
|
|
|
146
151
|
// Write JSON payload to stdin
|
|
@@ -214,6 +219,7 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
|
|
|
214
219
|
hooksCount: hooks?.length || 0,
|
|
215
220
|
onPrem: browserTaskMode === BrowserTaskMode.ON_PREM,
|
|
216
221
|
outputSchemaFields: outputSchema?.length || 0,
|
|
222
|
+
screenshotEnabled: screenshotConfig?.enabled || false,
|
|
217
223
|
});
|
|
218
224
|
|
|
219
225
|
const response = await mindedConnection.awaitEmit<InvokeBrowserTaskRequest, InvokeBrowserTaskResponse>(
|
package/src/browserTask/types.ts
CHANGED
|
@@ -25,6 +25,13 @@ export type ToolSchema = {
|
|
|
25
25
|
}>;
|
|
26
26
|
};
|
|
27
27
|
|
|
28
|
+
export type ScreenshotConfig = {
|
|
29
|
+
enabled: boolean;
|
|
30
|
+
s3_bucket?: string;
|
|
31
|
+
s3_prefix?: string;
|
|
32
|
+
aws_region?: string;
|
|
33
|
+
};
|
|
34
|
+
|
|
28
35
|
export interface InvokeBrowserTaskOptions {
|
|
29
36
|
sessionId: string;
|
|
30
37
|
cdpUrl: string;
|
|
@@ -34,4 +41,5 @@ export interface InvokeBrowserTaskOptions {
|
|
|
34
41
|
browserTaskMode: BrowserTaskMode;
|
|
35
42
|
toolSchemas?: ToolSchema[];
|
|
36
43
|
outputSchema?: OutputSchemaField[];
|
|
37
|
-
|
|
44
|
+
screenshotConfig?: ScreenshotConfig;
|
|
45
|
+
}
|
|
@@ -8,6 +8,7 @@ import { createHistoryStep } from '../utils/history';
|
|
|
8
8
|
import { HistoryStep } from '../types/Agent.types';
|
|
9
9
|
import { v4 as uuidv4 } from 'uuid';
|
|
10
10
|
import { invokeBrowserTask } from '../browserTask/executeBrowserTask';
|
|
11
|
+
import { ScreenshotConfig } from '../browserTask/types';
|
|
11
12
|
import { zodSchemaToParams } from '../platform/utils/tools';
|
|
12
13
|
import { getConfig } from '../platform/config';
|
|
13
14
|
import { compilePrompt } from './compilePrompt';
|
|
@@ -55,6 +56,42 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
|
|
|
55
56
|
const promptCompiledWithEnv = compilePrompt(prompt, { env: process.env });
|
|
56
57
|
|
|
57
58
|
const { browserTaskMode } = getConfig();
|
|
59
|
+
|
|
60
|
+
// Build screenshot config conditionally based on environment variables
|
|
61
|
+
const screenshotConfig = (() => {
|
|
62
|
+
// Only enable screenshots if S3 bucket is configured or explicitly enabled
|
|
63
|
+
const screenshotEnabled = process.env.ENABLE_BROWSER_SCREENSHOTS === 'true' || !!process.env.SCREENSHOT_S3_BUCKET;
|
|
64
|
+
|
|
65
|
+
if (!screenshotEnabled) {
|
|
66
|
+
return undefined;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Screenshots enabled - build config
|
|
70
|
+
const config: ScreenshotConfig = {
|
|
71
|
+
enabled: true,
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
// Add optional S3 configuration if provided
|
|
75
|
+
if (process.env.SCREENSHOT_S3_BUCKET) {
|
|
76
|
+
config.s3_bucket = process.env.SCREENSHOT_S3_BUCKET;
|
|
77
|
+
}
|
|
78
|
+
if (process.env.SCREENSHOT_S3_PREFIX) {
|
|
79
|
+
config.s3_prefix = process.env.SCREENSHOT_S3_PREFIX;
|
|
80
|
+
}
|
|
81
|
+
if (process.env.AWS_REGION) {
|
|
82
|
+
config.aws_region = process.env.AWS_REGION;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
logger.debug({
|
|
86
|
+
msg: 'Screenshot configuration enabled',
|
|
87
|
+
bucket: config.s3_bucket || 'default',
|
|
88
|
+
prefix: config.s3_prefix || 'default',
|
|
89
|
+
region: config.aws_region || 'default',
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
return config;
|
|
93
|
+
})();
|
|
94
|
+
|
|
58
95
|
// Invoke browser task via socket
|
|
59
96
|
const result = await invokeBrowserTask({
|
|
60
97
|
sessionId,
|
|
@@ -65,6 +102,7 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
|
|
|
65
102
|
browserTaskMode,
|
|
66
103
|
toolSchemas,
|
|
67
104
|
outputSchema,
|
|
105
|
+
screenshotConfig,
|
|
68
106
|
});
|
|
69
107
|
|
|
70
108
|
logger.debug({
|