@empiricalrun/test-gen 0.79.4 → 0.79.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/dist/agent/browsing/run.d.ts +1 -1
  3. package/dist/agent/browsing/run.d.ts.map +1 -1
  4. package/dist/agent/browsing/utils.d.ts +0 -7
  5. package/dist/agent/browsing/utils.d.ts.map +1 -1
  6. package/dist/agent/browsing/utils.js +1 -56
  7. package/dist/agent/master/run.d.ts +1 -1
  8. package/dist/agent/master/run.d.ts.map +1 -1
  9. package/dist/agent/master/run.js +1 -1
  10. package/dist/dashboard/client.d.ts +2 -45
  11. package/dist/dashboard/client.d.ts.map +1 -1
  12. package/dist/dashboard/client.js +4 -217
  13. package/dist/dashboard/index.d.ts +2 -4
  14. package/dist/dashboard/index.d.ts.map +1 -1
  15. package/dist/dashboard/index.js +10 -13
  16. package/dist/dashboard/tool-response-from-sandbox.d.ts +1 -4
  17. package/dist/dashboard/tool-response-from-sandbox.d.ts.map +1 -1
  18. package/dist/dashboard/tool-response-from-sandbox.js +4 -38
  19. package/dist/dashboard/tool-response.d.ts +1 -5
  20. package/dist/dashboard/tool-response.d.ts.map +1 -1
  21. package/dist/dashboard/tool-response.js +5 -87
  22. package/dist/dashboard/types.d.ts +1 -8
  23. package/dist/dashboard/types.d.ts.map +1 -1
  24. package/dist/dashboard/types.js +4 -14
  25. package/dist/file/server.d.ts +1 -1
  26. package/dist/file/server.d.ts.map +1 -1
  27. package/dist/index.d.ts.map +1 -1
  28. package/dist/index.js +3 -3
  29. package/dist/recorder/index.d.ts.map +1 -1
  30. package/dist/recorder/index.js +2 -2
  31. package/dist/tools/test-gen-browser.d.ts.map +1 -1
  32. package/dist/tools/test-gen-browser.js +3 -3
  33. package/package.json +3 -1
  34. package/tsconfig.tsbuildinfo +1 -1
  35. package/dist/agent/cua/computer.d.ts +0 -14
  36. package/dist/agent/cua/computer.d.ts.map +0 -1
  37. package/dist/agent/cua/computer.js +0 -191
  38. package/dist/agent/cua/index.d.ts +0 -22
  39. package/dist/agent/cua/index.d.ts.map +0 -1
  40. package/dist/agent/cua/index.js +0 -254
  41. package/dist/agent/cua/model.d.ts +0 -25
  42. package/dist/agent/cua/model.d.ts.map +0 -1
  43. package/dist/agent/cua/model.js +0 -66
  44. package/dist/agent/cua/pw-codegen/element-from-point.d.ts +0 -10
  45. package/dist/agent/cua/pw-codegen/element-from-point.d.ts.map +0 -1
  46. package/dist/agent/cua/pw-codegen/element-from-point.js +0 -135
  47. package/dist/agent/cua/pw-codegen/factory.d.ts +0 -15
  48. package/dist/agent/cua/pw-codegen/factory.d.ts.map +0 -1
  49. package/dist/agent/cua/pw-codegen/factory.js +0 -59
  50. package/dist/agent/cua/pw-codegen/index.d.ts +0 -7
  51. package/dist/agent/cua/pw-codegen/index.d.ts.map +0 -1
  52. package/dist/agent/cua/pw-codegen/index.js +0 -14
  53. package/dist/agent/cua/pw-codegen/pw-event-sink/index.d.ts +0 -43
  54. package/dist/agent/cua/pw-codegen/pw-event-sink/index.d.ts.map +0 -1
  55. package/dist/agent/cua/pw-codegen/pw-event-sink/index.js +0 -250
  56. package/dist/agent/cua/pw-codegen/pw-pause/for-recorder.d.ts +0 -14
  57. package/dist/agent/cua/pw-codegen/pw-pause/for-recorder.d.ts.map +0 -1
  58. package/dist/agent/cua/pw-codegen/pw-pause/for-recorder.js +0 -88
  59. package/dist/agent/cua/pw-codegen/pw-pause/index.d.ts +0 -17
  60. package/dist/agent/cua/pw-codegen/pw-pause/index.d.ts.map +0 -1
  61. package/dist/agent/cua/pw-codegen/pw-pause/index.js +0 -117
  62. package/dist/agent/cua/pw-codegen/pw-pause/ipc.d.ts +0 -3
  63. package/dist/agent/cua/pw-codegen/pw-pause/ipc.d.ts.map +0 -1
  64. package/dist/agent/cua/pw-codegen/pw-pause/ipc.js +0 -13
  65. package/dist/agent/cua/pw-codegen/pw-pause/patch.d.ts +0 -24
  66. package/dist/agent/cua/pw-codegen/pw-pause/patch.d.ts.map +0 -1
  67. package/dist/agent/cua/pw-codegen/pw-pause/patch.js +0 -197
  68. package/dist/agent/cua/pw-codegen/pw-pause/types.d.ts +0 -14
  69. package/dist/agent/cua/pw-codegen/pw-pause/types.d.ts.map +0 -1
  70. package/dist/agent/cua/pw-codegen/pw-pause/types.js +0 -2
  71. package/dist/agent/cua/pw-codegen/pw-pause/utils.d.ts +0 -2
  72. package/dist/agent/cua/pw-codegen/pw-pause/utils.d.ts.map +0 -1
  73. package/dist/agent/cua/pw-codegen/pw-pause/utils.js +0 -6
  74. package/dist/agent/cua/pw-codegen/types.d.ts +0 -47
  75. package/dist/agent/cua/pw-codegen/types.d.ts.map +0 -1
  76. package/dist/agent/cua/pw-codegen/types.js +0 -2
  77. package/dist/agent/cua/pw-codegen/version.d.ts +0 -25
  78. package/dist/agent/cua/pw-codegen/version.d.ts.map +0 -1
  79. package/dist/agent/cua/pw-codegen/version.js +0 -86
@@ -1,89 +1,7 @@
1
1
  "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
2
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.getDefaultMachineId = getDefaultMachineId;
7
- exports.sendToolResponse = sendToolResponse;
8
- exports.sendErrorToolResponse = sendErrorToolResponse;
9
- const os_1 = __importDefault(require("os"));
10
- const types_1 = require("./types");
11
- function getDefaultMachineId() {
12
- return os_1.default.hostname();
13
- }
14
- async function sendToolResponse(apiClient, chatSessionId, requestId, toolCalls, toolResults, checkpoint, logger, machineId) {
15
- const resolvedMachineId = machineId ?? getDefaultMachineId();
16
- try {
17
- if (chatSessionId) {
18
- await sendToolResponseForChatSession(apiClient, chatSessionId, toolCalls, toolResults, checkpoint, resolvedMachineId, logger);
19
- }
20
- else {
21
- await sendToolResponseForAdhocToolCall(apiClient, requestId, toolCalls, toolResults, resolvedMachineId, logger);
22
- }
23
- return true;
24
- }
25
- catch (error) {
26
- // Check if it's a 409 Conflict (tool calls were rejected by user)
27
- if (error instanceof types_1.NonRetryableHTTPError && error.status === 409) {
28
- logger.info("Tool calls were rejected by user (409 Conflict), skipping git push");
29
- return false;
30
- }
31
- // For other errors, re-throw
32
- logger.error("Failed to send tool response", error);
33
- throw error;
34
- }
35
- }
36
- async function sendErrorToolResponse(apiClient, chatSessionId, requestId, toolCalls, errorMessage, logger, machineId) {
37
- const resolvedMachineId = machineId ?? getDefaultMachineId();
38
- const toolResults = toolCalls.map(() => ({
39
- isError: true,
40
- result: errorMessage,
41
- }));
42
- if (chatSessionId) {
43
- return sendToolResponseForChatSession(apiClient, chatSessionId, toolCalls, toolResults, null, resolvedMachineId, logger);
44
- }
45
- else {
46
- return sendToolResponseForAdhocToolCall(apiClient, requestId, toolCalls, toolResults, resolvedMachineId, logger);
47
- }
48
- }
49
- async function sendToolResponseForChatSession(apiClient, chatSessionId, toolCalls, toolResults, checkpoint, machineId, logger) {
50
- const url = `/api/chat-sessions/${chatSessionId}/tool-response`;
51
- try {
52
- await apiClient.request(url, {
53
- method: "POST",
54
- body: {
55
- toolResults,
56
- toolCalls,
57
- checkpoint,
58
- machineId,
59
- },
60
- });
61
- }
62
- catch (error) {
63
- logger.error("Failed to send tool response for chat session", {
64
- chatSessionId,
65
- error,
66
- });
67
- throw error;
68
- }
69
- }
70
- async function sendToolResponseForAdhocToolCall(apiClient, requestId, toolCalls, toolResults, machineId, logger) {
71
- const url = `/api/tool-calls/${requestId}/response`;
72
- try {
73
- await apiClient.request(url, {
74
- method: "POST",
75
- body: {
76
- tool_results: toolResults,
77
- tool_calls: toolCalls,
78
- machine_id: machineId,
79
- },
80
- });
81
- }
82
- catch (error) {
83
- logger.error("Failed to send tool response for adhoc tool call", {
84
- requestId,
85
- error,
86
- });
87
- throw error;
88
- }
89
- }
3
+ exports.sendToolResponse = exports.sendErrorToolResponse = exports.getDefaultMachineId = void 0;
4
+ var dashboard_client_1 = require("@empiricalrun/dashboard-client");
5
+ Object.defineProperty(exports, "getDefaultMachineId", { enumerable: true, get: function () { return dashboard_client_1.getDefaultMachineId; } });
6
+ Object.defineProperty(exports, "sendErrorToolResponse", { enumerable: true, get: function () { return dashboard_client_1.sendErrorToolResponse; } });
7
+ Object.defineProperty(exports, "sendToolResponse", { enumerable: true, get: function () { return dashboard_client_1.sendToolResponse; } });
@@ -1,9 +1,2 @@
1
- export declare class HTTPError extends Error {
2
- status: number;
3
- constructor(message: string, status: number);
4
- }
5
- export declare class NonRetryableHTTPError extends HTTPError {
6
- }
7
- export declare class RetryableHTTPError extends HTTPError {
8
- }
1
+ export { HTTPError, NonRetryableHTTPError, RetryableHTTPError, } from "@empiricalrun/dashboard-client";
9
2
  //# sourceMappingURL=types.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/dashboard/types.ts"],"names":[],"mappings":"AAAA,qBAAa,SAAU,SAAQ,KAAK;IAClC,MAAM,EAAE,MAAM,CAAC;gBACH,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CAI5C;AAED,qBAAa,qBAAsB,SAAQ,SAAS;CAAG;AAEvD,qBAAa,kBAAmB,SAAQ,SAAS;CAAG"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/dashboard/types.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,gCAAgC,CAAC"}
@@ -1,17 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.RetryableHTTPError = exports.NonRetryableHTTPError = exports.HTTPError = void 0;
4
- class HTTPError extends Error {
5
- status;
6
- constructor(message, status) {
7
- super(message);
8
- this.status = status;
9
- }
10
- }
11
- exports.HTTPError = HTTPError;
12
- class NonRetryableHTTPError extends HTTPError {
13
- }
14
- exports.NonRetryableHTTPError = NonRetryableHTTPError;
15
- class RetryableHTTPError extends HTTPError {
16
- }
17
- exports.RetryableHTTPError = RetryableHTTPError;
4
+ var dashboard_client_1 = require("@empiricalrun/dashboard-client");
5
+ Object.defineProperty(exports, "HTTPError", { enumerable: true, get: function () { return dashboard_client_1.HTTPError; } });
6
+ Object.defineProperty(exports, "NonRetryableHTTPError", { enumerable: true, get: function () { return dashboard_client_1.NonRetryableHTTPError; } });
7
+ Object.defineProperty(exports, "RetryableHTTPError", { enumerable: true, get: function () { return dashboard_client_1.RetryableHTTPError; } });
@@ -1,5 +1,5 @@
1
+ import { BrowserAgentResult } from "@empiricalrun/cua";
1
2
  import { Usage } from "@empiricalrun/shared-types/chat-agent";
2
- import { BrowserAgentResult } from "../agent/cua";
3
3
  export type BrowserAgentIPCPayload = {
4
4
  task: string;
5
5
  result: BrowserAgentResult;
@@ -1 +1 @@
1
- {"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,uCAAuC,CAAC;AAK9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AASlD,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,CAAC;AAEF,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,MAAM,CAA4C;IAC1D,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,UAAU,CAAC,CAAa;IAEhC,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,KAAK,CAAoB;IAEjC,OAAO,CAAC,cAAc,CAAqB;gBAE/B,EACV,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GACX,EAAE;QACD,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,OAAO,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;KACzB;IAOD,iBAAiB;;;;IAIjB,iBAAiB;IAIjB,WAAW,CAAC,QAAQ,EAAE,MAAM;IAItB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;IAsHnC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB5B"}
1
+ {"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/file/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,KAAK,EAAE,MAAM,uCAAuC,CAAC;AAW9D,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,CAAC;AAEF,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,MAAM,CAA4C;IAC1D,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,UAAU,CAAC,CAAa;IAEhC,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,KAAK,CAAoB;IAEjC,OAAO,CAAC,cAAc,CAAqB;gBAE/B,EACV,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GACX,EAAE;QACD,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,OAAO,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;KACzB;IAOD,iBAAiB;;;;IAIjB,iBAAiB;IAIjB,WAAW,CAAC,QAAQ,EAAE,MAAM;IAItB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;IAsHnC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB5B"}
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAYhD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEpC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EACL,yBAAyB,EACzB,iBAAiB,GAClB,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAsBjF,wBAAsB,UAAU,CAC9B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,IAAI,GAAG,YAAY,EAC5B,KAAK,CAAC,EAAE,SAAS,iBA0DlB;AAED,wBAAsB,UAAU,CAAC,OAAO,EAAE,IAAI,iBAY7C"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAKhD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEpC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EACL,yBAAyB,EACzB,iBAAiB,GAClB,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAsBjF,wBAAsB,UAAU,CAC9B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,IAAI,GAAG,YAAY,EAC5B,KAAK,CAAC,EAAE,SAAS,iBA0DlB;AAED,wBAAsB,UAAU,CAAC,OAAO,EAAE,IAAI,iBAY7C"}
package/dist/index.js CHANGED
@@ -6,10 +6,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.generateNetworkTrace = exports.getFilenamesInZip = exports.extractFileFromZipFromUrl = exports.downloadBuild = void 0;
7
7
  exports.createTest = createTest;
8
8
  exports.recordTest = recordTest;
9
+ const cua_1 = require("@empiricalrun/cua");
10
+ const pw_pause_1 = require("@empiricalrun/cua/pw-codegen/pw-pause");
11
+ const for_recorder_1 = require("@empiricalrun/cua/pw-codegen/pw-pause/for-recorder");
9
12
  const llm_1 = require("@empiricalrun/llm");
10
- const cua_1 = require("./agent/cua");
11
- const pw_pause_1 = require("./agent/cua/pw-codegen/pw-pause");
12
- const for_recorder_1 = require("./agent/cua/pw-codegen/pw-pause/for-recorder");
13
13
  const run_1 = require("./agent/master/run");
14
14
  const scenarios_1 = require("./bin/utils/scenarios");
15
15
  const client_1 = __importDefault(require("./file/client"));
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/recorder/index.ts"],"names":[],"mappings":"AAwEA,UAAU,kBAAkB;IAC1B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;CACzB;AA4TD,wBAAsB,WAAW,CAAC,EAChC,QAAQ,EACR,OAAO,EACP,GAAG,GACJ,EAAE,kBAAkB,iBAGpB"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/recorder/index.ts"],"names":[],"mappings":"AAuEA,UAAU,kBAAkB;IAC1B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;CACzB;AA4TD,wBAAsB,WAAW,CAAC,EAChC,QAAQ,EACR,OAAO,EACP,GAAG,GACJ,EAAE,kBAAkB,iBAGpB"}
@@ -5,14 +5,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.runRecorder = runRecorder;
7
7
  const ast_parser_1 = require("@empiricalrun/ast-parser");
8
+ const pw_codegen_1 = require("@empiricalrun/cua/pw-codegen");
9
+ const pw_pause_1 = require("@empiricalrun/cua/pw-codegen/pw-pause");
8
10
  const test_run_1 = require("@empiricalrun/test-run");
9
11
  const detect_port_1 = __importDefault(require("detect-port"));
10
12
  const fs_1 = __importDefault(require("fs"));
11
13
  const inquirer_1 = __importDefault(require("inquirer"));
12
14
  const path_1 = __importDefault(require("path"));
13
15
  const utils_1 = require("../agent/browsing/utils");
14
- const pw_codegen_1 = require("../agent/cua/pw-codegen");
15
- const pw_pause_1 = require("../agent/cua/pw-codegen/pw-pause");
16
16
  const client_1 = require("../dashboard/client");
17
17
  const server_1 = require("../file/server");
18
18
  const logger_1 = require("../logger");
@@ -1 +1 @@
1
- {"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAGV,IAAI,EACL,MAAM,uCAAuC,CAAC;AA0B/C,eAAO,MAAM,4BAA4B,EAAE,IAsP1C,CAAC"}
1
+ {"version":3,"file":"test-gen-browser.d.ts","sourceRoot":"","sources":["../../src/tools/test-gen-browser.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAGV,IAAI,EACL,MAAM,uCAAuC,CAAC;AAkB/C,eAAO,MAAM,4BAA4B,EAAE,IAsP1C,CAAC"}
@@ -4,14 +4,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.generateTestWithBrowserAgent = void 0;
7
+ const model_1 = require("@empiricalrun/cua/model");
8
+ const pw_codegen_1 = require("@empiricalrun/cua/pw-codegen");
9
+ const pw_pause_1 = require("@empiricalrun/cua/pw-codegen/pw-pause");
7
10
  const test_run_1 = require("@empiricalrun/test-run");
8
11
  const fs_1 = __importDefault(require("fs"));
9
12
  const path_1 = __importDefault(require("path"));
10
13
  const run_1 = require("../agent/browsing/run");
11
14
  const utils_1 = require("../agent/browsing/utils");
12
- const model_1 = require("../agent/cua/model");
13
- const pw_codegen_1 = require("../agent/cua/pw-codegen");
14
- const pw_pause_1 = require("../agent/cua/pw-codegen/pw-pause");
15
15
  const utils_2 = require("../artifacts/utils");
16
16
  const web_1 = require("../bin/utils/platform/web");
17
17
  const scenarios_1 = require("../bin/utils/scenarios");
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@empiricalrun/test-gen",
3
- "version": "0.79.4",
3
+ "version": "0.79.5",
4
4
  "publishConfig": {
5
5
  "registry": "https://registry.npmjs.org/",
6
6
  "access": "public"
@@ -92,6 +92,8 @@
92
92
  "typescript": "^5.3.3",
93
93
  "zod": "^4.0.1",
94
94
  "@empiricalrun/ast-parser": "^0.0.10",
95
+ "@empiricalrun/cua": "^0.2.0",
96
+ "@empiricalrun/dashboard-client": "^0.2.0",
95
97
  "@empiricalrun/llm": "^0.25.2",
96
98
  "@empiricalrun/r2-uploader": "^0.9.1",
97
99
  "@empiricalrun/trace-utils": "^0.1.0",
@@ -1 +1 @@
1
- {"root":["./src/index.ts","./src/logger.ts","./src/actions/assert.ts","./src/actions/click.ts","./src/actions/done.ts","./src/actions/fill.ts","./src/actions/goto.ts","./src/actions/hover.ts","./src/actions/index.ts","./src/actions/next-task.ts","./src/actions/press.ts","./src/actions/text-content.ts","./src/actions/constants/index.ts","./src/actions/utils/index.ts","./src/agent/cli.ts","./src/agent/index.ts","./src/agent/base/index.ts","./src/agent/browsing/index.ts","./src/agent/browsing/run.ts","./src/agent/browsing/utils.ts","./src/agent/chat/agent-loop.ts","./src/agent/chat/exports.ts","./src/agent/chat/index.ts","./src/agent/chat/models.ts","./src/agent/chat/state.ts","./src/agent/chat/types.ts","./src/agent/chat/utils.ts","./src/agent/chat/prompt/pw-utils-docs.ts","./src/agent/chat/prompt/repo.ts","./src/agent/chat/prompt/test-case-def.ts","./src/agent/chat/prompt/trace-utils-docs.ts","./src/agent/code-review/index.ts","./src/agent/code-review/types.ts","./src/agent/code-review/xml-parser.ts","./src/agent/code-review/executor/index.ts","./src/agent/cua/computer.ts","./src/agent/cua/index.ts","./src/agent/cua/model.ts","./src/agent/cua/pw-codegen/element-from-point.ts","./src/agent/cua/pw-codegen/factory.ts","./src/agent/cua/pw-codegen/index.ts","./src/agent/cua/pw-codegen/types.ts","./src/agent/cua/pw-codegen/version.ts","./src/agent/cua/pw-codegen/pw-event-sink/index.ts","./src/agent/cua/pw-codegen/pw-pause/for-recorder.ts","./src/agent/cua/pw-codegen/pw-pause/index.ts","./src/agent/cua/pw-codegen/pw-pause/ipc.ts","./src/agent/cua/pw-codegen/pw-pause/patch.ts","./src/agent/cua/pw-codegen/pw-pause/types.ts","./src/agent/cua/pw-codegen/pw-pause/utils.ts","./src/agent/fast-triage/index.ts","./src/agent/master/action-tool-calls.ts","./src/agent/master/element-annotation.ts","./src/agent/master/execute-browser-action.ts","./src/agent/master/next-action.ts","./src/agent/master/planner.ts","./src/agent/master/run.ts","./src/agent/master/scroller.ts","./src/agent/master/with-hints.ts","./src/agent/master/browser-tests/cua.spec.ts","./src/agent/master/browser-tests/fixtures.ts","./src/agent/master/browser-tests/index.spec.ts","./src/agent/planner/run-time-planner.ts","./src/agent/planner/run.ts","./src/agent/triage/index.ts","./src/artifacts/index.ts","./src/artifacts/utils.ts","./src/auth/cli-auth.ts","./src/auth/index.ts","./src/auth/token-store.ts","./src/bin/environments.ts","./src/bin/index.ts","./src/bin/setup.ts","./src/bin/logger/index.ts","./src/bin/utils/context.ts","./src/bin/utils/index.ts","./src/bin/utils/fs/index.ts","./src/bin/utils/platform/web/index.ts","./src/bin/utils/platform/web/test-files/ts-path-import-validate.ts","./src/bin/utils/scenarios/index.ts","./src/browser-injected-scripts/annotate-elements.spec.ts","./src/constants/index.ts","./src/dashboard/client.ts","./src/dashboard/index.ts","./src/dashboard/tool-response-from-sandbox.ts","./src/dashboard/tool-response.ts","./src/dashboard/totp.ts","./src/dashboard/types.ts","./src/errors/index.ts","./src/file/client.ts","./src/file/server.ts","./src/file-info/index.ts","./src/file-info/adapters/file-system/index.ts","./src/file-info/adapters/file-system/reader.ts","./src/file-info/adapters/github/index.ts","./src/file-info/adapters/github/reader.ts","./src/generate-summary/frame-sampling.ts","./src/generate-summary/generate-error-stack-summary.ts","./src/generate-summary/generate-failed-step-screenshot-diff-summary.ts","./src/generate-summary/generate-grouped-summary.ts","./src/generate-summary/merge-summary.ts","./src/generate-summary/pick-videos-for-comparison.ts","./src/human-in-the-loop/cli.ts","./src/human-in-the-loop/index.ts","./src/human-in-the-loop/ipc.ts","./src/page/index.ts","./src/prompts/lib/ts-transformer.ts","./src/recorder/env-variables.ts","./src/recorder/index.ts","./src/recorder/request.ts","./src/recorder/temp-files.ts","./src/recorder/upload.ts","./src/recorder/validation.ts","./src/telemetry/index.ts","./src/test-build/index.ts","./src/tools/diagnosis-fetcher.ts","./src/tools/index.ts","./src/tools/list-environments.ts","./src/tools/run-test.ts","./src/tools/test-gen-browser.ts","./src/tools/analyse-video/index.ts","./src/tools/create-pull-request/index.ts","./src/tools/create-pull-request/utils.ts","./src/tools/definitions/analyse-video.ts","./src/tools/definitions/delete-file.ts","./src/tools/definitions/download-build.ts","./src/tools/definitions/grep.ts","./src/tools/definitions/list-tests-and-projects.ts","./src/tools/definitions/merge-conflicts.ts","./src/tools/definitions/publish-markdown-report.ts","./src/tools/definitions/rename-file.ts","./src/tools/definitions/run-test.ts","./src/tools/definitions/safe-bash.ts","./src/tools/definitions/str_replace_editor.ts","./src/tools/definitions/test-gen-browser.ts","./src/tools/definitions/trace-dot-zip.ts","./src/tools/definitions/upgrade-packages.ts","./src/tools/definitions/utils.ts","./src/tools/delete-file/index.ts","./src/tools/download-build/index.ts","./src/tools/executor/base.ts","./src/tools/executor/index.ts","./src/tools/executor/types.ts","./src/tools/executor/utils/checkpoint.ts","./src/tools/executor/utils/git.ts","./src/tools/executor/utils/index.ts","./src/tools/executor/utils/pr-description.ts","./src/tools/fetch-file/index.ts","./src/tools/fetch-file/utils.ts","./src/tools/fetch-session-diff/index.ts","./src/tools/file-operations/create.ts","./src/tools/file-operations/index.ts","./src/tools/file-operations/insert.ts","./src/tools/file-operations/replace.ts","./src/tools/file-operations/shared/helpers.ts","./src/tools/file-operations/view/index.ts","./src/tools/firecrawl-scrape/index.ts","./src/tools/grep/index.ts","./src/tools/grep/types.ts","./src/tools/grep/ripgrep/index.ts","./src/tools/issues-v1/index.ts","./src/tools/issues-v1/list-issues.ts","./src/tools/issues-v1/utils.ts","./src/tools/issues-v2/create-issue.ts","./src/tools/issues-v2/set-issue-description.ts","./src/tools/issues-v2/update-issue.ts","./src/tools/list-tests-and-projects/index.ts","./src/tools/merge-conflicts/index.ts","./src/tools/publish-markdown-report/index.ts","./src/tools/rename-file/index.ts","./src/tools/safe-bash/index.ts","./src/tools/test-run-fetcher/index.ts","./src/tools/test-run-fetcher/types.ts","./src/tools/trace-dot-zip/index.ts","./src/tools/trace-dot-zip/types.ts","./src/tools/trace-dot-zip/utils/console-trace.ts","./src/tools/trace-dot-zip/utils/extract-screenshots.ts","./src/tools/trace-dot-zip/utils/extract-steps.ts","./src/tools/trace-dot-zip/utils/extract-zip.ts","./src/tools/trace-dot-zip/utils/network-trace.ts","./src/tools/triage-summary/index.ts","./src/tools/triage-summary/types.ts","./src/tools/triage-summary/utils.ts","./src/tools/upgrade-packages/index.ts","./src/tools/upgrade-packages/utils.ts","./src/tools/utils/queue.ts","./src/tools/utils/validate-schema.ts","./src/types/handlebars.d.ts","./src/types/index.ts","./src/utils/SQSClient.ts","./src/utils/env.ts","./src/utils/exec.ts","./src/utils/file.ts","./src/utils/hash.ts","./src/utils/html.ts","./src/utils/index.ts","./src/utils/json.ts","./src/utils/model.ts","./src/utils/playwright-test-id.ts","./src/utils/repo-tree.ts","./src/utils/slug.ts","./src/utils/string.ts","./src/utils/stripAnsi.ts","./src/utils/url-validation.ts","./src/utils/dedup/dedup-image.ts","./src/utils/dedup/find-threshold.ts","./src/video-core/index.ts","./src/video-core/model-limits.ts","./src/video-core/storage-manager.ts","./src/video-core/types.ts","./src/video-core/utils.ts"],"version":"5.8.3"}
1
+ {"root":["./src/index.ts","./src/logger.ts","./src/actions/assert.ts","./src/actions/click.ts","./src/actions/done.ts","./src/actions/fill.ts","./src/actions/goto.ts","./src/actions/hover.ts","./src/actions/index.ts","./src/actions/next-task.ts","./src/actions/press.ts","./src/actions/text-content.ts","./src/actions/constants/index.ts","./src/actions/utils/index.ts","./src/agent/cli.ts","./src/agent/index.ts","./src/agent/base/index.ts","./src/agent/browsing/index.ts","./src/agent/browsing/run.ts","./src/agent/browsing/utils.ts","./src/agent/chat/agent-loop.ts","./src/agent/chat/exports.ts","./src/agent/chat/index.ts","./src/agent/chat/models.ts","./src/agent/chat/state.ts","./src/agent/chat/types.ts","./src/agent/chat/utils.ts","./src/agent/chat/prompt/pw-utils-docs.ts","./src/agent/chat/prompt/repo.ts","./src/agent/chat/prompt/test-case-def.ts","./src/agent/chat/prompt/trace-utils-docs.ts","./src/agent/code-review/index.ts","./src/agent/code-review/types.ts","./src/agent/code-review/xml-parser.ts","./src/agent/code-review/executor/index.ts","./src/agent/fast-triage/index.ts","./src/agent/master/action-tool-calls.ts","./src/agent/master/element-annotation.ts","./src/agent/master/execute-browser-action.ts","./src/agent/master/next-action.ts","./src/agent/master/planner.ts","./src/agent/master/run.ts","./src/agent/master/scroller.ts","./src/agent/master/with-hints.ts","./src/agent/master/browser-tests/cua.spec.ts","./src/agent/master/browser-tests/fixtures.ts","./src/agent/master/browser-tests/index.spec.ts","./src/agent/planner/run-time-planner.ts","./src/agent/planner/run.ts","./src/agent/triage/index.ts","./src/artifacts/index.ts","./src/artifacts/utils.ts","./src/auth/cli-auth.ts","./src/auth/index.ts","./src/auth/token-store.ts","./src/bin/environments.ts","./src/bin/index.ts","./src/bin/setup.ts","./src/bin/logger/index.ts","./src/bin/utils/context.ts","./src/bin/utils/index.ts","./src/bin/utils/fs/index.ts","./src/bin/utils/platform/web/index.ts","./src/bin/utils/platform/web/test-files/ts-path-import-validate.ts","./src/bin/utils/scenarios/index.ts","./src/browser-injected-scripts/annotate-elements.spec.ts","./src/constants/index.ts","./src/dashboard/client.ts","./src/dashboard/index.ts","./src/dashboard/tool-response-from-sandbox.ts","./src/dashboard/tool-response.ts","./src/dashboard/totp.ts","./src/dashboard/types.ts","./src/errors/index.ts","./src/file/client.ts","./src/file/server.ts","./src/file-info/index.ts","./src/file-info/adapters/file-system/index.ts","./src/file-info/adapters/file-system/reader.ts","./src/file-info/adapters/github/index.ts","./src/file-info/adapters/github/reader.ts","./src/generate-summary/frame-sampling.ts","./src/generate-summary/generate-error-stack-summary.ts","./src/generate-summary/generate-failed-step-screenshot-diff-summary.ts","./src/generate-summary/generate-grouped-summary.ts","./src/generate-summary/merge-summary.ts","./src/generate-summary/pick-videos-for-comparison.ts","./src/human-in-the-loop/cli.ts","./src/human-in-the-loop/index.ts","./src/human-in-the-loop/ipc.ts","./src/page/index.ts","./src/prompts/lib/ts-transformer.ts","./src/recorder/env-variables.ts","./src/recorder/index.ts","./src/recorder/request.ts","./src/recorder/temp-files.ts","./src/recorder/upload.ts","./src/recorder/validation.ts","./src/telemetry/index.ts","./src/test-build/index.ts","./src/tools/diagnosis-fetcher.ts","./src/tools/index.ts","./src/tools/list-environments.ts","./src/tools/run-test.ts","./src/tools/test-gen-browser.ts","./src/tools/analyse-video/index.ts","./src/tools/create-pull-request/index.ts","./src/tools/create-pull-request/utils.ts","./src/tools/definitions/analyse-video.ts","./src/tools/definitions/delete-file.ts","./src/tools/definitions/download-build.ts","./src/tools/definitions/grep.ts","./src/tools/definitions/list-tests-and-projects.ts","./src/tools/definitions/merge-conflicts.ts","./src/tools/definitions/publish-markdown-report.ts","./src/tools/definitions/rename-file.ts","./src/tools/definitions/run-test.ts","./src/tools/definitions/safe-bash.ts","./src/tools/definitions/str_replace_editor.ts","./src/tools/definitions/test-gen-browser.ts","./src/tools/definitions/trace-dot-zip.ts","./src/tools/definitions/upgrade-packages.ts","./src/tools/definitions/utils.ts","./src/tools/delete-file/index.ts","./src/tools/download-build/index.ts","./src/tools/executor/base.ts","./src/tools/executor/index.ts","./src/tools/executor/types.ts","./src/tools/executor/utils/checkpoint.ts","./src/tools/executor/utils/git.ts","./src/tools/executor/utils/index.ts","./src/tools/executor/utils/pr-description.ts","./src/tools/fetch-file/index.ts","./src/tools/fetch-file/utils.ts","./src/tools/fetch-session-diff/index.ts","./src/tools/file-operations/create.ts","./src/tools/file-operations/index.ts","./src/tools/file-operations/insert.ts","./src/tools/file-operations/replace.ts","./src/tools/file-operations/shared/helpers.ts","./src/tools/file-operations/view/index.ts","./src/tools/firecrawl-scrape/index.ts","./src/tools/grep/index.ts","./src/tools/grep/types.ts","./src/tools/grep/ripgrep/index.ts","./src/tools/issues-v1/index.ts","./src/tools/issues-v1/list-issues.ts","./src/tools/issues-v1/utils.ts","./src/tools/issues-v2/create-issue.ts","./src/tools/issues-v2/set-issue-description.ts","./src/tools/issues-v2/update-issue.ts","./src/tools/list-tests-and-projects/index.ts","./src/tools/merge-conflicts/index.ts","./src/tools/publish-markdown-report/index.ts","./src/tools/rename-file/index.ts","./src/tools/safe-bash/index.ts","./src/tools/test-run-fetcher/index.ts","./src/tools/test-run-fetcher/types.ts","./src/tools/trace-dot-zip/index.ts","./src/tools/trace-dot-zip/types.ts","./src/tools/trace-dot-zip/utils/console-trace.ts","./src/tools/trace-dot-zip/utils/extract-screenshots.ts","./src/tools/trace-dot-zip/utils/extract-steps.ts","./src/tools/trace-dot-zip/utils/extract-zip.ts","./src/tools/trace-dot-zip/utils/network-trace.ts","./src/tools/triage-summary/index.ts","./src/tools/triage-summary/types.ts","./src/tools/triage-summary/utils.ts","./src/tools/upgrade-packages/index.ts","./src/tools/upgrade-packages/utils.ts","./src/tools/utils/queue.ts","./src/tools/utils/validate-schema.ts","./src/types/handlebars.d.ts","./src/types/index.ts","./src/utils/SQSClient.ts","./src/utils/env.ts","./src/utils/exec.ts","./src/utils/file.ts","./src/utils/hash.ts","./src/utils/html.ts","./src/utils/index.ts","./src/utils/json.ts","./src/utils/model.ts","./src/utils/playwright-test-id.ts","./src/utils/repo-tree.ts","./src/utils/slug.ts","./src/utils/string.ts","./src/utils/stripAnsi.ts","./src/utils/url-validation.ts","./src/utils/dedup/dedup-image.ts","./src/utils/dedup/find-threshold.ts","./src/video-core/index.ts","./src/video-core/model-limits.ts","./src/video-core/storage-manager.ts","./src/video-core/types.ts","./src/video-core/utils.ts"],"version":"5.8.3"}
@@ -1,14 +0,0 @@
1
- import { ResponseComputerToolCall } from "openai/resources/responses/responses.mjs";
2
- import type { Page } from "playwright";
3
- import { BasePlaywrightCodegen } from "./pw-codegen/types";
4
- type OpenAIComputerAction = ResponseComputerToolCall.Click | ResponseComputerToolCall.DoubleClick | ResponseComputerToolCall.Drag | ResponseComputerToolCall.Keypress | ResponseComputerToolCall.Move | ResponseComputerToolCall.Screenshot | ResponseComputerToolCall.Scroll | ResponseComputerToolCall.Type | ResponseComputerToolCall.Wait;
5
- export declare function getScreenshot(page: Page): Promise<string>;
6
- export declare function executeModelAction(page: Page, action: OpenAIComputerAction | {
7
- type: "goto";
8
- url: string;
9
- }, codegen: BasePlaywrightCodegen): Promise<{
10
- actionSummary: string;
11
- actionCode: string;
12
- }>;
13
- export {};
14
- //# sourceMappingURL=computer.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AACpF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,KAAK,oBAAoB,GACrB,wBAAwB,CAAC,KAAK,GAC9B,wBAAwB,CAAC,WAAW,GACpC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,QAAQ,GACjC,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,UAAU,GACnC,wBAAwB,CAAC,MAAM,GAC/B,wBAAwB,CAAC,IAAI,GAC7B,wBAAwB,CAAC,IAAI,CAAC;AAElC,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,mBAO7C;AAgCD,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,oBAAoB,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EAC5D,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC;IACT,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAoKD"}
@@ -1,191 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.getScreenshot = getScreenshot;
4
- exports.executeModelAction = executeModelAction;
5
- async function getScreenshot(page) {
6
- // We override the `scale` option to "css" to ensure the screenshot is
7
- // consistent with the viewport size. Without this, the screenshot dimensions
8
- // are viewport size * device pixel ratio (= 3 on iPhone device emulation).
9
- // Docs: https://playwright.dev/docs/api/class-page#page-screenshot-option-scale
10
- const screenshotBytes = await page.screenshot({ scale: "css" });
11
- return Buffer.from(screenshotBytes).toString("base64");
12
- }
13
- const CUA_KEY_TO_PLAYWRIGHT_KEY = {
14
- "/": "Divide",
15
- "\\": "Backslash",
16
- alt: "Alt",
17
- arrowdown: "ArrowDown",
18
- arrowleft: "ArrowLeft",
19
- arrowright: "ArrowRight",
20
- arrowup: "ArrowUp",
21
- backspace: "Backspace",
22
- capslock: "CapsLock",
23
- // "cmd" and "ctrl" are both mapped to "ControlOrMeta" for platform
24
- // agnostic behavior, as opposed to cmd: "Meta" and ctrl: "Control"
25
- cmd: "ControlOrMeta",
26
- ctrl: "ControlOrMeta",
27
- delete: "Delete",
28
- end: "End",
29
- enter: "Enter",
30
- esc: "Escape",
31
- home: "Home",
32
- insert: "Insert",
33
- option: "Alt",
34
- pagedown: "PageDown",
35
- pageup: "PageUp",
36
- shift: "Shift",
37
- space: " ",
38
- super: "Meta",
39
- tab: "Tab",
40
- win: "Meta",
41
- };
42
- async function executeModelAction(page, action, codegen) {
43
- const actionType = action.type;
44
- let actionCode = "";
45
- let actionSummary = "";
46
- try {
47
- // For every action, this method:
48
- // 1. Record the action in codegen
49
- // -> This allows codegen to find locators for the element
50
- // before the action is executed - execution can be a destructive
51
- // operation that modifies the DOM
52
- // 2. Execute the action
53
- // 3. Add to action summary
54
- // 4. Add to action code
55
- switch (actionType) {
56
- case "goto": {
57
- const { url } = action;
58
- await codegen.recordAction({ type: "goto", url });
59
- await page.goto(url);
60
- actionSummary = `Navigated page to ${url}`;
61
- actionCode = await codegen.getCodeForLastAction();
62
- break;
63
- }
64
- case "click": {
65
- const { x, y, button = "left" } = action;
66
- console.log(`Action: click at (${x}, ${y}) with button '${button}'`);
67
- let pwButton = undefined;
68
- if (button === "left" || button === "right") {
69
- pwButton = button;
70
- }
71
- else if (button === "wheel") {
72
- pwButton = "middle";
73
- }
74
- if (pwButton) {
75
- await codegen.recordAction({ type: "click", x, y });
76
- await page.mouse.click(x, y, { button: pwButton });
77
- actionSummary = `Click at (${x}, ${y}) with button '${button}'`;
78
- actionCode = await codegen.getCodeForLastAction();
79
- }
80
- if (button === "back" || button === "forward") {
81
- if (button === "back") {
82
- await codegen.recordAction({ type: "back" });
83
- await page.goBack();
84
- actionSummary = `Navigated page back`;
85
- actionCode = await codegen.getCodeForLastAction();
86
- }
87
- else if (button === "forward") {
88
- await codegen.recordAction({ type: "forward" });
89
- await page.goForward();
90
- actionSummary = `Navigated page forward`;
91
- actionCode = await codegen.getCodeForLastAction();
92
- }
93
- }
94
- break;
95
- }
96
- case "double_click": {
97
- const { x, y } = action;
98
- console.log(`Action: doubleclick at (${x}, ${y})`);
99
- await codegen.recordAction({ type: "doubleclick", x, y });
100
- await page.mouse.dblclick(x, y, { button: "left" });
101
- actionSummary = `Double click at (${x}, ${y})`;
102
- actionCode = await codegen.getCodeForLastAction();
103
- break;
104
- }
105
- case "type": {
106
- const { text } = action;
107
- console.log(`Action: type text '${text}'`);
108
- await codegen.recordAction({ type: "type", text });
109
- await page.keyboard.type(text);
110
- actionSummary = `Type text '${text}'`;
111
- actionCode = await codegen.getCodeForLastAction();
112
- break;
113
- }
114
- case "move": {
115
- const { x, y } = action;
116
- console.log(`Action: mouse move to (${x}, ${y})`);
117
- await codegen.recordAction({ type: "move", x, y });
118
- await page.mouse.move(x, y);
119
- actionSummary = `Mouse move to (${x}, ${y})`;
120
- actionCode = await codegen.getCodeForLastAction();
121
- break;
122
- }
123
- case "drag": {
124
- const { path } = action;
125
- console.log(`Action: drag along path ${path}`);
126
- actionSummary = `Drag along path ${path}`;
127
- if (!path || path.length === 0) {
128
- break;
129
- }
130
- await codegen.recordAction({ type: "drag", path });
131
- await page.mouse.move(path[0].x, path[0].y);
132
- await page.mouse.down();
133
- for (let i = 1; i < path.length; i++) {
134
- await page.mouse.move(path[i].x, path[i].y);
135
- }
136
- await page.mouse.up();
137
- actionCode = await codegen.getCodeForLastAction();
138
- break;
139
- }
140
- case "scroll": {
141
- const { x, y, scroll_x, scroll_y } = action;
142
- console.log(`Action: scroll at (${x}, ${y}) with offsets (scroll_x=${scroll_x}, scroll_y=${scroll_y})`);
143
- await codegen.recordAction({
144
- type: "scroll",
145
- x,
146
- y,
147
- scroll_x,
148
- scroll_y,
149
- });
150
- await page.mouse.move(x, y);
151
- await page.mouse.wheel(scroll_x, scroll_y);
152
- actionSummary = `Scroll at (${x}, ${y}) with offsets (scroll_x=${scroll_x}, scroll_y=${scroll_y})`;
153
- actionCode = await codegen.getCodeForLastAction();
154
- break;
155
- }
156
- case "keypress": {
157
- const { keys } = action;
158
- const mappedKeys = keys.map((k) => {
159
- return CUA_KEY_TO_PLAYWRIGHT_KEY[k.toLowerCase()] || k;
160
- });
161
- await codegen.recordAction({ type: "keypress", keys: mappedKeys });
162
- const mappedKey = mappedKeys.join("+"); // ["CTRL", "A"] becomes ControlOrMeta+A
163
- console.log(`Action: keypress for keys ${keys} -> '${mappedKey}'`);
164
- await page.keyboard.press(mappedKey);
165
- actionSummary = `Keypress for keys ${keys} (mapped to '${mappedKey}' for Playwright)`;
166
- actionCode = await codegen.getCodeForLastAction();
167
- break;
168
- }
169
- case "wait": {
170
- console.log(`Action: wait`);
171
- await codegen.recordAction({ type: "wait" });
172
- await page.waitForTimeout(2000);
173
- actionSummary = `Wait for 2 seconds`;
174
- actionCode = await codegen.getCodeForLastAction();
175
- break;
176
- }
177
- case "screenshot": {
178
- // Nothing to do as screenshot is taken at each turn
179
- console.log(`Action: screenshot`);
180
- actionSummary = `Screenshot`;
181
- break;
182
- }
183
- default:
184
- console.log("Unrecognized action:", action);
185
- }
186
- }
187
- catch (err) {
188
- console.error(`Error handling action ${action.type}: ${err.message || err.toString()}`);
189
- }
190
- return { actionSummary, actionCode };
191
- }
@@ -1,22 +0,0 @@
1
- import { TraceClient } from "@empiricalrun/llm";
2
- import { ArtifactInputData, Usage } from "@empiricalrun/shared-types/chat-agent";
3
- import { Page } from "playwright";
4
- export type BrowserAgentResult = Array<{
5
- type: "text";
6
- text: string;
7
- } | {
8
- type: "screenshot";
9
- screenshot: ArtifactInputData;
10
- }>;
11
- export declare function createTestUsingComputerUseAgent({ page, task, trace, prefersElementFromPointCodegen, }: {
12
- page: Page;
13
- task: string;
14
- trace?: TraceClient;
15
- prefersElementFromPointCodegen?: boolean;
16
- }): Promise<{
17
- code: string;
18
- importPaths: string[];
19
- result: BrowserAgentResult;
20
- usage: Usage;
21
- }>;
22
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/cua/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EACL,iBAAiB,EACjB,KAAK,EACN,MAAM,uCAAuC,CAAC;AAS/C,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAmElC,MAAM,MAAM,kBAAkB,GAAG,KAAK,CAClC;IACE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,YAAY,CAAC;IACnB,UAAU,EAAE,iBAAiB,CAAC;CAC/B,CACJ,CAAC;AAEF,wBAAsB,+BAA+B,CAAC,EACpD,IAAI,EACJ,IAAI,EACJ,KAAK,EACL,8BAAsC,GACvC,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,8BAA8B,CAAC,EAAE,OAAO,CAAC;CAC1C,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd,CAAC,CAuOD"}