@empiricalrun/test-gen 0.51.1 → 0.51.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/agent/chat.d.ts +1 -5
- package/dist/agent/chat.d.ts.map +1 -1
- package/dist/agent/chat.js +14 -3
- package/dist/tools/test-run-fetcher/index.d.ts +4 -0
- package/dist/tools/test-run-fetcher/index.d.ts.map +1 -0
- package/dist/tools/test-run-fetcher/index.js +127 -0
- package/dist/tools/test-run-fetcher/types.d.ts +120 -0
- package/dist/tools/test-run-fetcher/types.d.ts.map +1 -0
- package/dist/tools/test-run-fetcher/types.js +2 -0
- package/package.json +3 -2
- package/dist/tools/test-run-fetcher.d.ts +0 -3
- package/dist/tools/test-run-fetcher.d.ts.map +0 -1
- package/dist/tools/test-run-fetcher.js +0 -59
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# @empiricalrun/test-gen
|
|
2
2
|
|
|
3
|
+
## 0.51.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- a32030e: feat: Formatted usage summary and coloured it
|
|
8
|
+
- Updated dependencies [a32030e]
|
|
9
|
+
- @empiricalrun/llm@0.11.2
|
|
10
|
+
|
|
11
|
+
## 0.51.2
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- ecd3c30: fix: show loader when claude is working
|
|
16
|
+
- ad6c96f: fix: prune test-run tool call response for input tokens
|
|
17
|
+
|
|
3
18
|
## 0.51.1
|
|
4
19
|
|
|
5
20
|
### Patch Changes
|
package/dist/agent/chat.d.ts
CHANGED
|
@@ -1,9 +1,5 @@
|
|
|
1
1
|
export declare function chatAgent({ chatModel, useDiskForChatState, }: {
|
|
2
2
|
chatModel?: "claude-3-7-sonnet-20250219" | "claude-3-5-sonnet-20241022";
|
|
3
3
|
useDiskForChatState?: boolean;
|
|
4
|
-
}): Promise<
|
|
5
|
-
input: number;
|
|
6
|
-
output: number;
|
|
7
|
-
cost: number;
|
|
8
|
-
}>;
|
|
4
|
+
}): Promise<string>;
|
|
9
5
|
//# sourceMappingURL=chat.d.ts.map
|
package/dist/agent/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/agent/chat.ts"],"names":[],"mappings":"AA+EA,wBAAsB,SAAS,CAAC,EAC9B,SAAwC,EACxC,mBAAmB,GACpB,EAAE;IACD,SAAS,CAAC,EAAE,4BAA4B,GAAG,4BAA4B,CAAC;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B,mBA4GA"}
|
package/dist/agent/chat.js
CHANGED
|
@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.chatAgent = void 0;
|
|
7
7
|
const claude_1 = require("@empiricalrun/llm/claude");
|
|
8
8
|
const path_1 = __importDefault(require("path"));
|
|
9
|
+
const picocolors_1 = require("picocolors");
|
|
9
10
|
const human_in_the_loop_1 = require("../human-in-the-loop");
|
|
10
11
|
const browser_agent_1 = require("../tools/browser-agent");
|
|
11
12
|
const diagnosis_fetcher_1 = require("../tools/diagnosis-fetcher");
|
|
@@ -72,8 +73,15 @@ const toolExecutors = {
|
|
|
72
73
|
str_replace_editor: claude_1.strReplaceEditorTool,
|
|
73
74
|
};
|
|
74
75
|
async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForChatState, }) {
|
|
76
|
+
const ora = (await import("ora")).default;
|
|
75
77
|
let userPrompt = undefined;
|
|
76
78
|
let chatState = useDiskForChatState ? claude_1.ChatState.load() : new claude_1.ChatState(false);
|
|
79
|
+
const handleSigInt = () => {
|
|
80
|
+
console.log(`\n${(0, picocolors_1.gray)("Usage summary -> " + chatState.getUsageSummary())}`);
|
|
81
|
+
process.exit(0);
|
|
82
|
+
};
|
|
83
|
+
process.once("SIGINT", handleSigInt);
|
|
84
|
+
process.once("SIGTERM", handleSigInt);
|
|
77
85
|
if (chatState.askUserForInput) {
|
|
78
86
|
// Show last message to the user for context when we loaded from disk
|
|
79
87
|
const messages = chatState.messages;
|
|
@@ -97,7 +105,7 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
|
|
|
97
105
|
catch (e) {
|
|
98
106
|
// https://github.com/SBoudrias/Inquirer.js/issues/1502#issuecomment-2275991680
|
|
99
107
|
if (e instanceof Error && e.name === "ExitPromptError") {
|
|
100
|
-
console.log("
|
|
108
|
+
console.log(`\n${(0, picocolors_1.gray)("Usage summary -> " + chatState.getUsageSummary())}`);
|
|
101
109
|
process.exit(0);
|
|
102
110
|
}
|
|
103
111
|
throw e;
|
|
@@ -115,12 +123,13 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
|
|
|
115
123
|
}
|
|
116
124
|
const toolUse = chatState.getPendingToolCall();
|
|
117
125
|
if (toolUse) {
|
|
118
|
-
|
|
126
|
+
const spinner = ora(`Executing tool ${toolUse.name} with args: ${JSON.stringify(toolUse.input)}`).start();
|
|
119
127
|
const toolExecutor = toolExecutors[toolUse.name];
|
|
120
128
|
if (!toolExecutor) {
|
|
121
129
|
throw new Error(`Tool ${toolUse.name} not found`);
|
|
122
130
|
}
|
|
123
131
|
const toolResult = await toolExecutor(toolUse.input);
|
|
132
|
+
spinner.succeed(`Tool ${toolUse.name} completed`);
|
|
124
133
|
chatState.pushMessage({
|
|
125
134
|
role: "user",
|
|
126
135
|
content: [
|
|
@@ -134,6 +143,7 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
|
|
|
134
143
|
});
|
|
135
144
|
continue;
|
|
136
145
|
}
|
|
146
|
+
const spinner = ora("Claude is working...").start();
|
|
137
147
|
const response = await (0, claude_1.createClaudeMessage)({
|
|
138
148
|
systemPrompt,
|
|
139
149
|
messages: chatState.getMessagesForCreateCompletion(),
|
|
@@ -141,6 +151,7 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
|
|
|
141
151
|
model: chatModel,
|
|
142
152
|
withStrReplaceEditor: true,
|
|
143
153
|
});
|
|
154
|
+
spinner.stop();
|
|
144
155
|
if (!response) {
|
|
145
156
|
throw new Error("No response from LLM");
|
|
146
157
|
}
|
|
@@ -151,7 +162,7 @@ async function chatAgent({ chatModel = "claude-3-7-sonnet-20250219", useDiskForC
|
|
|
151
162
|
}
|
|
152
163
|
}
|
|
153
164
|
const usageSummary = chatState.getUsageSummary();
|
|
154
|
-
console.log("Usage summary
|
|
165
|
+
console.log(`\n${(0, picocolors_1.gray)("Usage summary -> " + chatState.getUsageSummary())}`);
|
|
155
166
|
return usageSummary;
|
|
156
167
|
}
|
|
157
168
|
exports.chatAgent = chatAgent;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/test-run-fetcher/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAWrC,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAOnE;AAED,eAAO,MAAM,WAAW,EAAE,IAwHzB,CAAC"}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.testRunTool = exports.extractPathAfterSourceRepo = void 0;
|
|
4
|
+
const zod_1 = require("zod");
|
|
5
|
+
const TestRunSchema = zod_1.z.object({
|
|
6
|
+
testRunUrl: zod_1.z
|
|
7
|
+
.string()
|
|
8
|
+
.describe("The full URL of the test run (e.g. https://dash.empirical.run/sortment-tests/test-runs/20269 or with query params like ?status=failed)"),
|
|
9
|
+
});
|
|
10
|
+
function extractPathAfterSourceRepo(fullPath) {
|
|
11
|
+
const parts = fullPath.split("source-repo");
|
|
12
|
+
if (parts.length <= 1) {
|
|
13
|
+
return fullPath;
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
return parts[1].replace(/^\/|\/$/g, "");
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
exports.extractPathAfterSourceRepo = extractPathAfterSourceRepo;
|
|
20
|
+
exports.testRunTool = {
|
|
21
|
+
schema: {
|
|
22
|
+
name: "fetchTestRunDetails",
|
|
23
|
+
description: "Fetch details about a test run using its URL",
|
|
24
|
+
parameters: TestRunSchema,
|
|
25
|
+
},
|
|
26
|
+
execute: async (input) => {
|
|
27
|
+
const { testRunUrl } = input;
|
|
28
|
+
// Remove query parameters if they exist
|
|
29
|
+
const urlWithoutParams = testRunUrl.split("?")[0] || testRunUrl;
|
|
30
|
+
// Extract the run ID and repo name from the URL
|
|
31
|
+
const urlParts = urlWithoutParams.split("/");
|
|
32
|
+
const runId = urlParts.pop(); // Last part is the run ID
|
|
33
|
+
const repoName = urlParts[urlParts.length - 2]; // Second to last part is the repo name
|
|
34
|
+
if (!runId || !repoName) {
|
|
35
|
+
throw new Error("Invalid test run URL - could not extract run ID or repo name");
|
|
36
|
+
}
|
|
37
|
+
// Make the API call to fetch test run details
|
|
38
|
+
const response = await fetch(`https://dash.empirical.run/api/test-runs/${runId}?repo_name=${repoName}`, {
|
|
39
|
+
method: "GET",
|
|
40
|
+
headers: {
|
|
41
|
+
Authorization: "weQPMWKT",
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
if (!response.ok) {
|
|
45
|
+
return {
|
|
46
|
+
result: `Failed to fetch test run details: ${response.statusText}`,
|
|
47
|
+
isError: true,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
const data = (await response.json());
|
|
51
|
+
// To efficiently use input_tokens, we
|
|
52
|
+
// 1. Truncate stack trace to last 300 characters
|
|
53
|
+
// 2. Remove request/response headers from network metadata
|
|
54
|
+
// 3. Focus on failed tests only
|
|
55
|
+
if (data.data?.test_run?.flattenedSummaryDetails) {
|
|
56
|
+
data.data.test_run.flattenedSummaryDetails =
|
|
57
|
+
data.data.test_run.flattenedSummaryDetails.map((detail) => {
|
|
58
|
+
// Truncate stack trace to last 300 characters
|
|
59
|
+
if (detail.failed_run_metadata?.stack) {
|
|
60
|
+
const stack = detail.failed_run_metadata.stack;
|
|
61
|
+
// TODO: change to last 10 lines
|
|
62
|
+
detail.failed_run_metadata.stack =
|
|
63
|
+
stack.length > 300 ? `...${stack.slice(-300)}` : stack;
|
|
64
|
+
}
|
|
65
|
+
// Remove headers from network metadata
|
|
66
|
+
if (detail.network_metadata?.failed_calls) {
|
|
67
|
+
detail.network_metadata.failed_calls =
|
|
68
|
+
detail.network_metadata.failed_calls.map((call) => ({
|
|
69
|
+
...call,
|
|
70
|
+
request: { ...call.request, headers: [] },
|
|
71
|
+
response: { ...call.response, headers: [] },
|
|
72
|
+
}));
|
|
73
|
+
}
|
|
74
|
+
if (detail.network_metadata?.failed_calls_within_time_range) {
|
|
75
|
+
detail.network_metadata.failed_calls_within_time_range =
|
|
76
|
+
detail.network_metadata.failed_calls_within_time_range.map((call) => ({
|
|
77
|
+
...call,
|
|
78
|
+
request: { ...call.request, headers: [] },
|
|
79
|
+
response: { ...call.response, headers: [] },
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
return detail;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
const failedTests = data.data?.test_run?.flattenedSummaryDetails.filter((detail) => detail.status === "failed");
|
|
86
|
+
const failedTestInMarkdown = failedTests
|
|
87
|
+
?.map((detail) => {
|
|
88
|
+
const { nesting } = detail.failed_run_metadata;
|
|
89
|
+
return {
|
|
90
|
+
testName: nesting.slice(1).join(" > "),
|
|
91
|
+
fileName: extractPathAfterSourceRepo(detail.failed_run_metadata.location.file),
|
|
92
|
+
detail,
|
|
93
|
+
};
|
|
94
|
+
})
|
|
95
|
+
.map((test) => {
|
|
96
|
+
return `
|
|
97
|
+
### Test name: ${test.testName}
|
|
98
|
+
File: ${test.fileName}
|
|
99
|
+
|
|
100
|
+
Failure data below
|
|
101
|
+
|
|
102
|
+
\`\`\`json
|
|
103
|
+
${JSON.stringify(test.detail, null, 2)}
|
|
104
|
+
\`\`\`
|
|
105
|
+
`;
|
|
106
|
+
})
|
|
107
|
+
.join("\n\n");
|
|
108
|
+
const markdownResponse = `
|
|
109
|
+
# Test run details
|
|
110
|
+
|
|
111
|
+
## Run info
|
|
112
|
+
- Run ID: ${runId}
|
|
113
|
+
- Repository: ${repoName}
|
|
114
|
+
- Started at: ${data.data?.test_run?.testRun.run_started_at}
|
|
115
|
+
- Ended at: ${data.data?.test_run?.testRun.run_ended_at}
|
|
116
|
+
- Duration: ${data.data?.test_run?.testRun.duration} seconds
|
|
117
|
+
- Environment name: ${data.data?.test_run?.testRun.environment_name}
|
|
118
|
+
|
|
119
|
+
## Failed tests
|
|
120
|
+
${failedTestInMarkdown}
|
|
121
|
+
`;
|
|
122
|
+
return {
|
|
123
|
+
result: markdownResponse,
|
|
124
|
+
isError: false,
|
|
125
|
+
};
|
|
126
|
+
},
|
|
127
|
+
};
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
type Project = {
|
|
2
|
+
id: number;
|
|
3
|
+
org_id: number;
|
|
4
|
+
name: string;
|
|
5
|
+
repo_name: string;
|
|
6
|
+
};
|
|
7
|
+
type TestRun = {
|
|
8
|
+
id: number;
|
|
9
|
+
run_id: number;
|
|
10
|
+
project_id: number;
|
|
11
|
+
total_count: number;
|
|
12
|
+
unique_count: number;
|
|
13
|
+
success_count: number;
|
|
14
|
+
failed_count: number;
|
|
15
|
+
skipped_count: number;
|
|
16
|
+
flaky_count: number;
|
|
17
|
+
run_ended_at: string;
|
|
18
|
+
run_started_at: string;
|
|
19
|
+
created_at: string;
|
|
20
|
+
duration: number;
|
|
21
|
+
test_run_branch: string;
|
|
22
|
+
summary_url: string;
|
|
23
|
+
trigger_type: string;
|
|
24
|
+
state: string;
|
|
25
|
+
build_id: number;
|
|
26
|
+
environment_id: number;
|
|
27
|
+
metadata: Record<string, any>;
|
|
28
|
+
test_run_head_sha: string;
|
|
29
|
+
};
|
|
30
|
+
type TestRunEnriched = TestRun & {
|
|
31
|
+
environment_name: string | null;
|
|
32
|
+
environment_slug: string | null;
|
|
33
|
+
commit: string | null;
|
|
34
|
+
commit_url: string | null;
|
|
35
|
+
build_branch: string | null;
|
|
36
|
+
build_url: string | null;
|
|
37
|
+
};
|
|
38
|
+
type FailedRunMetadata = {
|
|
39
|
+
stack: string;
|
|
40
|
+
trace: string;
|
|
41
|
+
videos: string[];
|
|
42
|
+
nesting: string[];
|
|
43
|
+
location: {
|
|
44
|
+
file: string;
|
|
45
|
+
line: number;
|
|
46
|
+
column: number;
|
|
47
|
+
};
|
|
48
|
+
start_time: string;
|
|
49
|
+
pw_test_id: string;
|
|
50
|
+
varying_retries: boolean;
|
|
51
|
+
};
|
|
52
|
+
type CapturedNetworkFailure = {
|
|
53
|
+
endpoint: string;
|
|
54
|
+
status: number;
|
|
55
|
+
timestamp: string;
|
|
56
|
+
distanceFromFailureInMilliseconds: number;
|
|
57
|
+
method: string;
|
|
58
|
+
request: {
|
|
59
|
+
headers: {
|
|
60
|
+
[any: string]: string;
|
|
61
|
+
}[];
|
|
62
|
+
};
|
|
63
|
+
response: {
|
|
64
|
+
headers: {
|
|
65
|
+
[any: string]: string;
|
|
66
|
+
}[];
|
|
67
|
+
};
|
|
68
|
+
};
|
|
69
|
+
type TestCaseSummaryDetail = {
|
|
70
|
+
id: number;
|
|
71
|
+
test_case_id: number;
|
|
72
|
+
test_run_id: number;
|
|
73
|
+
status: string;
|
|
74
|
+
test_project: string;
|
|
75
|
+
failure_type: string;
|
|
76
|
+
failure_type_description: string;
|
|
77
|
+
last_commit_before_run_started_at: string;
|
|
78
|
+
merged_summary?: {
|
|
79
|
+
content: string;
|
|
80
|
+
} | null;
|
|
81
|
+
successful_run_metadata: {
|
|
82
|
+
videos: string[];
|
|
83
|
+
test_id?: string;
|
|
84
|
+
test_run_id?: string;
|
|
85
|
+
} | null;
|
|
86
|
+
failed_run_metadata: FailedRunMetadata;
|
|
87
|
+
visual_diff_summary: {
|
|
88
|
+
failedStep: string;
|
|
89
|
+
reason: string;
|
|
90
|
+
summary: string;
|
|
91
|
+
videos_picked_for_comparison: {
|
|
92
|
+
failure: string;
|
|
93
|
+
success: string;
|
|
94
|
+
};
|
|
95
|
+
} | null;
|
|
96
|
+
error_stack_summary?: {
|
|
97
|
+
content: string;
|
|
98
|
+
} | null;
|
|
99
|
+
network_metadata: {
|
|
100
|
+
failed_calls: CapturedNetworkFailure[];
|
|
101
|
+
failed_calls_within_time_range: CapturedNetworkFailure[];
|
|
102
|
+
} | null;
|
|
103
|
+
created_at: string;
|
|
104
|
+
slug: string;
|
|
105
|
+
failing_line: string;
|
|
106
|
+
};
|
|
107
|
+
export type GetTestRunApiResponse = {
|
|
108
|
+
data: {
|
|
109
|
+
test_run: {
|
|
110
|
+
project: Project;
|
|
111
|
+
testRun: TestRunEnriched;
|
|
112
|
+
flattenedSummaryDetails: TestCaseSummaryDetail[];
|
|
113
|
+
};
|
|
114
|
+
} | null;
|
|
115
|
+
error?: {
|
|
116
|
+
message: string;
|
|
117
|
+
};
|
|
118
|
+
};
|
|
119
|
+
export {};
|
|
120
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/tools/test-run-fetcher/types.ts"],"names":[],"mappings":"AACA,KAAK,OAAO,GAAG;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,OAAO,GAAG;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,KAAK,eAAe,GAAG,OAAO,GAAG;IAC/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC,EAAE,MAAM,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE;QACP,OAAO,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;SAAE,EAAE,CAAC;KACtC,CAAC;IACF,QAAQ,EAAE;QACR,OAAO,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;SAAE,EAAE,CAAC;KACtC,CAAC;CACH,CAAC;AAEF,KAAK,qBAAqB,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,MAAM,CAAC;IACjC,iCAAiC,EAAE,MAAM,CAAC;IAC1C,cAAc,CAAC,EAAE;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,uBAAuB,EAAE;QACvB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,IAAI,CAAC;IACT,mBAAmB,EAAE,iBAAiB,CAAC;IACvC,mBAAmB,EAAE;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,4BAA4B,EAAE;YAC5B,OAAO,EAAE,MAAM,CAAC;YAChB,OAAO,EAAE,MAAM,CAAC;SACjB,CAAC;KACH,GAAG,IAAI,CAAC;IACT,mBAAmB,CAAC,EAAE;QACpB,OAAO,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,gBAAgB,EAAE;QAChB,YAAY,EAAE,sBAAsB,EAAE,CAAC;QACvC,8BAA8B,EAAE,sBAAsB,EAAE,CAAC;KAC1D,GAAG,IAAI,CAAC;IACT,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE;QACJ,QAAQ,EAAE;YACR,OAAO,EAAE,OAAO,CAAC;YACjB,OAAO,EAAE,eAAe,CAAC;YACzB,uBAAuB,EAAE,qBAAqB,EAAE,CAAC;SAClD,CAAC;KACH,GAAG,IAAI,CAAC;IACT,KAAK,CAAC,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@empiricalrun/test-gen",
|
|
3
|
-
"version": "0.51.
|
|
3
|
+
"version": "0.51.3",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"registry": "https://registry.npmjs.org/",
|
|
6
6
|
"access": "public"
|
|
@@ -64,6 +64,7 @@
|
|
|
64
64
|
"minimatch": "^10.0.1",
|
|
65
65
|
"nanoid": "^5.0.7",
|
|
66
66
|
"openai": "4.87.3",
|
|
67
|
+
"ora": "^8.1.0",
|
|
67
68
|
"picocolors": "^1.0.1",
|
|
68
69
|
"prettier": "^3.2.5",
|
|
69
70
|
"remove-markdown": "^0.5.5",
|
|
@@ -73,7 +74,7 @@
|
|
|
73
74
|
"tsx": "^4.16.2",
|
|
74
75
|
"typescript": "^5.3.3",
|
|
75
76
|
"zod": "^3.23.8",
|
|
76
|
-
"@empiricalrun/llm": "^0.11.
|
|
77
|
+
"@empiricalrun/llm": "^0.11.2",
|
|
77
78
|
"@empiricalrun/r2-uploader": "^0.3.8",
|
|
78
79
|
"@empiricalrun/reporter": "^0.23.2",
|
|
79
80
|
"@empiricalrun/test-run": "^0.7.6"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"test-run-fetcher.d.ts","sourceRoot":"","sources":["../../src/tools/test-run-fetcher.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAUpC,eAAO,MAAM,WAAW,EAAE,IA0DzB,CAAC"}
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.testRunTool = void 0;
|
|
4
|
-
const zod_1 = require("zod");
|
|
5
|
-
const TestRunSchema = zod_1.z.object({
|
|
6
|
-
testRunUrl: zod_1.z
|
|
7
|
-
.string()
|
|
8
|
-
.describe("The full URL of the test run (e.g. https://dash.empirical.run/sortment-tests/test-runs/20269 or with query params like ?status=failed)"),
|
|
9
|
-
});
|
|
10
|
-
exports.testRunTool = {
|
|
11
|
-
schema: {
|
|
12
|
-
name: "fetchTestRunDetails",
|
|
13
|
-
description: "Fetch details about a test run using its URL",
|
|
14
|
-
parameters: TestRunSchema,
|
|
15
|
-
},
|
|
16
|
-
execute: async (input) => {
|
|
17
|
-
const { testRunUrl } = input;
|
|
18
|
-
// Remove query parameters if they exist
|
|
19
|
-
const urlWithoutParams = testRunUrl.split("?")[0] || testRunUrl;
|
|
20
|
-
// Extract the run ID and repo name from the URL
|
|
21
|
-
const urlParts = urlWithoutParams.split("/");
|
|
22
|
-
const runId = urlParts.pop(); // Last part is the run ID
|
|
23
|
-
const repoName = urlParts[urlParts.length - 2]; // Second to last part is the repo name
|
|
24
|
-
if (!runId || !repoName) {
|
|
25
|
-
throw new Error("Invalid test run URL - could not extract run ID or repo name");
|
|
26
|
-
}
|
|
27
|
-
// Make the API call to fetch test run details
|
|
28
|
-
const response = await fetch(`https://dash.empirical.run/api/test-runs/${runId}?repo_name=${repoName}`, {
|
|
29
|
-
method: "GET",
|
|
30
|
-
headers: {
|
|
31
|
-
Authorization: "weQPMWKT",
|
|
32
|
-
},
|
|
33
|
-
});
|
|
34
|
-
if (!response.ok) {
|
|
35
|
-
return {
|
|
36
|
-
result: `Failed to fetch test run details: ${response.statusText}`,
|
|
37
|
-
isError: true,
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
|
-
const data = await response.json();
|
|
41
|
-
// Format the response as markdown
|
|
42
|
-
const markdownResponse = `
|
|
43
|
-
# Test Run Details
|
|
44
|
-
|
|
45
|
-
## Run Information
|
|
46
|
-
- **Run ID**: ${runId}
|
|
47
|
-
- **Repository**: ${repoName}
|
|
48
|
-
|
|
49
|
-
## Test Run Data
|
|
50
|
-
\`\`\`json
|
|
51
|
-
${JSON.stringify(data, null, 2)}
|
|
52
|
-
\`\`\`
|
|
53
|
-
`;
|
|
54
|
-
return {
|
|
55
|
-
result: markdownResponse,
|
|
56
|
-
isError: false,
|
|
57
|
-
};
|
|
58
|
-
},
|
|
59
|
-
};
|