@loadmill/droid-cua 2.16.0 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +72 -0
- package/build/src/cli/headless-execution-config.js +15 -0
- package/build/src/cli/headless-runner.js +19 -7
- package/build/src/cli/scenario-planner.js +1 -0
- package/build/src/commands/help.js +3 -0
- package/build/src/core/prompts.js +1 -1
- package/build/src/core/scenario-planner.js +85 -0
- package/build/src/core/test-data-import.js +62 -0
- package/build/src/core/test-data.js +103 -0
- package/build/src/prompts/execution.js +13 -1
- package/build/src/prompts/scenario-planner.js +48 -0
- package/build/src/prompts/test-data-import.js +49 -0
- package/package.json +1 -1
package/build/index.js
CHANGED
|
@@ -13,6 +13,7 @@ import { createOpenAiExecutor } from "./src/device/openai.js";
|
|
|
13
13
|
import { logger } from "./src/utils/logger.js";
|
|
14
14
|
import { selectDevice } from "./src/cli/device-selector.js";
|
|
15
15
|
import { resolveExecutionAppContext } from "./src/core/app-context.js";
|
|
16
|
+
import { resolveExecutionTestData } from "./src/core/test-data.js";
|
|
16
17
|
import { createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
|
|
17
18
|
import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
|
|
18
19
|
import { resolveHeadlessTarget } from "./src/cli/headless-target.js";
|
|
@@ -20,6 +21,7 @@ import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
|
|
|
20
21
|
import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
|
|
21
22
|
import { listHeadlessInstructionFiles, resolveHeadlessInstructionsTarget } from "./src/cli/headless-instruction-target.js";
|
|
22
23
|
import { runHeadlessInstructionRuns } from "./src/cli/headless-runner.js";
|
|
24
|
+
import { planScenarioRuns } from "./src/cli/scenario-planner.js";
|
|
23
25
|
import { resolveLambdaTestCredentials, uploadLambdaTestCliApp } from "./src/cli/lambdatest-app-upload.js";
|
|
24
26
|
import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
|
|
25
27
|
import { getCloudProviderAdapter } from "./src/device/cloud/registry.js";
|
|
@@ -67,6 +69,41 @@ function wait(ms) {
|
|
|
67
69
|
function isLambdaTestAppAutomationUrl(value) {
|
|
68
70
|
return typeof value === "string" && value.startsWith("https://appautomation.lambdatest.com/test?");
|
|
69
71
|
}
|
|
72
|
+
async function loadScenarioCandidateRuns(runs) {
|
|
73
|
+
return await Promise.all(runs.map(async (run) => ({
|
|
74
|
+
...run,
|
|
75
|
+
instructions: run.instructions ?? await loadHeadlessInstructions(run.testPath),
|
|
76
|
+
})));
|
|
77
|
+
}
|
|
78
|
+
function printScenarioCasePreview(cases) {
|
|
79
|
+
console.log(`\nScenario preview: ${cases.length} case${cases.length === 1 ? "" : "s"}\n`);
|
|
80
|
+
cases.forEach((entry, index) => {
|
|
81
|
+
console.log(`${index + 1}. ${entry.caseName}`);
|
|
82
|
+
});
|
|
83
|
+
console.log("");
|
|
84
|
+
}
|
|
85
|
+
function buildScenarioInstructionTarget(originalTarget, scenarioPrompt) {
|
|
86
|
+
return {
|
|
87
|
+
type: "scenario",
|
|
88
|
+
path: originalTarget?.path || null,
|
|
89
|
+
sourceType: originalTarget?.type || null,
|
|
90
|
+
scenarioPrompt,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
function buildScenarioRuns({ candidateRuns, cases }) {
|
|
94
|
+
const runByTestName = new Map(candidateRuns.map((run) => [run.testName, run]));
|
|
95
|
+
return cases.map((entry) => {
|
|
96
|
+
const sourceRun = runByTestName.get(entry.testName);
|
|
97
|
+
return {
|
|
98
|
+
...sourceRun,
|
|
99
|
+
testName: entry.caseName,
|
|
100
|
+
sourceTestName: entry.testName,
|
|
101
|
+
scenarioCaseName: entry.caseName,
|
|
102
|
+
testDataContext: entry.testDataContext,
|
|
103
|
+
instructions: sourceRun.instructions,
|
|
104
|
+
};
|
|
105
|
+
});
|
|
106
|
+
}
|
|
70
107
|
async function resolveLambdaTestDashboardUrl(credentials, sessionId, { attempts = 1, delayMs = 1000 } = {}) {
|
|
71
108
|
const adapter = getCloudProviderAdapter("lambdatest");
|
|
72
109
|
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
@@ -107,10 +144,17 @@ async function main() {
|
|
|
107
144
|
if (isLambdaTestTarget && !isHeadlessInstructionsRun) {
|
|
108
145
|
throw new Error("--device-source lambdatest is only supported with --instructions headless execution.");
|
|
109
146
|
}
|
|
147
|
+
if (!isHeadlessInstructionsRun && args.scenario != null) {
|
|
148
|
+
throw new Error("--scenario can only be used with --instructions headless execution.");
|
|
149
|
+
}
|
|
150
|
+
if (!isHeadlessInstructionsRun && args["test-data"] != null) {
|
|
151
|
+
throw new Error("--test-data can only be used with --instructions headless execution.");
|
|
152
|
+
}
|
|
110
153
|
let preloadedHeadlessConfig = null;
|
|
111
154
|
let preloadedHeadlessTarget = null;
|
|
112
155
|
let headlessInstructionRuns = [];
|
|
113
156
|
let preloadedAppContext = null;
|
|
157
|
+
let preloadedTestData = null;
|
|
114
158
|
if (isHeadlessInstructionsRun) {
|
|
115
159
|
preloadedHeadlessConfig = await resolveHeadlessExecutionConfig(args);
|
|
116
160
|
preloadedHeadlessTarget = await resolveHeadlessInstructionsTarget(instructionsFile);
|
|
@@ -139,6 +183,14 @@ async function main() {
|
|
|
139
183
|
if (preloadedAppContext.status === "failed") {
|
|
140
184
|
throw new Error(`Could not read app context file ${preloadedAppContext.contextPath}: ${preloadedAppContext.message}`);
|
|
141
185
|
}
|
|
186
|
+
if (preloadedHeadlessConfig.scenarioPrompt) {
|
|
187
|
+
preloadedTestData = await resolveExecutionTestData({
|
|
188
|
+
testDataPath: preloadedHeadlessConfig.testDataPath,
|
|
189
|
+
});
|
|
190
|
+
if (preloadedTestData.status === "failed") {
|
|
191
|
+
throw new Error(`Could not read test data file ${preloadedTestData.testDataPath}: ${preloadedTestData.message}`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
142
194
|
}
|
|
143
195
|
// If no device specified, show interactive selection menu
|
|
144
196
|
if (!isBrowserTarget && !isLambdaTestTarget && !avdName && !platform) {
|
|
@@ -236,6 +288,26 @@ async function main() {
|
|
|
236
288
|
});
|
|
237
289
|
console.log(`Using AI provider: ${runtimeContext.config.llmProviderMode}`);
|
|
238
290
|
console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
|
|
291
|
+
if (headlessConfig.scenarioPrompt) {
|
|
292
|
+
const candidateRuns = await loadScenarioCandidateRuns(headlessInstructionRuns);
|
|
293
|
+
const scenarioCases = await planScenarioRuns({
|
|
294
|
+
client: runtimeContext.openAiClient,
|
|
295
|
+
runtimeConfig: runtimeContext.config,
|
|
296
|
+
scenarioPrompt: headlessConfig.scenarioPrompt,
|
|
297
|
+
tests: candidateRuns.map((run) => ({
|
|
298
|
+
testName: run.testName,
|
|
299
|
+
instructions: run.instructions,
|
|
300
|
+
})),
|
|
301
|
+
appContextText,
|
|
302
|
+
testDataText: preloadedTestData?.text || "",
|
|
303
|
+
});
|
|
304
|
+
printScenarioCasePreview(scenarioCases);
|
|
305
|
+
headlessInstructionRuns = buildScenarioRuns({
|
|
306
|
+
candidateRuns,
|
|
307
|
+
cases: scenarioCases,
|
|
308
|
+
});
|
|
309
|
+
preloadedHeadlessTarget = buildScenarioInstructionTarget(preloadedHeadlessTarget, headlessConfig.scenarioPrompt);
|
|
310
|
+
}
|
|
239
311
|
const result = await runHeadlessInstructionRuns({
|
|
240
312
|
runs: headlessInstructionRuns,
|
|
241
313
|
instructionsTarget: preloadedHeadlessTarget,
|
|
@@ -172,6 +172,10 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
172
172
|
const cwd = typeof options.cwd === "string" ? options.cwd : process.cwd();
|
|
173
173
|
const configPath = typeof args.config === "string" ? args.config : null;
|
|
174
174
|
const explicitContextPath = typeof args.context === "string" ? path.resolve(cwd, args.context) : null;
|
|
175
|
+
const scenarioPrompt = typeof args.scenario === "string" ? args.scenario.trim() : null;
|
|
176
|
+
const hasScenarioFlag = args.scenario != null;
|
|
177
|
+
const testDataPath = typeof args["test-data"] === "string" ? path.resolve(cwd, args["test-data"]) : null;
|
|
178
|
+
const hasTestDataFlag = args["test-data"] != null;
|
|
175
179
|
const noContext = args["no-context"] === true || args.context === false;
|
|
176
180
|
const contextOptimizationEnabledByFlag = args["context-optimization"] === true;
|
|
177
181
|
const noContextOptimization = args["no-context-optimization"] === true || args["context-optimization"] === false;
|
|
@@ -183,6 +187,15 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
183
187
|
if (explicitContextPath && noContext) {
|
|
184
188
|
throw new Error("--context and --no-context cannot be used together.");
|
|
185
189
|
}
|
|
190
|
+
if (hasScenarioFlag && !scenarioPrompt) {
|
|
191
|
+
throw new Error("--scenario must be a non-empty string.");
|
|
192
|
+
}
|
|
193
|
+
if (hasTestDataFlag && !testDataPath) {
|
|
194
|
+
throw new Error("--test-data must be a non-empty path.");
|
|
195
|
+
}
|
|
196
|
+
if (testDataPath && !scenarioPrompt) {
|
|
197
|
+
throw new Error("--test-data can only be used with --scenario.");
|
|
198
|
+
}
|
|
186
199
|
if (contextOptimizationEnabledByFlag) {
|
|
187
200
|
throw new Error("--context-optimization is not supported; Context Optimization is enabled by default.");
|
|
188
201
|
}
|
|
@@ -202,6 +215,8 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
|
|
|
202
215
|
promptCustomizations,
|
|
203
216
|
appContextEnabled: fileConfig?.appContextEnabled ?? true,
|
|
204
217
|
appContextPath: fileConfig?.appContextPath || null,
|
|
218
|
+
scenarioPrompt,
|
|
219
|
+
testDataPath,
|
|
205
220
|
contextOptimizationEnabled: fileConfig?.contextOptimizationEnabled ?? true,
|
|
206
221
|
contextOptimizationThreshold: fileConfig?.contextOptimizationThreshold ?? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD,
|
|
207
222
|
};
|
|
@@ -100,11 +100,12 @@ function buildHeadlessEngineOptions({ runtimeContext, headlessConfig, executionP
|
|
|
100
100
|
}
|
|
101
101
|
return engineOptions;
|
|
102
102
|
}
|
|
103
|
-
async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
|
|
103
|
+
async function runHeadlessTest({ runId, testPath, testName, instructions, session, deviceInfo, headlessConfig, appContext, testDataContext = "", runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
|
|
104
104
|
let executionMode = null;
|
|
105
105
|
try {
|
|
106
106
|
await headlessDebug.startExecutionSession(runId, {
|
|
107
107
|
testName,
|
|
108
|
+
...(testDataContext ? { testDataUsed: true } : {}),
|
|
108
109
|
platform: deviceInfo.platform,
|
|
109
110
|
deviceName: deviceInfo.device_name,
|
|
110
111
|
aimAssistMode: headlessConfig.aimAssistMode
|
|
@@ -119,7 +120,10 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
119
120
|
emitCliAppContextDebug(runId, appContext, appContextText);
|
|
120
121
|
session.updateResponseId(undefined);
|
|
121
122
|
session.clearMessages();
|
|
122
|
-
const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, {
|
|
123
|
+
const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, {
|
|
124
|
+
aimAssistMode: headlessConfig.aimAssistMode,
|
|
125
|
+
testDataContext,
|
|
126
|
+
});
|
|
123
127
|
session.setSystemPrompt(executionPrompt);
|
|
124
128
|
const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
|
|
125
129
|
runId,
|
|
@@ -188,9 +192,12 @@ async function runHeadlessTest({ runId, testPath, testName, instructions, sessio
|
|
|
188
192
|
throw error;
|
|
189
193
|
}
|
|
190
194
|
}
|
|
191
|
-
export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget }) {
|
|
192
|
-
const isBatchRun = instructionsTarget?.type === "directory";
|
|
193
|
-
if (
|
|
195
|
+
export async function runHeadlessInstructionRuns({ runs, instructionsTarget, session, deviceInfo, headlessConfig, appContext, runtimeContext, headlessDebug, recordScreenshots, screenshotDir, isBrowserTarget, isLambdaTestTarget, runHeadlessTestImpl = runHeadlessTest }) {
|
|
196
|
+
const isBatchRun = instructionsTarget?.type === "directory" || instructionsTarget?.type === "scenario";
|
|
197
|
+
if (instructionsTarget?.type === "scenario") {
|
|
198
|
+
console.log(`\nRunning ${runs.length} scenario case${runs.length === 1 ? "" : "s"}\n`);
|
|
199
|
+
}
|
|
200
|
+
else if (isBatchRun) {
|
|
194
201
|
console.log(`\nRunning ${runs.length} tests from: ${instructionsTarget.path}\n`);
|
|
195
202
|
}
|
|
196
203
|
let failures = 0;
|
|
@@ -205,7 +212,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
205
212
|
if (recordScreenshots && testScreenshotDir) {
|
|
206
213
|
await mkdir(testScreenshotDir, { recursive: true });
|
|
207
214
|
}
|
|
208
|
-
const result = await
|
|
215
|
+
const result = await runHeadlessTestImpl({
|
|
209
216
|
runId,
|
|
210
217
|
testPath: run.testPath,
|
|
211
218
|
testName: run.testName,
|
|
@@ -214,6 +221,7 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
214
221
|
deviceInfo,
|
|
215
222
|
headlessConfig,
|
|
216
223
|
appContext,
|
|
224
|
+
testDataContext: run.testDataContext || "",
|
|
217
225
|
runtimeContext,
|
|
218
226
|
headlessDebug,
|
|
219
227
|
recordScreenshots,
|
|
@@ -234,7 +242,11 @@ export async function runHeadlessInstructionRuns({ runs, instructionsTarget, ses
|
|
|
234
242
|
console.error(`\nTest failed: ${run.testName}: ${message}`);
|
|
235
243
|
}
|
|
236
244
|
}
|
|
237
|
-
if (
|
|
245
|
+
if (instructionsTarget?.type === "scenario") {
|
|
246
|
+
const passed = runs.length - failures;
|
|
247
|
+
console.log(`\nScenario run complete: ${passed}/${runs.length} passed`);
|
|
248
|
+
}
|
|
249
|
+
else if (isBatchRun) {
|
|
238
250
|
const passed = runs.length - failures;
|
|
239
251
|
console.log(`\nFolder run complete: ${passed}/${runs.length} passed`);
|
|
240
252
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { planScenarioRuns, validateScenarioCases, } from "../core/scenario-planner.js";
|
|
@@ -26,6 +26,8 @@ export async function handleHelp(args, session, context) {
|
|
|
26
26
|
addOutput({ type: 'info', text: ' --os-version <version> LambdaTest OS version' });
|
|
27
27
|
addOutput({ type: 'info', text: ' --app <path> LambdaTest app package path (.apk or .ipa)' });
|
|
28
28
|
addOutput({ type: 'info', text: ' --instructions <file> Run test file in headless mode' });
|
|
29
|
+
addOutput({ type: 'info', text: ' --scenario <prompt> Generate and run scenario cases for headless instructions' });
|
|
30
|
+
addOutput({ type: 'info', text: ' --test-data <file> Optional test-data.md file for --scenario' });
|
|
29
31
|
addOutput({ type: 'info', text: ' --session-mode <mode> Browser session mode for web runs: persistent or fresh' });
|
|
30
32
|
addOutput({ type: 'info', text: ' --config <file> Headless execution JSON config for prompt parity' });
|
|
31
33
|
addOutput({ type: 'info', text: ' --llm-provider <mode> Headless AI provider: loadmill or openai' });
|
|
@@ -80,6 +82,7 @@ export async function handleHelp(args, session, context) {
|
|
|
80
82
|
addOutput({ type: 'info', text: ' droid-cua --avd "iPhone 16" (iOS Simulator, auto-detected)' });
|
|
81
83
|
addOutput({ type: 'info', text: ' droid-cua --platform ios --avd MySim (Force iOS platform)' });
|
|
82
84
|
addOutput({ type: 'info', text: ' droid-cua --instructions tests/login.dcua --context app/context.md' });
|
|
85
|
+
addOutput({ type: 'info', text: ' droid-cua --instructions tests --test-data tests/test-data.md --scenario "Run every checkout test with each buyer account"' });
|
|
83
86
|
addOutput({ type: 'info', text: ' droid-cua --target web --instructions tests/search.dcua' });
|
|
84
87
|
addOutput({ type: 'info', text: ' droid-cua --device-source lambdatest --platform android --device-name "Galaxy S24" --os-version 14 --app app.apk --instructions tests/login.dcua' });
|
|
85
88
|
addOutput({ type: 'info', text: ' /create login-test (design a new test)' });
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { buildBaseSystemPrompt } from "../prompts/base.js";
|
|
2
2
|
export { buildDesignModePrompt, buildDesignRecoveryPrompt, buildDesignScriptGenerationPrompt, } from "../prompts/design.js";
|
|
3
|
-
export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, } from "../prompts/execution.js";
|
|
3
|
+
export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, buildTestDataContextSection, } from "../prompts/execution.js";
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { resolveEffectiveHelperModel } from "./runtime-config.js";
|
|
2
|
+
import { buildScenarioPlannerMessages } from "../prompts/scenario-planner.js";
|
|
3
|
+
function parseScenarioPlannerContent(content) {
|
|
4
|
+
try {
|
|
5
|
+
return JSON.parse(content);
|
|
6
|
+
}
|
|
7
|
+
catch (error) {
|
|
8
|
+
const message = error instanceof Error ? error.message : "Invalid JSON.";
|
|
9
|
+
throw new Error(`Could not parse scenario planner response: ${message}`);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
function normalizeCaseName(value, fallback) {
|
|
13
|
+
const trimmed = typeof value === "string" ? value.trim() : "";
|
|
14
|
+
return trimmed || fallback;
|
|
15
|
+
}
|
|
16
|
+
function dedupeCaseNames(cases) {
|
|
17
|
+
const usedNames = new Set();
|
|
18
|
+
const baseCounts = new Map();
|
|
19
|
+
return cases.map((entry) => {
|
|
20
|
+
const baseName = entry.caseName;
|
|
21
|
+
let nextIndex = baseCounts.get(baseName) || 1;
|
|
22
|
+
let caseName = baseName;
|
|
23
|
+
while (usedNames.has(caseName)) {
|
|
24
|
+
nextIndex += 1;
|
|
25
|
+
caseName = `${baseName} (${nextIndex})`;
|
|
26
|
+
}
|
|
27
|
+
baseCounts.set(baseName, nextIndex);
|
|
28
|
+
usedNames.add(caseName);
|
|
29
|
+
return {
|
|
30
|
+
...entry,
|
|
31
|
+
caseName,
|
|
32
|
+
};
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
export function validateScenarioCases(rawCases, candidateTests) {
|
|
36
|
+
if (!Array.isArray(rawCases) || rawCases.length === 0) {
|
|
37
|
+
throw new Error("Scenario planner did not return any cases.");
|
|
38
|
+
}
|
|
39
|
+
const candidateNames = new Set(candidateTests.map((test) => test.testName));
|
|
40
|
+
const normalizedCases = rawCases.map((rawCase, index) => {
|
|
41
|
+
if (!rawCase || typeof rawCase !== "object" || Array.isArray(rawCase)) {
|
|
42
|
+
throw new Error(`Scenario case ${index + 1} must be an object.`);
|
|
43
|
+
}
|
|
44
|
+
const testName = typeof rawCase.testName === "string" ? rawCase.testName.trim() : "";
|
|
45
|
+
if (!candidateNames.has(testName)) {
|
|
46
|
+
throw new Error(`Scenario case ${index + 1} references unknown test "${testName || "(missing)"}".`);
|
|
47
|
+
}
|
|
48
|
+
const caseName = normalizeCaseName(rawCase.caseName, `${testName} / Case ${index + 1}`);
|
|
49
|
+
const testDataContext = typeof rawCase.testDataContext === "string" ? rawCase.testDataContext.trim() : "";
|
|
50
|
+
return {
|
|
51
|
+
testName,
|
|
52
|
+
caseName,
|
|
53
|
+
testDataContext,
|
|
54
|
+
};
|
|
55
|
+
});
|
|
56
|
+
return dedupeCaseNames(normalizedCases);
|
|
57
|
+
}
|
|
58
|
+
export async function planScenarioRuns({ client, runtimeConfig, scenarioPrompt, tests, appContextText = "", testDataText = "", }) {
|
|
59
|
+
if (!client?.chat?.completions?.create) {
|
|
60
|
+
throw new Error("Scenario planning requires an LLM client.");
|
|
61
|
+
}
|
|
62
|
+
const prompt = typeof scenarioPrompt === "string" ? scenarioPrompt.trim() : "";
|
|
63
|
+
if (!prompt) {
|
|
64
|
+
throw new Error("--scenario must be a non-empty string.");
|
|
65
|
+
}
|
|
66
|
+
if (!Array.isArray(tests) || tests.length === 0) {
|
|
67
|
+
throw new Error("Scenario planning requires at least one candidate test.");
|
|
68
|
+
}
|
|
69
|
+
const response = await client.chat.completions.create({
|
|
70
|
+
model: resolveEffectiveHelperModel({
|
|
71
|
+
llmProviderMode: runtimeConfig?.llmProviderMode,
|
|
72
|
+
cuaModel: runtimeConfig?.cuaModel,
|
|
73
|
+
}),
|
|
74
|
+
messages: buildScenarioPlannerMessages({
|
|
75
|
+
scenarioPrompt: prompt,
|
|
76
|
+
tests,
|
|
77
|
+
appContextText,
|
|
78
|
+
testDataText,
|
|
79
|
+
}),
|
|
80
|
+
response_format: { type: "json_object" },
|
|
81
|
+
});
|
|
82
|
+
const content = response.choices?.[0]?.message?.content || "{}";
|
|
83
|
+
const parsed = parseScenarioPlannerContent(content);
|
|
84
|
+
return validateScenarioCases(parsed.cases, tests);
|
|
85
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { resolveEffectiveHelperModel } from "./runtime-config.js";
|
|
3
|
+
import { buildTestDataImportMessages } from "../prompts/test-data-import.js";
|
|
4
|
+
export const TEST_DATA_IMPORT_EXTENSIONS = [".csv", ".txt"];
|
|
5
|
+
export function isSupportedTestDataImportPath(filePath) {
|
|
6
|
+
return TEST_DATA_IMPORT_EXTENSIONS.includes(path.extname(String(filePath ?? "")).toLowerCase());
|
|
7
|
+
}
|
|
8
|
+
export function assertSupportedTestDataImportPath(filePath) {
|
|
9
|
+
if (!isSupportedTestDataImportPath(filePath)) {
|
|
10
|
+
throw new Error("Test data import supports CSV and TXT files only.");
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function parseTestDataImportContent(content) {
|
|
14
|
+
try {
|
|
15
|
+
return JSON.parse(content);
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
const message = error instanceof Error ? error.message : "Invalid JSON.";
|
|
19
|
+
throw new Error(`Could not parse test data import response: ${message}`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function normalizeGeneratedContent(value) {
|
|
23
|
+
const content = typeof value === "string" ? value.trim() : "";
|
|
24
|
+
if (!content) {
|
|
25
|
+
throw new Error("Test data import returned empty content.");
|
|
26
|
+
}
|
|
27
|
+
if (/^```/.test(content)) {
|
|
28
|
+
throw new Error("Test data import returned code-fenced content instead of markdown.");
|
|
29
|
+
}
|
|
30
|
+
return content.endsWith("\n") ? content : `${content}\n`;
|
|
31
|
+
}
|
|
32
|
+
export async function importTestDataMarkdown({ client, runtimeConfig, currentContent = "", importedFileName, importedFileText, instructions = "", }) {
|
|
33
|
+
if (!client?.chat?.completions?.create) {
|
|
34
|
+
throw new Error("Test data import requires an LLM client.");
|
|
35
|
+
}
|
|
36
|
+
const fileName = typeof importedFileName === "string" ? importedFileName.trim() : "";
|
|
37
|
+
if (!fileName) {
|
|
38
|
+
throw new Error("Imported file name is required.");
|
|
39
|
+
}
|
|
40
|
+
const fileText = typeof importedFileText === "string" ? importedFileText : "";
|
|
41
|
+
if (!fileText.trim()) {
|
|
42
|
+
throw new Error("Selected import file is empty.");
|
|
43
|
+
}
|
|
44
|
+
const response = await client.chat.completions.create({
|
|
45
|
+
model: resolveEffectiveHelperModel({
|
|
46
|
+
llmProviderMode: runtimeConfig?.llmProviderMode,
|
|
47
|
+
cuaModel: runtimeConfig?.cuaModel,
|
|
48
|
+
}),
|
|
49
|
+
messages: buildTestDataImportMessages({
|
|
50
|
+
currentContent,
|
|
51
|
+
importedFileName: fileName,
|
|
52
|
+
importedFileText: fileText,
|
|
53
|
+
instructions,
|
|
54
|
+
}),
|
|
55
|
+
response_format: { type: "json_object" },
|
|
56
|
+
});
|
|
57
|
+
const content = response.choices?.[0]?.message?.content || "{}";
|
|
58
|
+
const parsed = parseTestDataImportContent(content);
|
|
59
|
+
return {
|
|
60
|
+
content: normalizeGeneratedContent(parsed.content),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { access, readFile } from "fs/promises";
|
|
3
|
+
import { constants as fsConstants } from "fs";
|
|
4
|
+
export const TEST_DATA_FILENAME = "test-data.md";
|
|
5
|
+
export const DEFAULT_TEST_DATA_TEMPLATE = `# Test Data
|
|
6
|
+
<!-- Concrete data the agent can use during Scenario Runs. Keep entries named and specific. -->
|
|
7
|
+
<!-- Examples below are commented out so they are guidance, not usable test data. -->
|
|
8
|
+
|
|
9
|
+
## Accounts
|
|
10
|
+
<!--
|
|
11
|
+
### Standard buyer
|
|
12
|
+
Email:
|
|
13
|
+
Password:
|
|
14
|
+
Notes: When should the agent use this account?
|
|
15
|
+
-->
|
|
16
|
+
|
|
17
|
+
## Products
|
|
18
|
+
<!--
|
|
19
|
+
### In-stock product
|
|
20
|
+
Name:
|
|
21
|
+
SKU:
|
|
22
|
+
Notes:
|
|
23
|
+
-->
|
|
24
|
+
|
|
25
|
+
## Payment Methods
|
|
26
|
+
<!--
|
|
27
|
+
### Valid card
|
|
28
|
+
Number:
|
|
29
|
+
Expiration:
|
|
30
|
+
CVV:
|
|
31
|
+
Notes:
|
|
32
|
+
-->
|
|
33
|
+
|
|
34
|
+
## Roles and Permissions
|
|
35
|
+
|
|
36
|
+
## Regions and Locales
|
|
37
|
+
|
|
38
|
+
## Other Data
|
|
39
|
+
`;
|
|
40
|
+
export function getDefaultProjectTestDataPath(projectPath) {
|
|
41
|
+
return path.join(projectPath, TEST_DATA_FILENAME);
|
|
42
|
+
}
|
|
43
|
+
export function normalizeTestDataTemplateText(text) {
|
|
44
|
+
return String(text ?? "").replace(/\r\n?/g, "\n").replace(/[ \t]+$/gm, "").trimEnd();
|
|
45
|
+
}
|
|
46
|
+
export function isDefaultTestDataTemplate(text) {
|
|
47
|
+
if (!String(text ?? "").trim()) {
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
return normalizeTestDataTemplateText(text) === normalizeTestDataTemplateText(DEFAULT_TEST_DATA_TEMPLATE);
|
|
51
|
+
}
|
|
52
|
+
export async function readTestDataFile(filePath) {
|
|
53
|
+
await access(filePath, fsConstants.R_OK);
|
|
54
|
+
return await readFile(filePath, "utf-8");
|
|
55
|
+
}
|
|
56
|
+
export async function loadTestData({ testDataPath }) {
|
|
57
|
+
if (!testDataPath) {
|
|
58
|
+
return { text: "", testDataPath: null };
|
|
59
|
+
}
|
|
60
|
+
const text = await readTestDataFile(testDataPath);
|
|
61
|
+
return { text, testDataPath };
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* @param {{
|
|
65
|
+
* testDataPath?: string | null,
|
|
66
|
+
* missingFileStatus?: string | null,
|
|
67
|
+
* }} [options]
|
|
68
|
+
*/
|
|
69
|
+
export async function resolveExecutionTestData({ testDataPath = null, missingFileStatus = null, } = {}) {
|
|
70
|
+
if (!testDataPath) {
|
|
71
|
+
return {
|
|
72
|
+
status: "omitted",
|
|
73
|
+
text: "",
|
|
74
|
+
testDataPath: null,
|
|
75
|
+
message: null,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
const result = await loadTestData({ testDataPath });
|
|
80
|
+
return {
|
|
81
|
+
status: result.text.trim().length > 0 ? "loaded" : "empty",
|
|
82
|
+
text: result.text,
|
|
83
|
+
testDataPath: result.testDataPath,
|
|
84
|
+
message: null,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
if (missingFileStatus && error && typeof error === "object" && error.code === "ENOENT") {
|
|
89
|
+
return {
|
|
90
|
+
status: missingFileStatus,
|
|
91
|
+
text: "",
|
|
92
|
+
testDataPath,
|
|
93
|
+
message: null,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
status: "failed",
|
|
98
|
+
text: "",
|
|
99
|
+
testDataPath,
|
|
100
|
+
message: error instanceof Error ? error.message : "Unknown test data error",
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -9,6 +9,17 @@ export function buildAppContextSection(text) {
|
|
|
9
9
|
The following describes the app you are testing.
|
|
10
10
|
Use this to understand screen layouts, terminology, navigation, and expected behavior.
|
|
11
11
|
|
|
12
|
+
${trimmed}`;
|
|
13
|
+
}
|
|
14
|
+
export function buildTestDataContextSection(text) {
|
|
15
|
+
const trimmed = typeof text === "string" ? text.trim() : "";
|
|
16
|
+
if (!trimmed) {
|
|
17
|
+
return "";
|
|
18
|
+
}
|
|
19
|
+
return `TEST DATA FOR THIS RUN:
|
|
20
|
+
The following values are test data for this execution. Use them as literal run data when performing the test.
|
|
21
|
+
They do not replace or override the test instructions.
|
|
22
|
+
|
|
12
23
|
${trimmed}`;
|
|
13
24
|
}
|
|
14
25
|
export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
|
|
@@ -16,6 +27,7 @@ export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, ap
|
|
|
16
27
|
const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
|
|
17
28
|
const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
|
|
18
29
|
const appContextSection = buildAppContextSection(appContextText);
|
|
30
|
+
const testDataContextSection = buildTestDataContextSection(options.testDataContext);
|
|
19
31
|
const browserTarget = isBrowserTarget(deviceInfo);
|
|
20
32
|
const prompt = `${basePrompt}
|
|
21
33
|
|
|
@@ -52,7 +64,7 @@ Your process:
|
|
|
52
64
|
7. Stop immediately - no commentary, no questions
|
|
53
65
|
|
|
54
66
|
Each instruction is independent. Do not reference previous instructions or ask about next steps.
|
|
55
|
-
${appContextSection ? `\n\n${appContextSection}` : ""}`;
|
|
67
|
+
${appContextSection ? `\n\n${appContextSection}` : ""}${testDataContextSection ? `\n\n${testDataContextSection}` : ""}`;
|
|
56
68
|
return appendCustomSections(prompt, [
|
|
57
69
|
{ title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
|
|
58
70
|
{ title: "Execution Mode Instructions", text: executionCustomText }
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
function formatCandidateTests(tests) {
|
|
2
|
+
return tests.map((test, index) => {
|
|
3
|
+
const content = Array.isArray(test.instructions) ? test.instructions.join("\n") : "";
|
|
4
|
+
return [
|
|
5
|
+
`TEST ${index + 1}`,
|
|
6
|
+
`Name: ${test.testName}`,
|
|
7
|
+
"Instructions:",
|
|
8
|
+
content || "(empty)",
|
|
9
|
+
].join("\n");
|
|
10
|
+
}).join("\n\n---\n\n");
|
|
11
|
+
}
|
|
12
|
+
export function buildScenarioPlannerMessages({ scenarioPrompt, tests, appContextText = "", testDataText = "", }) {
|
|
13
|
+
return [
|
|
14
|
+
{
|
|
15
|
+
role: "system",
|
|
16
|
+
content: [
|
|
17
|
+
"You prepare concrete test executions for a computer-using agent.",
|
|
18
|
+
"The agent runs manual-style app tests: it receives a test brief, app context, and optional run data, then operates the app UI.",
|
|
19
|
+
"Your job is to turn the user's scenario request into the exact ordered cases the agent should run.",
|
|
20
|
+
"Return JSON only with this shape:",
|
|
21
|
+
"{\"cases\":[{\"testName\":\"exact-test-name.dcua\",\"caseName\":\"test-name.dcua / data label\",\"testDataContext\":\"raw data text for this case\"}]}",
|
|
22
|
+
"Rules:",
|
|
23
|
+
"- testName must exactly match one of the provided candidate test names.",
|
|
24
|
+
"- Create an ordered flat list of cases to run.",
|
|
25
|
+
"- caseName is an ephemeral display name for logs and reports. Keep it short and human-readable.",
|
|
26
|
+
"- testDataContext must contain only the concrete data relevant to that case.",
|
|
27
|
+
"- Do not invent data that was not provided in test data, context, or the scenario prompt.",
|
|
28
|
+
"- If no extra data is needed for a case, use an empty string for testDataContext.",
|
|
29
|
+
].join("\n")
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
role: "user",
|
|
33
|
+
content: [
|
|
34
|
+
"Scenario prompt:",
|
|
35
|
+
scenarioPrompt,
|
|
36
|
+
"",
|
|
37
|
+
"App context:",
|
|
38
|
+
appContextText && appContextText.trim() ? appContextText.trim() : "(none)",
|
|
39
|
+
"",
|
|
40
|
+
"Project test data:",
|
|
41
|
+
testDataText && testDataText.trim() ? testDataText.trim() : "(none)",
|
|
42
|
+
"",
|
|
43
|
+
"Candidate tests:",
|
|
44
|
+
formatCandidateTests(tests),
|
|
45
|
+
].join("\n")
|
|
46
|
+
}
|
|
47
|
+
];
|
|
48
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { DEFAULT_TEST_DATA_TEMPLATE, isDefaultTestDataTemplate } from "../core/test-data.js";
|
|
2
|
+
export function buildTestDataImportMessages({ currentContent = "", importedFileName, importedFileText, instructions = "", }) {
|
|
3
|
+
const currentIsTemplateOnly = isDefaultTestDataTemplate(currentContent);
|
|
4
|
+
const userInstructions = typeof instructions === "string" && instructions.trim()
|
|
5
|
+
? instructions.trim()
|
|
6
|
+
: "(none)";
|
|
7
|
+
return [
|
|
8
|
+
{
|
|
9
|
+
role: "system",
|
|
10
|
+
content: [
|
|
11
|
+
"You convert imported data into durable project test data for computer-using-agent scenario runs.",
|
|
12
|
+
"The durable format is test-data.md: readable markdown that describes concrete entities and run data the agent can use.",
|
|
13
|
+
"Return JSON only with this shape:",
|
|
14
|
+
"{\"content\":\"# Test Data\\n...complete markdown...\"}",
|
|
15
|
+
"Rules:",
|
|
16
|
+
"- Return the complete replacement content for test-data.md, not a patch.",
|
|
17
|
+
"- Do not invent values that are not present in the current file, imported file, or user instructions.",
|
|
18
|
+
"- If currentContent is template-only, replace it with real imported data and do not keep commented template examples.",
|
|
19
|
+
"- If currentContent is not template-only, preserve its useful structure and add or update entities within it.",
|
|
20
|
+
"- For matrix-like CSV rows, prefer splitting repeated dimensions into separate entities when the structure is clear.",
|
|
21
|
+
"- If rows are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
|
|
22
|
+
"- User instructions can guide structure and naming, but content must remain concrete data, not test instructions.",
|
|
23
|
+
"- Output markdown only inside the JSON content string; do not use code fences.",
|
|
24
|
+
].join("\n")
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
role: "user",
|
|
28
|
+
content: [
|
|
29
|
+
"Current test-data.md status:",
|
|
30
|
+
currentIsTemplateOnly ? "template-only" : "existing user content",
|
|
31
|
+
"",
|
|
32
|
+
"Default template used for template-only detection:",
|
|
33
|
+
DEFAULT_TEST_DATA_TEMPLATE.trimEnd(),
|
|
34
|
+
"",
|
|
35
|
+
"Current test-data.md content:",
|
|
36
|
+
String(currentContent ?? "").trim() ? String(currentContent) : "(empty)",
|
|
37
|
+
"",
|
|
38
|
+
"Imported file:",
|
|
39
|
+
importedFileName,
|
|
40
|
+
"",
|
|
41
|
+
"Imported file text:",
|
|
42
|
+
importedFileText,
|
|
43
|
+
"",
|
|
44
|
+
"User import instructions:",
|
|
45
|
+
userInstructions,
|
|
46
|
+
].join("\n")
|
|
47
|
+
}
|
|
48
|
+
];
|
|
49
|
+
}
|