@exaudeus/workrail 0.1.10 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/services/validation-engine.d.ts +2 -0
- package/dist/application/services/validation-engine.js +66 -0
- package/dist/application/services/workflow-service.d.ts +2 -0
- package/dist/application/services/workflow-service.js +95 -0
- package/dist/infrastructure/storage/multi-directory-workflow-storage.js +7 -3
- package/dist/types/mcp-types.d.ts +22 -0
- package/dist/types/workflow-types.d.ts +14 -0
- package/dist/validation/response-validator.js +24 -2
- package/package.json +6 -2
- package/spec/workflow.schema.json +54 -93
- package/workflows/coding-task-workflow-with-loops.json +3 -3
- package/workflows/intelligent-test-case-generation.json +237 -0
|
@@ -41,4 +41,6 @@ export declare class ValidationEngine {
|
|
|
41
41
|
validateLoopStep(step: LoopStep, workflow: Workflow): ValidationResult;
|
|
42
42
|
validateWorkflow(workflow: Workflow): ValidationResult;
|
|
43
43
|
private isValidVariableName;
|
|
44
|
+
private validateStepFunctionCalls;
|
|
45
|
+
private isArgTypeValid;
|
|
44
46
|
}
|
|
@@ -297,6 +297,12 @@ class ValidationEngine {
|
|
|
297
297
|
issues.push(`Nested loops are not currently supported. Inline step '${inlineStep.id}' is a loop`);
|
|
298
298
|
suggestions.push('Refactor to avoid nested loops');
|
|
299
299
|
}
|
|
300
|
+
const callValidation = this.validateStepFunctionCalls(inlineStep, workflow.functionDefinitions || [], step.functionDefinitions || []);
|
|
301
|
+
if (!callValidation.valid) {
|
|
302
|
+
issues.push(...callValidation.issues.map(i => `Step '${inlineStep.id}': ${i}`));
|
|
303
|
+
if (callValidation.suggestions)
|
|
304
|
+
suggestions.push(...callValidation.suggestions);
|
|
305
|
+
}
|
|
300
306
|
}
|
|
301
307
|
}
|
|
302
308
|
}
|
|
@@ -358,6 +364,12 @@ class ValidationEngine {
|
|
|
358
364
|
if (!step.prompt) {
|
|
359
365
|
issues.push(`Step '${step.id}' missing required prompt`);
|
|
360
366
|
}
|
|
367
|
+
const callValidation = this.validateStepFunctionCalls(step, workflow.functionDefinitions || []);
|
|
368
|
+
if (!callValidation.valid) {
|
|
369
|
+
issues.push(...callValidation.issues.map(i => `Step '${step.id}': ${i}`));
|
|
370
|
+
if (callValidation.suggestions)
|
|
371
|
+
suggestions.push(...callValidation.suggestions);
|
|
372
|
+
}
|
|
361
373
|
}
|
|
362
374
|
}
|
|
363
375
|
const loopBodySteps = new Set();
|
|
@@ -392,5 +404,59 @@ class ValidationEngine {
|
|
|
392
404
|
isValidVariableName(name) {
|
|
393
405
|
return /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(name);
|
|
394
406
|
}
|
|
407
|
+
validateStepFunctionCalls(step, workflowDefs, loopDefs = []) {
|
|
408
|
+
const allDefs = {};
|
|
409
|
+
const addDefs = (defs) => {
|
|
410
|
+
(defs || []).forEach(d => { allDefs[d.name] = d; });
|
|
411
|
+
};
|
|
412
|
+
addDefs(workflowDefs);
|
|
413
|
+
addDefs(loopDefs);
|
|
414
|
+
addDefs(step.functionDefinitions);
|
|
415
|
+
const issues = [];
|
|
416
|
+
const suggestions = [];
|
|
417
|
+
const calls = step.functionCalls;
|
|
418
|
+
if (!calls || calls.length === 0) {
|
|
419
|
+
return { valid: true, issues: [], suggestions: [] };
|
|
420
|
+
}
|
|
421
|
+
for (const call of calls) {
|
|
422
|
+
const def = allDefs[call.name];
|
|
423
|
+
if (!def) {
|
|
424
|
+
issues.push(`Unknown function '${call.name}' in functionCalls`);
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
if (def.parameters && def.parameters.length > 0) {
|
|
428
|
+
const args = call.args || {};
|
|
429
|
+
for (const param of def.parameters) {
|
|
430
|
+
if (param.required && !(param.name in args)) {
|
|
431
|
+
issues.push(`Missing required parameter '${param.name}' for function '${call.name}'`);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
for (const [argName, argValue] of Object.entries(args)) {
|
|
435
|
+
const spec = def.parameters.find(p => p.name === argName);
|
|
436
|
+
if (!spec) {
|
|
437
|
+
suggestions.push(`Unknown argument '${argName}' for function '${call.name}'`);
|
|
438
|
+
continue;
|
|
439
|
+
}
|
|
440
|
+
if (!this.isArgTypeValid(argValue, spec)) {
|
|
441
|
+
issues.push(`Invalid type for '${call.name}.${argName}': expected ${spec.type}`);
|
|
442
|
+
}
|
|
443
|
+
if (spec.enum && !spec.enum.includes(argValue)) {
|
|
444
|
+
issues.push(`Invalid value for '${call.name}.${argName}': must be one of ${spec.enum.join(', ')}`);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
return { valid: issues.length === 0, issues, suggestions };
|
|
450
|
+
}
|
|
451
|
+
isArgTypeValid(value, spec) {
|
|
452
|
+
switch (spec.type) {
|
|
453
|
+
case 'string': return typeof value === 'string';
|
|
454
|
+
case 'number': return typeof value === 'number' && Number.isFinite(value);
|
|
455
|
+
case 'boolean': return typeof value === 'boolean';
|
|
456
|
+
case 'array': return Array.isArray(value);
|
|
457
|
+
case 'object': return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
458
|
+
default: return true;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
395
461
|
}
|
|
396
462
|
exports.ValidationEngine = ValidationEngine;
|
|
@@ -34,6 +34,8 @@ export declare class DefaultWorkflowService implements WorkflowService {
|
|
|
34
34
|
context?: ConditionContext;
|
|
35
35
|
}>;
|
|
36
36
|
private buildStepPrompt;
|
|
37
|
+
private collectConditionVars;
|
|
38
|
+
private collectEqualsValues;
|
|
37
39
|
private findLoopStepById;
|
|
38
40
|
validateStepOutput(workflowId: string, stepId: string, output: string): Promise<{
|
|
39
41
|
valid: boolean;
|
|
@@ -189,6 +189,61 @@ class DefaultWorkflowService {
|
|
|
189
189
|
}
|
|
190
190
|
return this.getNextStep(workflowId, completedSteps, loopStartSizeCheck.context);
|
|
191
191
|
}
|
|
192
|
+
if (!nextStep) {
|
|
193
|
+
const remainingConditionalSteps = workflow.steps.filter((step) => {
|
|
194
|
+
if (completed.includes(step.id))
|
|
195
|
+
return false;
|
|
196
|
+
if (loopBodySteps.has(step.id))
|
|
197
|
+
return false;
|
|
198
|
+
return !!step.runCondition;
|
|
199
|
+
});
|
|
200
|
+
if (remainingConditionalSteps.length > 0) {
|
|
201
|
+
const requiredVars = new Set();
|
|
202
|
+
const allowedValues = {};
|
|
203
|
+
for (const step of remainingConditionalSteps) {
|
|
204
|
+
const condition = step.runCondition;
|
|
205
|
+
this.collectConditionVars(condition, requiredVars);
|
|
206
|
+
this.collectEqualsValues(condition, allowedValues);
|
|
207
|
+
}
|
|
208
|
+
const issues = [];
|
|
209
|
+
for (const variableName of requiredVars) {
|
|
210
|
+
const currentValue = enhancedContext[variableName];
|
|
211
|
+
const allowed = allowedValues[variableName]
|
|
212
|
+
? Array.from(allowedValues[variableName])
|
|
213
|
+
: [];
|
|
214
|
+
if (currentValue === undefined || currentValue === null || currentValue === '') {
|
|
215
|
+
if (allowed.length > 0) {
|
|
216
|
+
issues.push(`Set '${variableName}' to one of: ${allowed.map(v => `'${v}'`).join(', ')}`);
|
|
217
|
+
}
|
|
218
|
+
else {
|
|
219
|
+
issues.push(`Provide a value for '${variableName}'`);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
else if (allowed.length > 0) {
|
|
223
|
+
const matchesExactly = allowed.some(v => v === String(currentValue));
|
|
224
|
+
const matchesCaseInsensitive = allowed.some(v => v.toLowerCase() === String(currentValue).toLowerCase());
|
|
225
|
+
if (!matchesExactly) {
|
|
226
|
+
if (matchesCaseInsensitive) {
|
|
227
|
+
issues.push(`Normalize casing for '${variableName}': use one of ${allowed.map(v => `'${v}'`).join(', ')} (current '${currentValue}')`);
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
issues.push(`Adjust '${variableName}' to one of: ${allowed.map(v => `'${v}'`).join(', ')} (current '${currentValue}')`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
if (issues.length > 0) {
|
|
236
|
+
return {
|
|
237
|
+
step: null,
|
|
238
|
+
guidance: {
|
|
239
|
+
prompt: `No eligible step due to unmet conditions. Please update context:\n- ${issues.join('\n- ')}`
|
|
240
|
+
},
|
|
241
|
+
isComplete: false,
|
|
242
|
+
context: enhancedContext
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
192
247
|
const isComplete = !nextStep;
|
|
193
248
|
let finalPrompt = 'Workflow complete.';
|
|
194
249
|
if (nextStep) {
|
|
@@ -232,6 +287,46 @@ class DefaultWorkflowService {
|
|
|
232
287
|
}
|
|
233
288
|
return finalPrompt;
|
|
234
289
|
}
|
|
290
|
+
collectConditionVars(condition, sink) {
|
|
291
|
+
if (!condition || typeof condition !== 'object')
|
|
292
|
+
return;
|
|
293
|
+
if (typeof condition.var === 'string' && condition.var.length > 0) {
|
|
294
|
+
sink.add(condition.var);
|
|
295
|
+
}
|
|
296
|
+
if (Array.isArray(condition.and)) {
|
|
297
|
+
for (const sub of condition.and)
|
|
298
|
+
this.collectConditionVars(sub, sink);
|
|
299
|
+
}
|
|
300
|
+
if (Array.isArray(condition.or)) {
|
|
301
|
+
for (const sub of condition.or)
|
|
302
|
+
this.collectConditionVars(sub, sink);
|
|
303
|
+
}
|
|
304
|
+
if (condition.not)
|
|
305
|
+
this.collectConditionVars(condition.not, sink);
|
|
306
|
+
}
|
|
307
|
+
collectEqualsValues(condition, sink) {
|
|
308
|
+
if (!condition || typeof condition !== 'object')
|
|
309
|
+
return;
|
|
310
|
+
if (typeof condition.var === 'string' && Object.prototype.hasOwnProperty.call(condition, 'equals')) {
|
|
311
|
+
const variableName = condition.var;
|
|
312
|
+
const value = condition.equals;
|
|
313
|
+
if (value !== undefined && value !== null) {
|
|
314
|
+
if (!sink[variableName])
|
|
315
|
+
sink[variableName] = new Set();
|
|
316
|
+
sink[variableName].add(String(value));
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
if (Array.isArray(condition.and)) {
|
|
320
|
+
for (const sub of condition.and)
|
|
321
|
+
this.collectEqualsValues(sub, sink);
|
|
322
|
+
}
|
|
323
|
+
if (Array.isArray(condition.or)) {
|
|
324
|
+
for (const sub of condition.or)
|
|
325
|
+
this.collectEqualsValues(sub, sink);
|
|
326
|
+
}
|
|
327
|
+
if (condition.not)
|
|
328
|
+
this.collectEqualsValues(condition.not, sink);
|
|
329
|
+
}
|
|
235
330
|
findLoopStepById(workflow, stepId) {
|
|
236
331
|
const step = workflow.steps.find(s => s.id === stepId);
|
|
237
332
|
return step && (0, workflow_types_1.isLoopStep)(step) ? step : null;
|
|
@@ -20,9 +20,13 @@ class MultiDirectoryWorkflowStorage {
|
|
|
20
20
|
resolveDirectories(options) {
|
|
21
21
|
const directories = [];
|
|
22
22
|
if (!options.excludeBundled) {
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
const bundledSrcDir = path_1.default.resolve(__dirname, '../../../../workflows');
|
|
24
|
+
const bundledDistDir = path_1.default.resolve(__dirname, '../../../workflows');
|
|
25
|
+
if ((0, fs_1.existsSync)(bundledSrcDir)) {
|
|
26
|
+
directories.push(bundledSrcDir);
|
|
27
|
+
}
|
|
28
|
+
else if ((0, fs_1.existsSync)(bundledDistDir)) {
|
|
29
|
+
directories.push(bundledDistDir);
|
|
26
30
|
}
|
|
27
31
|
}
|
|
28
32
|
if (!options.excludeUser) {
|
|
@@ -118,6 +118,24 @@ export interface WorkflowGetRequest extends MCPToolCallRequest {
|
|
|
118
118
|
id: string;
|
|
119
119
|
};
|
|
120
120
|
}
|
|
121
|
+
export interface FunctionParameter {
|
|
122
|
+
name: string;
|
|
123
|
+
type: 'string' | 'number' | 'boolean' | 'array' | 'object';
|
|
124
|
+
required?: boolean;
|
|
125
|
+
description?: string;
|
|
126
|
+
enum?: Array<string | number | boolean>;
|
|
127
|
+
default?: unknown;
|
|
128
|
+
}
|
|
129
|
+
export interface FunctionDefinition {
|
|
130
|
+
name: string;
|
|
131
|
+
definition: string;
|
|
132
|
+
parameters?: FunctionParameter[];
|
|
133
|
+
scope?: 'workflow' | 'loop' | 'step';
|
|
134
|
+
}
|
|
135
|
+
export interface FunctionCall {
|
|
136
|
+
name: string;
|
|
137
|
+
args: Record<string, unknown>;
|
|
138
|
+
}
|
|
121
139
|
export interface WorkflowStep {
|
|
122
140
|
id: string;
|
|
123
141
|
title: string;
|
|
@@ -127,6 +145,9 @@ export interface WorkflowStep {
|
|
|
127
145
|
askForFiles?: boolean;
|
|
128
146
|
requireConfirmation?: boolean;
|
|
129
147
|
runCondition?: object;
|
|
148
|
+
functionDefinitions?: FunctionDefinition[];
|
|
149
|
+
functionCalls?: FunctionCall[];
|
|
150
|
+
functionReferences?: string[];
|
|
130
151
|
}
|
|
131
152
|
export interface Workflow {
|
|
132
153
|
id: string;
|
|
@@ -137,6 +158,7 @@ export interface Workflow {
|
|
|
137
158
|
clarificationPrompts?: string[];
|
|
138
159
|
steps: WorkflowStep[];
|
|
139
160
|
metaGuidance?: string[];
|
|
161
|
+
functionDefinitions?: FunctionDefinition[];
|
|
140
162
|
}
|
|
141
163
|
export interface WorkflowGetResponse extends MCPToolCallResponse {
|
|
142
164
|
result: Workflow;
|
|
@@ -20,6 +20,7 @@ export interface WorkflowStep {
|
|
|
20
20
|
requireConfirmation?: boolean;
|
|
21
21
|
runCondition?: object;
|
|
22
22
|
functionDefinitions?: FunctionDefinition[];
|
|
23
|
+
functionCalls?: FunctionCall[];
|
|
23
24
|
functionReferences?: string[];
|
|
24
25
|
}
|
|
25
26
|
export interface LoopStep extends WorkflowStep {
|
|
@@ -82,11 +83,24 @@ export interface LoopPhaseReference {
|
|
|
82
83
|
totalSteps: number;
|
|
83
84
|
functionDefinitions?: FunctionDefinition[];
|
|
84
85
|
}
|
|
86
|
+
export interface FunctionParameter {
|
|
87
|
+
name: string;
|
|
88
|
+
type: 'string' | 'number' | 'boolean' | 'array' | 'object';
|
|
89
|
+
required?: boolean;
|
|
90
|
+
description?: string;
|
|
91
|
+
enum?: Array<string | number | boolean>;
|
|
92
|
+
default?: unknown;
|
|
93
|
+
}
|
|
85
94
|
export interface FunctionDefinition {
|
|
86
95
|
name: string;
|
|
87
96
|
definition: string;
|
|
97
|
+
parameters?: FunctionParameter[];
|
|
88
98
|
scope?: 'workflow' | 'loop' | 'step';
|
|
89
99
|
}
|
|
100
|
+
export interface FunctionCall {
|
|
101
|
+
name: string;
|
|
102
|
+
args: Record<string, unknown>;
|
|
103
|
+
}
|
|
90
104
|
export declare function isLoopStep(step: WorkflowStep | LoopStep): step is LoopStep;
|
|
91
105
|
export declare function isFirstLoopIteration(context: EnhancedContext | OptimizedLoopContext): boolean;
|
|
92
106
|
export interface WorkflowValidationResult {
|
|
@@ -11,6 +11,24 @@ const workflowSummarySchema = zod_1.z.object({
|
|
|
11
11
|
category: zod_1.z.string(),
|
|
12
12
|
version: zod_1.z.string()
|
|
13
13
|
});
|
|
14
|
+
const functionParameterSchema = zod_1.z.object({
|
|
15
|
+
name: zod_1.z.string(),
|
|
16
|
+
type: zod_1.z.enum(['string', 'number', 'boolean', 'array', 'object']),
|
|
17
|
+
required: zod_1.z.boolean().optional(),
|
|
18
|
+
description: zod_1.z.string().optional(),
|
|
19
|
+
enum: zod_1.z.array(zod_1.z.union([zod_1.z.string(), zod_1.z.number(), zod_1.z.boolean()])).optional(),
|
|
20
|
+
default: zod_1.z.any().optional()
|
|
21
|
+
});
|
|
22
|
+
const functionDefinitionSchema = zod_1.z.object({
|
|
23
|
+
name: zod_1.z.string(),
|
|
24
|
+
definition: zod_1.z.string(),
|
|
25
|
+
parameters: zod_1.z.array(functionParameterSchema).optional(),
|
|
26
|
+
scope: zod_1.z.enum(['workflow', 'loop', 'step']).optional()
|
|
27
|
+
});
|
|
28
|
+
const functionCallSchema = zod_1.z.object({
|
|
29
|
+
name: zod_1.z.string(),
|
|
30
|
+
args: zod_1.z.record(zod_1.z.any())
|
|
31
|
+
});
|
|
14
32
|
const workflowStepSchema = zod_1.z.object({
|
|
15
33
|
id: zod_1.z.string().regex(idRegex),
|
|
16
34
|
title: zod_1.z.string(),
|
|
@@ -19,7 +37,10 @@ const workflowStepSchema = zod_1.z.object({
|
|
|
19
37
|
guidance: zod_1.z.array(zod_1.z.string()).optional(),
|
|
20
38
|
askForFiles: zod_1.z.boolean().optional(),
|
|
21
39
|
requireConfirmation: zod_1.z.boolean().optional(),
|
|
22
|
-
runCondition: zod_1.z.object({}).optional()
|
|
40
|
+
runCondition: zod_1.z.object({}).optional(),
|
|
41
|
+
functionDefinitions: zod_1.z.array(functionDefinitionSchema).optional(),
|
|
42
|
+
functionCalls: zod_1.z.array(functionCallSchema).optional(),
|
|
43
|
+
functionReferences: zod_1.z.array(zod_1.z.string()).optional()
|
|
23
44
|
});
|
|
24
45
|
const workflowSchema = zod_1.z.object({
|
|
25
46
|
id: zod_1.z.string().regex(idRegex),
|
|
@@ -29,7 +50,8 @@ const workflowSchema = zod_1.z.object({
|
|
|
29
50
|
preconditions: zod_1.z.array(zod_1.z.string()).optional(),
|
|
30
51
|
clarificationPrompts: zod_1.z.array(zod_1.z.string()).optional(),
|
|
31
52
|
steps: zod_1.z.array(workflowStepSchema),
|
|
32
|
-
metaGuidance: zod_1.z.array(zod_1.z.string()).optional()
|
|
53
|
+
metaGuidance: zod_1.z.array(zod_1.z.string()).optional(),
|
|
54
|
+
functionDefinitions: zod_1.z.array(functionDefinitionSchema).optional()
|
|
33
55
|
});
|
|
34
56
|
const workflowMetadataSchema = zod_1.z.object({
|
|
35
57
|
id: zod_1.z.string().regex(idRegex),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@exaudeus/workrail",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "MCP server for structured workflow orchestration and step-by-step task guidance",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"bin": {
|
|
@@ -16,7 +16,8 @@
|
|
|
16
16
|
"release": "bash -c './scripts/release.sh'",
|
|
17
17
|
"dev": "npm run build && node dist/mcp-server.js",
|
|
18
18
|
"prepare": "npm run build",
|
|
19
|
-
"watch": "tsc --watch"
|
|
19
|
+
"watch": "tsc --watch",
|
|
20
|
+
"preinstall": "node -e \"const v=parseInt(process.versions.node.split('.') [0],10); if(v<20){console.error('WorkRail requires Node.js >=20. Current: '+process.versions.node+'\\nPlease upgrade: https://nodejs.org/'); process.exit(1);}\""
|
|
20
21
|
},
|
|
21
22
|
"dependencies": {
|
|
22
23
|
"@modelcontextprotocol/sdk": "^0.5.0",
|
|
@@ -42,5 +43,8 @@
|
|
|
42
43
|
"@types/node": "^20.19.9",
|
|
43
44
|
"@types/semver": "^7.7.0",
|
|
44
45
|
"typescript": "^5.8.3"
|
|
46
|
+
},
|
|
47
|
+
"engines": {
|
|
48
|
+
"node": ">=20"
|
|
45
49
|
}
|
|
46
50
|
}
|
|
@@ -90,6 +90,13 @@
|
|
|
90
90
|
"type": "string",
|
|
91
91
|
"description": "The function definition explaining what it does"
|
|
92
92
|
},
|
|
93
|
+
"parameters": {
|
|
94
|
+
"type": "array",
|
|
95
|
+
"description": "Optional machine-readable parameter specifications for this function.",
|
|
96
|
+
"items": {
|
|
97
|
+
"$ref": "#/$defs/functionParameter"
|
|
98
|
+
}
|
|
99
|
+
},
|
|
93
100
|
"scope": {
|
|
94
101
|
"type": "string",
|
|
95
102
|
"enum": ["workflow", "loop", "step"],
|
|
@@ -117,23 +124,34 @@
|
|
|
117
124
|
"minLength": 3,
|
|
118
125
|
"maxLength": 64
|
|
119
126
|
},
|
|
120
|
-
"
|
|
127
|
+
"functionParameter": {
|
|
121
128
|
"type": "object",
|
|
129
|
+
"description": "Specification for a single function parameter.",
|
|
122
130
|
"properties": {
|
|
123
|
-
"name": {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
131
|
+
"name": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" },
|
|
132
|
+
"type": { "type": "string", "enum": ["string", "number", "boolean", "array", "object"] },
|
|
133
|
+
"required": { "type": "boolean" },
|
|
134
|
+
"description": { "type": "string" },
|
|
135
|
+
"enum": {
|
|
136
|
+
"type": "array",
|
|
137
|
+
"items": { "oneOf": [ {"type": "string"}, {"type": "number"}, {"type": "boolean"} ] },
|
|
138
|
+
"minItems": 1
|
|
127
139
|
},
|
|
128
|
-
"
|
|
129
|
-
|
|
130
|
-
|
|
140
|
+
"default": {}
|
|
141
|
+
},
|
|
142
|
+
"required": ["name", "type"],
|
|
143
|
+
"additionalProperties": false
|
|
144
|
+
},
|
|
145
|
+
"functionDefinition": {
|
|
146
|
+
"type": "object",
|
|
147
|
+
"properties": {
|
|
148
|
+
"name": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" },
|
|
149
|
+
"definition": { "type": "string" },
|
|
150
|
+
"parameters": {
|
|
151
|
+
"type": "array",
|
|
152
|
+
"items": { "$ref": "#/$defs/functionParameter" }
|
|
131
153
|
},
|
|
132
|
-
"scope": {
|
|
133
|
-
"type": "string",
|
|
134
|
-
"enum": ["workflow", "loop", "step"],
|
|
135
|
-
"description": "Scope where this function is available"
|
|
136
|
-
}
|
|
154
|
+
"scope": { "type": "string", "enum": ["workflow", "loop", "step"] }
|
|
137
155
|
},
|
|
138
156
|
"required": ["name", "definition"],
|
|
139
157
|
"additionalProperties": false
|
|
@@ -141,87 +159,21 @@
|
|
|
141
159
|
"standardStep": {
|
|
142
160
|
"type": "object",
|
|
143
161
|
"properties": {
|
|
144
|
-
"id": {
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
},
|
|
148
|
-
"
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
},
|
|
154
|
-
"
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
"minLength": 1,
|
|
158
|
-
"maxLength": 2048
|
|
159
|
-
},
|
|
160
|
-
"agentRole": {
|
|
161
|
-
"type": "string",
|
|
162
|
-
"description": "Optional behavioral instructions for AI agents defining how they should approach and execute this step. This content is not shown to users.",
|
|
163
|
-
"minLength": 10,
|
|
164
|
-
"maxLength": 1024
|
|
165
|
-
},
|
|
166
|
-
"guidance": {
|
|
167
|
-
"type": "array",
|
|
168
|
-
"description": "Optional array of strings providing tactical advice for this step.",
|
|
169
|
-
"items": {
|
|
170
|
-
"type": "string"
|
|
171
|
-
}
|
|
172
|
-
},
|
|
173
|
-
"askForFiles": {
|
|
174
|
-
"type": "boolean",
|
|
175
|
-
"description": "Whether the agent should ask for relevant files before executing the step.",
|
|
176
|
-
"default": false
|
|
177
|
-
},
|
|
178
|
-
"hasValidation": {
|
|
179
|
-
"type": "boolean",
|
|
180
|
-
"description": "Whether this step has validation logic that should be called. Set to true for steps with validationCriteria to optimize execution performance.",
|
|
181
|
-
"default": false
|
|
182
|
-
},
|
|
183
|
-
"requireConfirmation": {
|
|
184
|
-
"$ref": "#/$defs/confirmationRule"
|
|
185
|
-
},
|
|
186
|
-
"runCondition": {
|
|
187
|
-
"$ref": "#/$defs/condition"
|
|
188
|
-
},
|
|
189
|
-
"validationCriteria": {
|
|
190
|
-
"description": "Optional validation rules to check step output quality. Can be an array of rules (all must pass) or a composition object with logical operators.",
|
|
191
|
-
"oneOf": [
|
|
192
|
-
{
|
|
193
|
-
"type": "array",
|
|
194
|
-
"description": "Array of validation rules (all must pass)",
|
|
195
|
-
"items": {
|
|
196
|
-
"$ref": "#/$defs/validationRule"
|
|
197
|
-
}
|
|
198
|
-
},
|
|
199
|
-
{
|
|
200
|
-
"$ref": "#/$defs/validationComposition"
|
|
201
|
-
}
|
|
202
|
-
]
|
|
203
|
-
},
|
|
204
|
-
"functionDefinitions": {
|
|
205
|
-
"type": "array",
|
|
206
|
-
"description": "Function definitions specific to this step. Agents can reference these for understanding step-specific patterns.",
|
|
207
|
-
"items": {
|
|
208
|
-
"$ref": "#/$defs/functionDefinition"
|
|
209
|
-
}
|
|
210
|
-
},
|
|
211
|
-
"functionReferences": {
|
|
212
|
-
"type": "array",
|
|
213
|
-
"description": "References to functions defined at workflow or loop level that this step uses.",
|
|
214
|
-
"items": {
|
|
215
|
-
"type": "string",
|
|
216
|
-
"pattern": "^[a-zA-Z_][a-zA-Z0-9_]*\\(\\)$"
|
|
217
|
-
}
|
|
218
|
-
}
|
|
162
|
+
"id": { "$ref": "#/$defs/stepId", "description": "Unique identifier for the step" },
|
|
163
|
+
"title": { "type": "string", "minLength": 1, "maxLength": 128 },
|
|
164
|
+
"prompt": { "type": "string", "minLength": 1, "maxLength": 2048 },
|
|
165
|
+
"agentRole": { "type": "string", "minLength": 10, "maxLength": 1024 },
|
|
166
|
+
"guidance": { "type": "array", "items": { "type": "string" } },
|
|
167
|
+
"askForFiles": { "type": "boolean", "default": false },
|
|
168
|
+
"hasValidation": { "type": "boolean", "default": false },
|
|
169
|
+
"requireConfirmation": { "$ref": "#/$defs/confirmationRule" },
|
|
170
|
+
"runCondition": { "$ref": "#/$defs/condition" },
|
|
171
|
+
"validationCriteria": { "oneOf": [ { "type": "array", "items": { "$ref": "#/$defs/validationRule" } }, { "$ref": "#/$defs/validationComposition" } ] },
|
|
172
|
+
"functionDefinitions": { "type": "array", "items": { "$ref": "#/$defs/functionDefinition" } },
|
|
173
|
+
"functionCalls": { "type": "array", "items": { "$ref": "#/$defs/functionCall" } },
|
|
174
|
+
"functionReferences": { "type": "array", "items": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*\\(\\)$" } }
|
|
219
175
|
},
|
|
220
|
-
"required": [
|
|
221
|
-
"id",
|
|
222
|
-
"title",
|
|
223
|
-
"prompt"
|
|
224
|
-
],
|
|
176
|
+
"required": ["id", "title", "prompt"],
|
|
225
177
|
"additionalProperties": false
|
|
226
178
|
},
|
|
227
179
|
"loopStep": {
|
|
@@ -563,6 +515,15 @@
|
|
|
563
515
|
"$ref": "#/$defs/validationComposition"
|
|
564
516
|
}
|
|
565
517
|
]
|
|
518
|
+
},
|
|
519
|
+
"functionCall": {
|
|
520
|
+
"type": "object",
|
|
521
|
+
"properties": {
|
|
522
|
+
"name": { "type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" },
|
|
523
|
+
"args": { "type": "object", "additionalProperties": true }
|
|
524
|
+
},
|
|
525
|
+
"required": ["name", "args"],
|
|
526
|
+
"additionalProperties": false
|
|
566
527
|
}
|
|
567
528
|
}
|
|
568
529
|
}
|
|
@@ -431,7 +431,7 @@
|
|
|
431
431
|
{
|
|
432
432
|
"id": "phase-6-prep",
|
|
433
433
|
"title": "PREP: Prepare for Step {{currentStepNumber + 1}}",
|
|
434
|
-
"prompt": "**PREPARATION PHASE for Step {{currentStepNumber + 1}}/{{totalImplementationSteps}}**\n\nBefore implementing this step, you must first PREPARE:\n\n1. **Read implementation_plan.md** and locate step #{{currentStepNumber + 1}}\n2. **Extract step details**: Note the title, description, and expected outputs\n3. **Confirm prerequisites**: Verify the previous step (if any) was completed correctly\n4. **Validate current state**: Ensure the plan for this step is still valid in the current codebase\n5. **List requirements**: Identify all required inputs, files, or dependencies\n\n**BRANCH SETUP (first iteration only):** If this is the first implementation step ({{currentStepNumber}} === 0):\n- Check git availability with 'git status'\n- If available, create feature branch: 'git checkout -b wip-[unique-task-id]'\n- Track the featureBranch variable for later use\n- If git unavailable: Skip branching, log in CONTEXT.md\n\n**
|
|
434
|
+
"prompt": "**PREPARATION PHASE for Step {{currentStepNumber + 1}}/{{totalImplementationSteps}}**\n\nBefore implementing this step, you must first PREPARE:\n\n1. **Read implementation_plan.md** and locate step #{{currentStepNumber + 1}}\n2. **Extract step details**: Note the title, description, and expected outputs\n3. **Confirm prerequisites**: Verify the previous step (if any) was completed correctly\n4. **Validate current state**: Ensure the plan for this step is still valid in the current codebase\n5. **List requirements**: Identify all required inputs, files, or dependencies\n\n**BRANCH SETUP (first iteration only):** If this is the first implementation step ({{currentStepNumber}} === 0):\n- Check git availability with 'git status'\n- If available, create feature branch: 'git checkout -b wip-[unique-task-id]'\n- Track the featureBranch variable for later use\n- If git unavailable: Skip branching, log in CONTEXT.md\n\n**Do not proceed if anything is unclear or missing.**",
|
|
435
435
|
"agentRole": "You are preparing to implement a specific step from the plan. Be meticulous in verifying all prerequisites are met before proceeding.",
|
|
436
436
|
"guidance": [
|
|
437
437
|
"This is step {{currentStepNumber + 1}} of {{totalImplementationSteps}} total implementation steps",
|
|
@@ -444,7 +444,7 @@
|
|
|
444
444
|
{
|
|
445
445
|
"id": "phase-6-implement",
|
|
446
446
|
"title": "IMPLEMENT: Execute Step {{currentStepNumber + 1}}",
|
|
447
|
-
"prompt": "**IMPLEMENTATION PHASE for Step {{currentStepNumber + 1}}/{{totalImplementationSteps}}**\n\nNow implement the step you just prepared for:\n\n**Instructions:**\n1. Re-read step #{{currentStepNumber + 1}} from implementation_plan.md\n2. Focus only on this single step\n3. useTools() to make code changes\n4. Follow quality standards\n5. Adapt to unexpected discoveries\n6. createFile() for ALL code changes\n\n**Remember:** applyUserRules() and matchPatterns() throughout.\n\n**Progress Tracking:**\n- This is step {{currentStepNumber + 1}} of {{totalImplementationSteps}}\n- If we've done > 20 steps total, pause for user intervention\n\n**CONTEXT UPDATES:** If this is every 3rd step ({{currentStepNumber + 1}} % 3 === 0):\n- Update CONTEXT.md\n- addResumptionJson(phase-6-implement)\n- updateDecisionLog()\n- List files modified with line ranges\n\n
|
|
447
|
+
"prompt": "**IMPLEMENTATION PHASE for Step {{currentStepNumber + 1}}/{{totalImplementationSteps}}**\n\nNow implement the step you just prepared for:\n\n**Instructions:**\n1. Re-read step #{{currentStepNumber + 1}} from implementation_plan.md\n2. Focus only on this single step\n3. useTools() to make code changes\n4. Follow quality standards\n5. Adapt to unexpected discoveries\n6. createFile() for ALL code changes\n\n**Remember:** applyUserRules() and matchPatterns() throughout.\n\n**Progress Tracking:**\n- This is step {{currentStepNumber + 1}} of {{totalImplementationSteps}}\n- If we've done > 20 steps total, pause for user intervention\n\n**CONTEXT UPDATES:** If this is every 3rd step ({{currentStepNumber + 1}} % 3 === 0):\n- Update CONTEXT.md\n- addResumptionJson(phase-6-implement)\n- updateDecisionLog()\n- List files modified with line ranges\n\n",
|
|
448
448
|
"agentRole": "You are implementing a specific step from the approved plan. Focus on precise execution while maintaining code quality.",
|
|
449
449
|
"guidance": [
|
|
450
450
|
"Implement only what this step requires",
|
|
@@ -456,7 +456,7 @@
|
|
|
456
456
|
{
|
|
457
457
|
"id": "phase-6-verify",
|
|
458
458
|
"title": "VERIFY: Validate Step {{currentStepNumber + 1}}",
|
|
459
|
-
"prompt": "**VERIFICATION PHASE for Step {{currentStepNumber + 1}}/{{totalImplementationSteps}}**\n\nVerify the implementation is complete and correct:\n\n**Required:** verifyImplementation()\n\n**COMMIT Decision (if all passes):**\n- checkAutomation(commit)\n- gitCommit(type, scope: description)\n- If git unavailable: Log in CONTEXT.md\n\n**FAILURE PROTOCOL:** If verification fails after 2 attempts:\n1. Do not try a third time\n2. Fall back to alternative tools\n3. updateDecisionLog() with failure details\n4. Present summary and recommendations\n5. Set 'verificationFailed' context variable to true\n\n
|
|
459
|
+
"prompt": "**VERIFICATION PHASE for Step {{currentStepNumber + 1}}/{{totalImplementationSteps}}**\n\nVerify the implementation is complete and correct:\n\n**Required:** verifyImplementation()\n\n**COMMIT Decision (if all passes):**\n- checkAutomation(commit)\n- gitCommit(type, scope: description)\n- If git unavailable: Log in CONTEXT.md\n\n**FAILURE PROTOCOL:** If verification fails after 2 attempts:\n1. Do not try a third time\n2. Fall back to alternative tools\n3. updateDecisionLog() with failure details\n4. Present summary and recommendations\n5. Set 'verificationFailed' context variable to true\n\n",
|
|
460
460
|
"agentRole": "You are verifying the implementation meets all quality standards. Be thorough but respect failure bounds.",
|
|
461
461
|
"guidance": [
|
|
462
462
|
"All three verification steps must pass",
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "intelligent-test-case-generation",
|
|
3
|
+
"name": "Intelligent Test Case Generation from Tickets",
|
|
4
|
+
"version": "0.0.1",
|
|
5
|
+
"description": "Transforms ticket requirements into systematic test cases using evidence-driven analysis, dual-brain processing (NLP + LLM), document discovery, and progressive scenario expansion. Produces integration and end-to-end tests optimized for developer readability and LLM consumption with confidence scoring and validation loops.",
|
|
6
|
+
"preconditions": [
|
|
7
|
+
"Agent has access to codebase analysis tools (grep, file readers, semantic search)",
|
|
8
|
+
"User can provide ticket information in any standard format (JIRA, GitHub Issues, etc.)",
|
|
9
|
+
"Agent has access to document analysis capabilities for BRDs, architecture docs, specs",
|
|
10
|
+
"Workflow execution environment supports confidence scoring and validation loops"
|
|
11
|
+
],
|
|
12
|
+
"clarificationPrompts": [
|
|
13
|
+
"What ticket system are you using? (JIRA, GitHub Issues, Azure DevOps, etc.)",
|
|
14
|
+
"What's your team's experience level with test case creation? (Mixed junior/senior, mostly senior, mostly junior)",
|
|
15
|
+
"Do you have existing test patterns or frameworks we should align with?",
|
|
16
|
+
"What automation level do you prefer? (High: auto-approve >8.0 confidence, Medium: standard confirmations, Low: extra validation)",
|
|
17
|
+
"Are there specific document repositories we should search? (Confluence, SharePoint, Git repos, etc.)"
|
|
18
|
+
],
|
|
19
|
+
"metaGuidance": [
|
|
20
|
+
"PAUSE POLICY: If required variables are missing and no assumptions accepted, pause and ask targeted questions before proceeding.",
|
|
21
|
+
"EVIDENCE POLICY: If evidenceScore < threshold, request more sources and do not proceed until satisfied or assumptions are accepted.",
|
|
22
|
+
"ASSUMPTION POLICY: Proceed with documented assumptions only after explicit confirmation; record risks and mitigation.",
|
|
23
|
+
"DUAL-BRAIN PROCESSING: Always use both structured NLP extraction AND creative LLM expansion for comprehensive coverage.",
|
|
24
|
+
"CONFIDENCE CALIBRATION: Use mathematical confidence framework with adjustable thresholds based on automation level.",
|
|
25
|
+
"LIVING DOCUMENTATION: Maintain TEST_CONTEXT.md throughout workflow. Update after major milestones for session continuity.",
|
|
26
|
+
"HYPOTHESIS-DRIVEN TESTING: Generate test hypotheses first, then validate with evidence.",
|
|
27
|
+
"**FUNCTION DEFINITIONS:** fun analyzeTicket(depth=1|2|3) = 'Analyze ticket at specified depth levels: L1=requirements, L2=dependencies, L3=system impact. Extract entities, actions, data flows, acceptance criteria.'",
|
|
28
|
+
"fun evaluateReadiness() = 'Compute requiredVarsFound, missingVars[], completenessScore (0-1) from context. Set proceed/question/assumption flags.'",
|
|
29
|
+
"fun collectEvidence(minimumSources=3) = 'Gather supporting evidence for hypotheses. Score evidence quality 1-10. Document sources and confidence levels.'",
|
|
30
|
+
"fun discoverDocuments(types) = 'Search for missing documentation via semantic search. Types may include BRDs, ArchitectureDocs, ApiSpecs, ExistingTests. Score relevance and completeness.'",
|
|
31
|
+
"fun scoreAndValidate(autoApproveThreshold=8.0) = 'Score item using criteria (1-10). Apply gates: >8.0 auto-approve; 6.0–7.9 review; <6.0 require additional evidence.'",
|
|
32
|
+
"fun updateTestContext(sections) = 'Maintain TEST_CONTEXT.md: ticket analysis, evidence, hypotheses, confidence scores, validation results.'",
|
|
33
|
+
"fun dualBrainProcess(enableNlp=true, enableCreative=true) = 'Stage 1: NLP extraction of structured data (entities, dependencies, flows). Stage 2: LLM creative expansion (edge cases, negative paths, integration scenarios).'",
|
|
34
|
+
"fun generateTestHypotheses(count=5) = 'Create 3–5 test hypotheses focused on integration/E2E scenarios. Include risk assessment.'",
|
|
35
|
+
"fun progressiveExpansion(levels) = 'Expand scenarios in layers: Core→Edge→System→Chaos. Adapt complexity to user experience.'",
|
|
36
|
+
"fun adaptToUserLevel(level=Junior|Mixed|Senior) = 'Deliver content adapted to the selected user level.'",
|
|
37
|
+
"fun scoreConfidence(weights) = 'Calculate confidence (1-10) using: requirements clarity, system context, evidence quality, coverage completeness.'",
|
|
38
|
+
"fun validateCoverage(minScore=7) = 'Assess coverage completeness; identify gaps; score coverage 1-10; suggest additional scenarios if needed.'",
|
|
39
|
+
"fun formatDualOutput(human=true, machine=true) = 'Generate both human-readable (Gherkin/Markdown) and machine-readable (JSON/structured) formats simultaneously.'",
|
|
40
|
+
"fun buildTicketSignature() = 'Extract entities, acceptance criteria, identifiers (ticket ID), impacted modules, user flows; output tokens/phrases and canonical signature.'",
|
|
41
|
+
"fun scoreDocRelevance(chunk, signature) = 'Compute lexical+semantic relevance; detect explicit mentions of ticket ID/criteria; return relevanceScore 0–1 with rationale.'",
|
|
42
|
+
"fun extractCitations(criteria) = 'For each acceptance criterion, locate explicit quotes/snippets with {doc, section, quote, relevanceScore}.'",
|
|
43
|
+
"fun computeCriteriaCoverage() = 'Compute fraction (0–1) of acceptance criteria with ≥1 strong citation.'",
|
|
44
|
+
"fun computeTicketSpecificEvidenceScore() = 'Aggregate evidence quality emphasizing ticket specificity; scale 1–10.'",
|
|
45
|
+
"fun computeDocNoiseRatio() = 'Compute proportion (0–1) of cited text that is generic/non-ticket; lower is better.'"
|
|
46
|
+
],
|
|
47
|
+
"steps": [
|
|
48
|
+
{
|
|
49
|
+
"id": "phase-0-intelligent-triage",
|
|
50
|
+
"title": "Phase 0: Intelligent Ticket Triage & Context Setup",
|
|
51
|
+
"prompt": "**SYSTEMATIC TEST GENERATION BEGINS** - Transform ticket requirements into comprehensive test cases through evidence-driven analysis.\n\nanalyzeTicket(depth=3) and buildTicketSignature() to establish foundation:\n\n**STEP 1: Ticket Analysis & Classification**\n- **Ticket Information**: Please provide the complete ticket details (title, description, acceptance criteria, labels, etc.)\n- **System Context**: What system/component does this ticket affect?\n- **Change Type**: Is this a new feature, bug fix, enhancement, or refactoring?\n- **Risk Assessment**: What's the business impact if this functionality fails?\n\n**STEP 2: Complexity & Scope Assessment**\nBased on the ticket analysis, I'll classify the testing complexity:\n- **Simple**: Single component, clear requirements, minimal integration\n- **Standard**: Multiple components, moderate integration, some unknowns\n- **Complex**: Cross-system changes, high integration, significant risk\n\n**STEP 3: User Experience Level**\nWhat's your experience level with test case creation?\n- **Junior**: Need detailed guidance and explanations\n- **Mixed**: Team has varying skill levels\n- **Senior**: Prefer efficient, advanced patterns\n\n**STEP 4: Automation Preferences**\nWhat automation level do you prefer?\n- **High**: Auto-approve confidence >8.0, minimal confirmations\n- **Medium**: Standard confirmations for key decisions\n- **Low**: Extra validation for comprehensive coverage\n\n**OUTPUTS**: Set context variables for adaptive workflow execution, including ticketSignature.",
|
|
52
|
+
"agentRole": "You are a senior test strategy specialist with expertise in analyzing software requirements and determining optimal testing approaches. Your role is to quickly assess ticket complexity, understand system context, and set up the workflow for systematic test case generation.",
|
|
53
|
+
"guidance": [
|
|
54
|
+
"THOROUGH ANALYSIS: Extract all relevant information from ticket details",
|
|
55
|
+
"CONTEXT AWARENESS: Understand the ticket's place in the larger system",
|
|
56
|
+
"ADAPTIVE SETUP: Configure workflow based on user experience and preferences",
|
|
57
|
+
"EVIDENCE FOUNDATION: Establish the evidence base for systematic test generation"
|
|
58
|
+
],
|
|
59
|
+
"requireConfirmation": true
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"id": "phase-0b-readiness",
|
|
63
|
+
"title": "Phase 0b: Readiness Evaluation (Soft Gate)",
|
|
64
|
+
"prompt": "Evaluate readiness: compute requiredVarsFound, missingVars[], completenessScore (0-1), gather evidence, set evidenceScore (1-10). Outcome flags: proceedNeeded, questioningNeeded, assumptionNeeded.",
|
|
65
|
+
"askForFiles": true,
|
|
66
|
+
"hasValidation": true,
|
|
67
|
+
"validationCriteria": [
|
|
68
|
+
{ "type": "length", "min": 200, "message": "Summarize readiness and evidence (≥200 chars)" }
|
|
69
|
+
],
|
|
70
|
+
"requireConfirmation": true
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"id": "phase-0c-questioning-loop",
|
|
74
|
+
"type": "loop",
|
|
75
|
+
"title": "Resolve Missing Inputs",
|
|
76
|
+
"loop": { "type": "forEach", "items": "missingVars", "itemVar": "missing", "maxIterations": 5 },
|
|
77
|
+
"body": [
|
|
78
|
+
{
|
|
79
|
+
"id": "ask-missing",
|
|
80
|
+
"title": "Provide: {{missing}}",
|
|
81
|
+
"prompt": "Please provide '{{missing}}'. If unknown, choose an option or confirm an assumption. Attach links/files if applicable.",
|
|
82
|
+
"askForFiles": true,
|
|
83
|
+
"requireConfirmation": true
|
|
84
|
+
}
|
|
85
|
+
]
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"id": "phase-0d-assumptions",
|
|
89
|
+
"title": "Assumption Proposal & Risks",
|
|
90
|
+
"prompt": "Propose explicit assumptions for remaining gaps with risks and mitigation. Confirm to proceed and record assumptions[].",
|
|
91
|
+
"requireConfirmation": true
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"id": "phase-1-evidence-gathering",
|
|
95
|
+
"runCondition": {
|
|
96
|
+
"and": [
|
|
97
|
+
{ "or": [ { "var": "completenessScore", "gte": 0.7 }, { "var": "assumptionAccepted", "equals": true } ] },
|
|
98
|
+
{ "var": "evidenceScore", "gte": 7.5 }
|
|
99
|
+
]
|
|
100
|
+
},
|
|
101
|
+
"title": "Phase 1: Evidence Gathering & Ticket-Specific Scoring",
|
|
102
|
+
"prompt": "collectEvidence(ticket requirements) and discoverDocuments(system context)\n\n**TICKET-SPECIFIC EVIDENCE COLLECTION**\n- scoreDocRelevance(each document chunk, ticketSignature)\n- extractCitations(acceptance criteria) and build evidenceMap: criterion -> [{doc, section, quote, relevanceScore}]\n- computeCriteriaCoverage()\n- computeTicketSpecificEvidenceScore()\n- computeDocNoiseRatio()\n\nscoreAndValidate(evidence completeness, 7/10 threshold)\n\nupdateTestContext(evidence-analysis, evidenceMap, criteriaCoverage, ticketSpecificEvidenceScore, docNoiseRatio)\n\n**QUALITY GATE**: Require criteriaCoverage ≥ 0.7, ticketSpecificEvidenceScore ≥ 7.0, docNoiseRatio ≤ 0.3 before proceeding.",
|
|
103
|
+
"agentRole": "You are a systematic evidence analyst specializing in gathering comprehensive context for test case generation. Your expertise lies in identifying information gaps, discovering relevant documentation, and building complete evidence bases for reliable test design.",
|
|
104
|
+
"guidance": [
|
|
105
|
+
"SYSTEMATIC COLLECTION: Use structured approach to gather all relevant evidence",
|
|
106
|
+
"GAP IDENTIFICATION: Actively identify and address missing information",
|
|
107
|
+
"DOCUMENT DISCOVERY: Use semantic search and intelligent questioning to find context",
|
|
108
|
+
"QUALITY THRESHOLD: Don't proceed without sufficient evidence foundation"
|
|
109
|
+
],
|
|
110
|
+
"validationCriteria": [
|
|
111
|
+
{ "type": "length", "min": 200, "message": "Evidence collection must be comprehensive (minimum 200 characters of analysis)" },
|
|
112
|
+
{ "type": "regex", "pattern": "criteriaCoverage|coverage", "flags": "i", "message": "Include criteriaCoverage metric" },
|
|
113
|
+
{ "type": "regex", "pattern": "docNoiseRatio", "flags": "i", "message": "Include docNoiseRatio metric" },
|
|
114
|
+
{ "type": "regex", "pattern": "evidenceMap|citation|quote", "flags": "i", "message": "Provide explicit citations/quotes per criterion" }
|
|
115
|
+
],
|
|
116
|
+
"hasValidation": true
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"id": "phase-1b-evidence-audit",
|
|
120
|
+
"title": "Phase 1b: Evidence Audit (Ticket-Specific)",
|
|
121
|
+
"prompt": "Audit the ticket-specific evidence before hypothesis formation.\n\n**Produce:**\n- evidenceMap (acceptance criterion -> citations with {doc, section, quote, relevanceScore})\n- criteriaCoverage (0–1), ticketSpecificEvidenceScore (1–10), docNoiseRatio (0–1)\n- Brief rationale for any low-coverage criteria and targeted follow-up actions\n\nConfirm metrics and highlight gaps. Do not proceed if coverage/noise thresholds are not met.",
|
|
122
|
+
"validationCriteria": [
|
|
123
|
+
{ "type": "regex", "pattern": "evidenceMap|criterion", "flags": "i", "message": "Include evidenceMap mapping criteria to citations" },
|
|
124
|
+
{ "type": "regex", "pattern": "criteriaCoverage|ticketSpecificEvidenceScore|docNoiseRatio", "flags": "i", "message": "Report ticket-specific metrics" }
|
|
125
|
+
],
|
|
126
|
+
"hasValidation": true,
|
|
127
|
+
"requireConfirmation": true
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"id": "phase-2-gap-check",
|
|
131
|
+
"title": "Phase 2 Gap Check",
|
|
132
|
+
"prompt": "Quick gap check before hypotheses. If new gaps detected, append to openQuestions and run questioning loop once.",
|
|
133
|
+
"runCondition": { "or": [ { "var": "newGapsDetected", "equals": true }, { "var": "newGapsDetected", "equals": false } ] }
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"id": "phase-2-dual-brain-analysis",
|
|
137
|
+
"runCondition": {
|
|
138
|
+
"and": [
|
|
139
|
+
{ "or": [ { "var": "completenessScore", "gte": 0.7 }, { "var": "assumptionAccepted", "equals": true } ] },
|
|
140
|
+
{ "var": "evidenceScore", "gte": 7.5 },
|
|
141
|
+
{ "var": "criteriaCoverage", "gte": 0.7 },
|
|
142
|
+
{ "var": "docNoiseRatio", "lte": 0.3 },
|
|
143
|
+
{ "var": "ticketSpecificEvidenceScore", "gte": 7.0 }
|
|
144
|
+
]
|
|
145
|
+
},
|
|
146
|
+
"title": "Phase 2: Dual-Brain Processing & Hypothesis Formation",
|
|
147
|
+
"prompt": "dualBrainProcess(collected evidence) and generateTestHypotheses(ticket, context)\n\n**DUAL-BRAIN PROCESSING ARCHITECTURE**\nCombine structured NLP analysis with creative LLM expansion to generate comprehensive test hypotheses.\n\nscoreAndValidate(each hypothesis, testability criteria)\n\nupdateTestContext(hypothesis-analysis, ranked test hypotheses with risk assessment)\n\n**QUALITY GATE**: Maximum 5 hypotheses, each scored for testability and business impact.",
|
|
148
|
+
"agentRole": "You are a dual-brain test analysis expert combining systematic NLP processing with creative scenario generation. Your role is to extract structured information while also discovering creative test scenarios that human analysts might miss.",
|
|
149
|
+
"guidance": [
|
|
150
|
+
"STRUCTURED EXTRACTION: Use NLP techniques for systematic information parsing",
|
|
151
|
+
"CREATIVE EXPANSION: Generate innovative test scenarios beyond obvious cases",
|
|
152
|
+
"HYPOTHESIS RIGOR: Base all hypotheses on concrete evidence from analysis",
|
|
153
|
+
"RISK PRIORITIZATION: Focus on high-impact, high-probability test scenarios"
|
|
154
|
+
],
|
|
155
|
+
"validationCriteria": [
|
|
156
|
+
{ "type": "regex", "pattern": "hypothesis|test case|scenario", "flags": "i", "message": "Output must contain test hypotheses or scenarios" }
|
|
157
|
+
],
|
|
158
|
+
"hasValidation": true
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
"id": "phase-3-gap-check",
|
|
162
|
+
"title": "Phase 3 Gap Check",
|
|
163
|
+
"prompt": "Quick gap check before scenario generation. If new gaps detected, append to openQuestions and run questioning loop once.",
|
|
164
|
+
"runCondition": { "or": [ { "var": "newGapsDetected", "equals": true }, { "var": "newGapsDetected", "equals": false } ] }
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
"id": "phase-3-progressive-test-generation",
|
|
168
|
+
"runCondition": {
|
|
169
|
+
"and": [
|
|
170
|
+
{ "or": [ { "var": "completenessScore", "gte": 0.7 }, { "var": "assumptionAccepted", "equals": true } ] },
|
|
171
|
+
{ "var": "evidenceScore", "gte": 7.5 },
|
|
172
|
+
{ "var": "criteriaCoverage", "gte": 0.7 },
|
|
173
|
+
{ "var": "docNoiseRatio", "lte": 0.3 },
|
|
174
|
+
{ "var": "ticketSpecificEvidenceScore", "gte": 7.0 }
|
|
175
|
+
]
|
|
176
|
+
},
|
|
177
|
+
"title": "Phase 3: Progressive Test Scenario Generation",
|
|
178
|
+
"prompt": "progressiveExpansion(validated hypotheses, user experience level)\n\n**PROGRESSIVE TEST EXPANSION ARCHITECTURE**\nGenerate layered test scenarios: Core→Edge→System→Chaos, adapted to user experience level.\n\nadaptToUserLevel(test scenario complexity, user experience level)\n\nupdateTestContext(test-scenarios, layered test scenarios with traceability)\n\n**OUTPUT**: Comprehensive test scenarios adapted to user capabilities.",
|
|
179
|
+
"agentRole": "You are a progressive test design architect specializing in creating layered test scenarios that scale with team expertise. Your role is to generate comprehensive test coverage while adapting complexity to user capabilities.",
|
|
180
|
+
"guidance": [
|
|
181
|
+
"LAYERED APPROACH: Build test scenarios in progressive complexity layers",
|
|
182
|
+
"ADAPTIVE COMPLEXITY: Match scenario depth to user experience level",
|
|
183
|
+
"COMPREHENSIVE COVERAGE: Ensure all critical paths are tested",
|
|
184
|
+
"PRACTICAL IMPLEMENTATION: Include setup, execution, and cleanup details"
|
|
185
|
+
]
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
"id": "phase-4-confidence-scoring",
|
|
189
|
+
"runCondition": {
|
|
190
|
+
"and": [
|
|
191
|
+
{ "or": [ { "var": "completenessScore", "gte": 0.7 }, { "var": "assumptionAccepted", "equals": true } ] },
|
|
192
|
+
{ "var": "evidenceScore", "gte": 7.5 },
|
|
193
|
+
{ "var": "criteriaCoverage", "gte": 0.7 },
|
|
194
|
+
{ "var": "docNoiseRatio", "lte": 0.3 },
|
|
195
|
+
{ "var": "ticketSpecificEvidenceScore", "gte": 7.0 }
|
|
196
|
+
]
|
|
197
|
+
},
|
|
198
|
+
"title": "Phase 4: Confidence Scoring & Quality Validation",
|
|
199
|
+
"prompt": "scoreConfidence(generated test cases, evidence base) and validateCoverage(scenarios, requirements)\n\n**CONFIDENCE SCORING FRAMEWORK**\nApply mathematical confidence framework using 4-factor scoring: Requirements Clarity, System Context, Evidence Quality, Coverage Completeness.\n\nscoreAndValidate(test scenarios, confidence criteria)\n\nupdateTestContext(confidence-analysis, scored scenarios with coverage gaps)\n\n**OUTPUT**: Confidence-scored test scenarios with coverage analysis and gap recommendations.",
|
|
200
|
+
"agentRole": "You are a test quality assurance specialist with expertise in confidence scoring and coverage analysis. Your role is to objectively assess test scenario quality and ensure comprehensive coverage of requirements.",
|
|
201
|
+
"guidance": [
|
|
202
|
+
"OBJECTIVE SCORING: Use mathematical framework for consistent confidence assessment",
|
|
203
|
+
"COVERAGE ANALYSIS: Systematically map test scenarios to requirements",
|
|
204
|
+
"GAP IDENTIFICATION: Actively identify and address coverage gaps",
|
|
205
|
+
"QUALITY GATES: Apply appropriate thresholds based on automation preferences"
|
|
206
|
+
],
|
|
207
|
+
"validationCriteria": [
|
|
208
|
+
{ "type": "regex", "pattern": "confidence|score|coverage", "flags": "i", "message": "Output must include confidence scoring and coverage analysis" }
|
|
209
|
+
],
|
|
210
|
+
"hasValidation": true
|
|
211
|
+
},
|
|
212
|
+
{
|
|
213
|
+
"id": "phase-5-dual-format-output",
|
|
214
|
+
"runCondition": {
|
|
215
|
+
"and": [
|
|
216
|
+
{ "or": [ { "var": "completenessScore", "gte": 0.7 }, { "var": "assumptionAccepted", "equals": true } ] },
|
|
217
|
+
{ "var": "evidenceScore", "gte": 7.5 },
|
|
218
|
+
{ "var": "criteriaCoverage", "gte": 0.7 },
|
|
219
|
+
{ "var": "docNoiseRatio", "lte": 0.3 },
|
|
220
|
+
{ "var": "ticketSpecificEvidenceScore", "gte": 7.0 }
|
|
221
|
+
]
|
|
222
|
+
},
|
|
223
|
+
"title": "Phase 5: Dual-Format Output Generation",
|
|
224
|
+
"prompt": "formatDualOutput(validated test scenarios) - Generate both human-readable and LLM-consumable formats\n\n**DUAL-FORMAT GENERATION STRATEGY**\n\n**FORMAT 1: Human-Readable (Developer-Optimized)**\nClear, scannable format for manual review and JIRA integration:\n- Test scenarios with objectives, preconditions, steps, expected results\n- Priority levels and confidence scores\n- Proper markdown formatting for JIRA copy-paste\n- Traceability links to requirements\n\n**FORMAT 2: LLM-Readable (Machine-Optimized)**\nJSON format optimized for programmatic consumption:\n- Structured metadata with generation info and confidence thresholds\n- Test scenarios with IDs, types, priorities, and confidence scores\n- Requirements mapping and test data specifications\n- Tags for categorization and tool integration\n\n**SIMULTANEOUS GENERATION**\n- Generate both formats from the same source data\n- Ensure consistency between formats\n- Validate format integrity and completeness\n- Optimize for different consumption patterns\n\n**OUTPUT**: Complete test scenarios in both human-readable and machine-readable formats, ready for JIRA integration and future automation",
|
|
225
|
+
"agentRole": "You are a dual-format output specialist with expertise in creating documentation optimized for both human consumption and machine processing. Your role is to generate test scenarios that serve both immediate manual use and future automation needs.",
|
|
226
|
+
"guidance": [
|
|
227
|
+
"DUAL OPTIMIZATION: Create formats optimized for different consumption patterns",
|
|
228
|
+
"CONSISTENCY MAINTENANCE: Ensure both formats contain identical information",
|
|
229
|
+
"JIRA INTEGRATION: Format human-readable output for easy ticket integration",
|
|
230
|
+
"AUTOMATION READY: Structure machine-readable output for future tool consumption"
|
|
231
|
+
],
|
|
232
|
+
"requireConfirmation": true
|
|
233
|
+
}
|
|
234
|
+
]
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
|