@zibby/core 0.3.6 ā 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +34 -32
- package/dist/package.json +2 -1
- package/dist/register-built-in-strategies.js +25 -23
- package/dist/strategies/claude-strategy.js +3 -1
- package/dist/strategies/index.js +25 -23
- package/dist/templates/browser-test-automation/graph.mjs +20 -6
- package/dist/templates/browser-test-automation/nodes/generate-script.mjs +14 -3
- package/dist/templates/browser-test-automation/nodes/preflight.mjs +50 -15
- package/dist/templates/browser-test-automation/state.js +61 -0
- package/dist/templates/code-analysis/README.md +60 -0
- package/dist/templates/code-analysis/graph.mjs +33 -0
- package/dist/templates/code-analysis/nodes/analyze-ticket-node.js +1 -1
- package/dist/templates/code-analysis/nodes/create-pr-node.js +1 -1
- package/dist/templates/code-analysis/nodes/generate-code-node.js +1 -1
- package/dist/templates/code-analysis/nodes/generate-test-cases-node.js +1 -1
- package/dist/templates/code-analysis/nodes/services/prMetaService.js +1 -1
- package/dist/templates/code-analysis/state.js +14 -6
- package/dist/templates/generate-test-cases/README.md +72 -0
- package/dist/templates/generate-test-cases/graph.mjs +46 -0
- package/dist/templates/generate-test-cases/nodes/generate-test-cases-node.js +381 -0
- package/dist/templates/generate-test-cases/nodes/setup-node.js +142 -0
- package/dist/templates/generate-test-cases/state.js +54 -0
- package/dist/templates/index.js +53 -0
- package/package.json +2 -1
- package/templates/browser-test-automation/graph.mjs +20 -6
- package/templates/browser-test-automation/nodes/generate-script.mjs +14 -3
- package/templates/browser-test-automation/nodes/preflight.mjs +50 -15
- package/templates/browser-test-automation/state.js +61 -0
- package/templates/code-analysis/README.md +60 -0
- package/templates/code-analysis/graph.mjs +33 -0
- package/templates/code-analysis/nodes/analyze-ticket-node.js +1 -1
- package/templates/code-analysis/nodes/create-pr-node.js +1 -1
- package/templates/code-analysis/nodes/generate-code-node.js +1 -1
- package/templates/code-analysis/nodes/generate-test-cases-node.js +1 -1
- package/templates/code-analysis/nodes/services/prMetaService.js +1 -1
- package/templates/code-analysis/state.js +14 -6
- package/templates/generate-test-cases/README.md +72 -0
- package/templates/generate-test-cases/graph.mjs +46 -0
- package/templates/generate-test-cases/nodes/generate-test-cases-node.js +381 -0
- package/templates/generate-test-cases/nodes/setup-node.js +142 -0
- package/templates/generate-test-cases/state.js +54 -0
- package/templates/index.js +53 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Setup Node - Clone repositories and initialize git baseline
|
|
3
|
+
* Used by: analysisGraph, implementationGraph
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { spawn } from 'child_process';
|
|
7
|
+
import { join } from 'path';
|
|
8
|
+
import { z } from 'zod';
|
|
9
|
+
|
|
10
|
+
const SetupOutputSchema = z.object({
|
|
11
|
+
success: z.boolean(),
|
|
12
|
+
clonedRepos: z.array(z.object({
|
|
13
|
+
name: z.string(),
|
|
14
|
+
path: z.string(),
|
|
15
|
+
isPrimary: z.boolean().optional()
|
|
16
|
+
})),
|
|
17
|
+
baselineCommit: z.string()
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
export const setupNode = {
|
|
21
|
+
name: 'setup',
|
|
22
|
+
outputSchema: SetupOutputSchema,
|
|
23
|
+
execute: async (state) => {
|
|
24
|
+
console.log('\nš§ Setting up environment...');
|
|
25
|
+
|
|
26
|
+
const { workspace, repos, githubToken } = state;
|
|
27
|
+
const gitlabToken = process.env.GITLAB_TOKEN || '';
|
|
28
|
+
const gitlabUrl = process.env.GITLAB_URL || '';
|
|
29
|
+
|
|
30
|
+
// DEBUG: Log token status
|
|
31
|
+
console.log(`š GitHub Token: ${githubToken ? 'Present' : 'MISSING'}`);
|
|
32
|
+
console.log(`š GitLab Token: ${gitlabToken ? 'Present' : 'MISSING'}`);
|
|
33
|
+
if (gitlabUrl) console.log(`š GitLab URL: ${gitlabUrl}`);
|
|
34
|
+
|
|
35
|
+
// Log environment
|
|
36
|
+
console.log('Container: ECS Fargate');
|
|
37
|
+
console.log('Memory: 4GB');
|
|
38
|
+
console.log('CPU: 2 vCPU');
|
|
39
|
+
console.log('Tools: Node.js, Git, Cursor CLI, Zibby CLI');
|
|
40
|
+
console.log(`Working directory: ${workspace}`);
|
|
41
|
+
|
|
42
|
+
// Clone repositories
|
|
43
|
+
console.log('\nš¦ Cloning repositories...');
|
|
44
|
+
|
|
45
|
+
const clonedRepos = [];
|
|
46
|
+
for (const repo of repos) {
|
|
47
|
+
console.log(`Cloning ${repo.name}...`);
|
|
48
|
+
|
|
49
|
+
const repoDir = join(workspace, repo.name);
|
|
50
|
+
|
|
51
|
+
// Use token for authentication based on provider
|
|
52
|
+
let cloneUrl = repo.url;
|
|
53
|
+
let cloneEnv = {};
|
|
54
|
+
const isGitlab = repo.provider === 'gitlab' || (gitlabUrl && repo.url.includes(new URL(gitlabUrl).host));
|
|
55
|
+
const isGithub = repo.provider === 'github' || repo.url.includes('github.com');
|
|
56
|
+
|
|
57
|
+
if (isGithub && githubToken) {
|
|
58
|
+
cloneUrl = repo.url.replace('https://github.com', `https://x-access-token:${githubToken}@github.com`);
|
|
59
|
+
cloneEnv = { ...process.env, GIT_TERMINAL_PROMPT: '0', GIT_ASKPASS: 'echo' };
|
|
60
|
+
} else if (isGitlab && gitlabToken && gitlabUrl) {
|
|
61
|
+
try {
|
|
62
|
+
const gitlabHost = new URL(gitlabUrl).host;
|
|
63
|
+
cloneUrl = repo.url.replace(`https://${gitlabHost}`, `https://oauth2:${gitlabToken}@${gitlabHost}`);
|
|
64
|
+
} catch (e) {
|
|
65
|
+
console.warn(`ā ļø Failed to parse GITLAB_URL: ${e.message}`);
|
|
66
|
+
}
|
|
67
|
+
cloneEnv = { ...process.env, GIT_TERMINAL_PROMPT: '0', GIT_ASKPASS: 'echo' };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Shallow clone with progress output (async, non-blocking)
|
|
71
|
+
await execCommand(
|
|
72
|
+
`git clone --progress --depth 1 --branch ${repo.branch} "${cloneUrl}" "${repoDir}"`,
|
|
73
|
+
workspace,
|
|
74
|
+
cloneEnv
|
|
75
|
+
);
|
|
76
|
+
console.log(`ā Cloned ${repo.name} on branch ${repo.branch}`);
|
|
77
|
+
|
|
78
|
+
clonedRepos.push({
|
|
79
|
+
name: repo.name,
|
|
80
|
+
path: repoDir,
|
|
81
|
+
isPrimary: repo.isPrimary
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Initialize git in workspace for diff tracking
|
|
86
|
+
await execCommand('git init', workspace);
|
|
87
|
+
await execCommand('git config user.email "zibby@agent.com"', workspace);
|
|
88
|
+
await execCommand('git config user.name "Zibby Agent"', workspace);
|
|
89
|
+
await execCommand('git add .', workspace);
|
|
90
|
+
await execCommand('git commit --allow-empty -m "baseline"', workspace);
|
|
91
|
+
|
|
92
|
+
console.log('ā
Environment ready');
|
|
93
|
+
|
|
94
|
+
const baselineCommit = await execCommand('git rev-parse HEAD', workspace);
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
success: true,
|
|
98
|
+
clonedRepos,
|
|
99
|
+
baselineCommit: baselineCommit.trim()
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
// Async version using spawn - streams output in real-time, doesn't block event loop
|
|
105
|
+
async function execCommand(command, cwd, env = {}) {
|
|
106
|
+
return new Promise((resolve, reject) => {
|
|
107
|
+
const proc = spawn(command, {
|
|
108
|
+
cwd,
|
|
109
|
+
shell: true,
|
|
110
|
+
env: Object.keys(env).length > 0 ? env : process.env
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
let stdout = '';
|
|
114
|
+
let stderr = '';
|
|
115
|
+
|
|
116
|
+
// Stream stdout as it comes (triggers middleware setInterval!)
|
|
117
|
+
proc.stdout.on('data', (data) => {
|
|
118
|
+
const output = data.toString();
|
|
119
|
+
stdout += output;
|
|
120
|
+
console.log(output.trimEnd());
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
// Stream stderr as it comes
|
|
124
|
+
proc.stderr.on('data', (data) => {
|
|
125
|
+
const output = data.toString();
|
|
126
|
+
stderr += output;
|
|
127
|
+
console.log(output.trimEnd());
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
proc.on('close', (code) => {
|
|
131
|
+
if (code !== 0) {
|
|
132
|
+
reject(new Error(`Command failed with exit code ${code}: ${command}`));
|
|
133
|
+
} else {
|
|
134
|
+
resolve(stdout || stderr || '');
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
proc.on('error', (err) => {
|
|
139
|
+
reject(new Error(`Command error: ${command} - ${err.message}`));
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* State schema for the generate-test-cases standalone template.
|
|
3
|
+
*
|
|
4
|
+
* Same shape as code-analysis (workspace + repos + ticketContext) PLUS
|
|
5
|
+
* a `codeImplementation` field ā the diff this template generates tests
|
|
6
|
+
* for. In code-analysis that field is produced by the upstream
|
|
7
|
+
* generate_code node; here, the user provides it directly.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { z } from 'zod';
|
|
11
|
+
|
|
12
|
+
export const generateTestCasesStateSchema = z.object({
|
|
13
|
+
workspace: z.string().describe('Local workspace path'),
|
|
14
|
+
|
|
15
|
+
repos: z.array(z.object({
|
|
16
|
+
name: z.string(),
|
|
17
|
+
url: z.string().url(),
|
|
18
|
+
path: z.string().optional(),
|
|
19
|
+
branch: z.string().default('main'),
|
|
20
|
+
isPrimary: z.boolean().default(false),
|
|
21
|
+
})).optional().describe('Repository configurations (cloned by setup node so the LLM can explore routing/components)'),
|
|
22
|
+
|
|
23
|
+
ticketContext: z.object({
|
|
24
|
+
key: z.string().regex(/^[A-Z]+-\d+$/, 'Invalid ticket format (expected PROJ-123)').optional(),
|
|
25
|
+
ticketKey: z.string().optional(),
|
|
26
|
+
summary: z.string().min(1).describe('Ticket summary/title'),
|
|
27
|
+
description: z.any().optional().describe('Ticket description (string or ADF object)'),
|
|
28
|
+
acceptanceCriteria: z.string().optional(),
|
|
29
|
+
type: z.string().optional(),
|
|
30
|
+
priority: z.string().optional(),
|
|
31
|
+
labels: z.array(z.string()).optional(),
|
|
32
|
+
components: z.array(z.string()).optional(),
|
|
33
|
+
}).describe('Jira/ticket context ā informs test priorities + naming'),
|
|
34
|
+
|
|
35
|
+
// The new direct-input field that distinguishes this standalone template
|
|
36
|
+
// from code-analysis. In code-analysis this comes from generate_code's
|
|
37
|
+
// output; here the user supplies it (e.g. from `git diff` of a PR they
|
|
38
|
+
// want tests for).
|
|
39
|
+
codeImplementation: z.object({
|
|
40
|
+
diff: z.string().describe('Unified-diff string of the changes'),
|
|
41
|
+
changedFiles: z.array(z.string()).describe('List of file paths touched'),
|
|
42
|
+
}).describe('Code changes to generate tests for'),
|
|
43
|
+
|
|
44
|
+
githubToken: z.string().optional().describe('GitHub PAT (needed only if repos[].url requires auth)'),
|
|
45
|
+
model: z.string().default('auto').describe('AI model to use'),
|
|
46
|
+
nodeConfigs: z.record(z.string(), z.any()).optional().describe('Per-node configuration overrides (e.g. extractContext for test credentials)'),
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// Clean isolation: this schema declares ONLY what the template's nodes
|
|
50
|
+
// actually need. No EXECUTION_ID / PROGRESS_QUEUE_URL / SQS_AUTH_TOKEN
|
|
51
|
+
// / PROJECT_API_TOKEN ā those were legacy analysis-UI plumbing fields
|
|
52
|
+
// and the new templates run via the standard `workflow run` /
|
|
53
|
+
// `workflow trigger` cloud pipeline, which has its own progress
|
|
54
|
+
// reporting outside the state object.
|
package/dist/templates/index.js
CHANGED
|
@@ -12,6 +12,18 @@ export const TEMPLATES = {
|
|
|
12
12
|
description: 'Complete browser test automation workflow with title generation, live execution, and script generation',
|
|
13
13
|
path: join(__dirname, 'browser-test-automation'),
|
|
14
14
|
default: true,
|
|
15
|
+
// Suggested slug for `zibby workflow new <slug> -t <name>`. Used in
|
|
16
|
+
// the `template list` scaffold hint so the printed command is
|
|
17
|
+
// copy-paste-ready instead of `your-workflow-name`. Users can still
|
|
18
|
+
// pick anything they want at scaffold time.
|
|
19
|
+
defaultSlug: 'browser-tests',
|
|
20
|
+
// Runtime deps the scaffolded copy needs in addition to @zibby/core.
|
|
21
|
+
// graph.mjs now imports state.js which `import { z } from 'zod'`s
|
|
22
|
+
// directly, so the user's package.json must declare zod or the
|
|
23
|
+
// scaffolded workflow fails on first import.
|
|
24
|
+
deps: {
|
|
25
|
+
zod: '^3.23.0',
|
|
26
|
+
},
|
|
15
27
|
features: [
|
|
16
28
|
'Preflight analysis: extract title + assertion checklist from spec',
|
|
17
29
|
'Execute test live with AI + browser (Claude or Cursor)',
|
|
@@ -19,6 +31,47 @@ export const TEMPLATES = {
|
|
|
19
31
|
'Real-time streaming output',
|
|
20
32
|
'Video recording of browser sessions'
|
|
21
33
|
]
|
|
34
|
+
},
|
|
35
|
+
'code-analysis': {
|
|
36
|
+
name: 'code-analysis',
|
|
37
|
+
displayName: 'Code Analysis (Ticket ā Code + Tests)',
|
|
38
|
+
description: 'Multi-node workflow that analyzes a Jira ticket against a code repo, generates code changes, and emits test cases',
|
|
39
|
+
path: join(__dirname, 'code-analysis'),
|
|
40
|
+
defaultSlug: 'ticket-analyzer',
|
|
41
|
+
// Runtime deps the scaffolded copy needs in addition to @zibby/core.
|
|
42
|
+
// Merged into the generated package.json so `npm install` works
|
|
43
|
+
// without manual edits. Browser-test doesn't declare any because
|
|
44
|
+
// its nodes only depend on @zibby/core.
|
|
45
|
+
deps: {
|
|
46
|
+
axios: '^1.6.0',
|
|
47
|
+
handlebars: '^4.7.8',
|
|
48
|
+
zod: '^3.23.0',
|
|
49
|
+
},
|
|
50
|
+
features: [
|
|
51
|
+
'Clone repos + snapshot git baseline',
|
|
52
|
+
'LLM analysis of ticket against codebase (canProceed gate)',
|
|
53
|
+
'Conditional routing: skip code-gen if ticket is invalid',
|
|
54
|
+
'Generate scoped code changes',
|
|
55
|
+
'Generate test cases covering the changes',
|
|
56
|
+
'Customizable prompts in prompts/*.md'
|
|
57
|
+
]
|
|
58
|
+
},
|
|
59
|
+
'generate-test-cases': {
|
|
60
|
+
name: 'generate-test-cases',
|
|
61
|
+
displayName: 'Generate Test Cases (Diff ā Test Specs)',
|
|
62
|
+
description: 'Standalone slice ā takes an existing code diff and generates plain-English test specifications for it. Skips ticket-analysis and code-gen.',
|
|
63
|
+
path: join(__dirname, 'generate-test-cases'),
|
|
64
|
+
defaultSlug: 'tests-from-diff',
|
|
65
|
+
deps: {
|
|
66
|
+
zod: '^3.23.0',
|
|
67
|
+
},
|
|
68
|
+
features: [
|
|
69
|
+
'Two-node graph: setup ā generate_test_cases',
|
|
70
|
+
'Takes a PR diff directly as state input (no upstream code-gen needed)',
|
|
71
|
+
'LLM explores codebase routing/components for accurate test steps',
|
|
72
|
+
'Emits 4-8 prioritized test specs (Critical/High/Medium/Low)',
|
|
73
|
+
'Plain-English test steps ā runnable by AI agents'
|
|
74
|
+
]
|
|
22
75
|
}
|
|
23
76
|
};
|
|
24
77
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zibby/core",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.8",
|
|
4
4
|
"description": "Core test automation engine with multi-agent and multi-MCP support",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"./sync": "./dist/sync/index.js",
|
|
11
11
|
"./function-bridge.js": "./dist/function-bridge.js",
|
|
12
12
|
"./function-skill-registry.js": "./dist/function-skill-registry.js",
|
|
13
|
+
"./utils/adf-converter.js": "./dist/utils/adf-converter.js",
|
|
13
14
|
"./utils/ast-utils.js": "./dist/utils/ast-utils.js",
|
|
14
15
|
"./utils/mcp-config-writer.js": "./dist/utils/mcp-config-writer.js",
|
|
15
16
|
"./utils/node-schema-parser.js": "./dist/utils/node-schema-parser.js",
|
|
@@ -6,30 +6,44 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { WorkflowAgent, WorkflowGraph } from '@zibby/core';
|
|
9
|
-
import {
|
|
10
|
-
preflightNode,
|
|
11
|
-
executeLiveNode,
|
|
9
|
+
import {
|
|
10
|
+
preflightNode,
|
|
11
|
+
executeLiveNode,
|
|
12
12
|
generateScriptNode,
|
|
13
13
|
} from './nodes/index.mjs';
|
|
14
14
|
import { BrowserTestResultHandler } from './result-handler.mjs';
|
|
15
|
+
import { browserTestAutomationStateSchema } from './state.js';
|
|
15
16
|
|
|
16
17
|
export class BrowserTestAutomationAgent extends WorkflowAgent {
|
|
17
18
|
buildGraph() {
|
|
18
19
|
const graph = new WorkflowGraph();
|
|
20
|
+
graph.setStateSchema(browserTestAutomationStateSchema);
|
|
19
21
|
|
|
20
22
|
graph.addNode('preflight', preflightNode);
|
|
21
23
|
graph.addNode('execute_live', executeLiveNode);
|
|
22
24
|
graph.addNode('generate_script', generateScriptNode);
|
|
23
25
|
|
|
24
26
|
graph.setEntryPoint('preflight');
|
|
25
|
-
|
|
26
|
-
|
|
27
|
+
|
|
28
|
+
// Short-circuit when preflight produced nothing usable. Triggered when:
|
|
29
|
+
// - the user invoked `zibby workflow run browser-tests` with no spec
|
|
30
|
+
// (state.input is undefined / empty), so preflight had nothing to
|
|
31
|
+
// analyze and the LLM came back with `assertions: []`
|
|
32
|
+
// - the spec is so vague the LLM can't extract any assertions
|
|
33
|
+
// Without this gate the graph would barrel into execute_live, fire up
|
|
34
|
+
// a real browser session + a second expensive LLM call, then waste
|
|
35
|
+
// ~30s before failing ā bad UX and bad bill.
|
|
36
|
+
graph.addConditionalEdges('preflight', (state) => {
|
|
37
|
+
const assertions = state.preflight?.assertions || [];
|
|
38
|
+
return assertions.length > 0 ? 'execute_live' : 'END';
|
|
39
|
+
});
|
|
40
|
+
|
|
27
41
|
graph.addConditionalEdges('execute_live', (state) => {
|
|
28
42
|
const result = state.execute_live;
|
|
29
43
|
const hasExecution = (result?.steps?.length > 0) || (result?.actions?.length > 0);
|
|
30
44
|
return hasExecution ? 'generate_script' : 'END';
|
|
31
45
|
});
|
|
32
|
-
|
|
46
|
+
|
|
33
47
|
graph.addEdge('generate_script', 'END');
|
|
34
48
|
return graph;
|
|
35
49
|
}
|
|
@@ -28,7 +28,18 @@ export const generateScriptNode = {
|
|
|
28
28
|
const recorded = loadRecordedActions(state.sessionPath);
|
|
29
29
|
const setupHint = formatSetupHint(detectLoginPattern(recorded));
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
// `state.outputPath` is computed from `state.specPath` by the
|
|
32
|
+
// framework. For `zibby test <spec>` runs it's set. For
|
|
33
|
+
// `workflow trigger` runs there's no spec file on disk, so
|
|
34
|
+
// outputPath comes back undefined and the prompt rendered
|
|
35
|
+
// "Generate and verify Playwright test at undefined" ā which
|
|
36
|
+
// caused the LLM to literally write to a file named `undefined`.
|
|
37
|
+
// Fall back to a path under the session dir so the test always
|
|
38
|
+
// lands somewhere sensible.
|
|
39
|
+
const outputPath = state.outputPath
|
|
40
|
+
|| (state.sessionPath ? `${state.sessionPath}/generate_script/generated-test.spec.js` : 'tests/generated-test.spec.js');
|
|
41
|
+
|
|
42
|
+
return `Generate and verify Playwright test at ${outputPath}
|
|
32
43
|
|
|
33
44
|
Test Spec:
|
|
34
45
|
${state.testSpec}
|
|
@@ -85,9 +96,9 @@ RULES:
|
|
|
85
96
|
|
|
86
97
|
WORKFLOW:
|
|
87
98
|
1. Study the codebase FIRST ā search tests/ for existing helpers, fixtures, and shared setup files. Read them. Reuse what exists. Do NOT create files that duplicate existing ones.
|
|
88
|
-
2. Write test to ${
|
|
99
|
+
2. Write test to ${outputPath} (after the run, a copy is mirrored under ${state.sessionPath}/generate_script/ for Studio ā you may also write directly there if you prefer)
|
|
89
100
|
3. Verify syntax: run node --check on the file. If it fails, fix and re-check before proceeding.
|
|
90
|
-
4. Run: PLAYWRIGHT_HEADLESS=1 npx playwright test ${
|
|
101
|
+
4. Run: PLAYWRIGHT_HEADLESS=1 npx playwright test ${outputPath} --reporter=line --timeout=60000
|
|
91
102
|
5. If fails: try selectors in order ā (a) getByRole (b) getByText (c) getByTestId (d) add waitForSelector. Never retry the same selector twice.
|
|
92
103
|
6. MAX 2 ATTEMPTS then STOP
|
|
93
104
|
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* - generate_script: must implement each assertion in the test
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { z } from '@zibby/core';
|
|
16
|
+
import { z, invokeAgent } from '@zibby/core';
|
|
17
17
|
import { writeFileSync } from 'fs';
|
|
18
18
|
import { join } from 'path';
|
|
19
19
|
|
|
@@ -27,8 +27,57 @@ const PreflightOutputSchema = z.object({
|
|
|
27
27
|
assertions: z.array(AssertionSchema).describe('Every expected result from the spec as a verifiable assertion')
|
|
28
28
|
});
|
|
29
29
|
|
|
30
|
+
// Detect "no usable spec" before invoking the LLM. Catches:
|
|
31
|
+
// - state.testSpec is undefined (workflow run with no input)
|
|
32
|
+
// - empty string / whitespace only
|
|
33
|
+
// - the literal string "undefined" (some upstream paths stringify
|
|
34
|
+
// undefined into state, e.g. when --param isn't passed)
|
|
35
|
+
function isMissingSpec(spec) {
|
|
36
|
+
if (spec == null) return true;
|
|
37
|
+
const s = String(spec).trim();
|
|
38
|
+
return s === '' || s.toLowerCase() === 'undefined';
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const PROMPT = (testSpec) => `Analyze this test specification and extract:
|
|
42
|
+
1. A concise test title (5-10 words, action-oriented). If you find a ticket ID (e.g., PROJ-123, ACME-456), prefix the title with it.
|
|
43
|
+
2. Every expected result as a verifiable assertion. Each assertion must be something the browser can check after execution.
|
|
44
|
+
|
|
45
|
+
Test Spec:
|
|
46
|
+
${testSpec}
|
|
47
|
+
|
|
48
|
+
IMPORTANT: You MUST create ONE assertion for EACH expected result in the spec. Do NOT skip any.
|
|
49
|
+
|
|
50
|
+
Return ONLY this JSON:
|
|
51
|
+
{ "title": "TICKET-ID: Short action title", "assertions": [ { "description": "...", "expected": "..." }, ... ] }`;
|
|
52
|
+
|
|
30
53
|
export const preflightNode = {
|
|
31
54
|
name: 'preflight',
|
|
55
|
+
outputSchema: PreflightOutputSchema,
|
|
56
|
+
|
|
57
|
+
async execute(state) {
|
|
58
|
+
// Early exit BEFORE the LLM call when there's no spec to analyze.
|
|
59
|
+
// Without this guard the node fires the LLM, gets back
|
|
60
|
+
// `{title: "No test specification provided", assertions: []}`, and
|
|
61
|
+
// the graph's conditional edge then skips execute_live ā but we've
|
|
62
|
+
// still paid for one preflight LLM call we didn't need to make.
|
|
63
|
+
// Returning empty assertions here triggers the same skip-to-END
|
|
64
|
+
// path in graph.mjs's preflight conditional edge.
|
|
65
|
+
if (isMissingSpec(state.testSpec)) {
|
|
66
|
+
console.log('ā ļø No test spec provided ā skipping browser run.');
|
|
67
|
+
console.log(' Pass a spec via: zibby test "<inline spec>" or zibby test path/to/spec.txt');
|
|
68
|
+
return {
|
|
69
|
+
title: 'No test specification provided',
|
|
70
|
+
assertions: [],
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const result = await invokeAgent(PROMPT(state.testSpec), {
|
|
75
|
+
state,
|
|
76
|
+
model: state.model || 'auto',
|
|
77
|
+
schema: PreflightOutputSchema,
|
|
78
|
+
});
|
|
79
|
+
return result?.structured || result;
|
|
80
|
+
},
|
|
32
81
|
|
|
33
82
|
async onComplete(state, result) {
|
|
34
83
|
const sessionPath = state.sessionPath || process.env.ZIBBY_SESSION_PATH;
|
|
@@ -42,18 +91,4 @@ export const preflightNode = {
|
|
|
42
91
|
}
|
|
43
92
|
return result;
|
|
44
93
|
},
|
|
45
|
-
|
|
46
|
-
prompt: (state) => `Analyze this test specification and extract:
|
|
47
|
-
1. A concise test title (5-10 words, action-oriented). If you find a ticket ID (e.g., PROJ-123, ACME-456), prefix the title with it.
|
|
48
|
-
2. Every expected result as a verifiable assertion. Each assertion must be something the browser can check after execution.
|
|
49
|
-
|
|
50
|
-
Test Spec:
|
|
51
|
-
${state.testSpec}
|
|
52
|
-
|
|
53
|
-
IMPORTANT: You MUST create ONE assertion for EACH expected result in the spec. Do NOT skip any.
|
|
54
|
-
|
|
55
|
-
Return ONLY this JSON:
|
|
56
|
-
{ "title": "TICKET-ID: Short action title", "assertions": [ { "description": "...", "expected": "..." }, ... ] }`,
|
|
57
|
-
|
|
58
|
-
outputSchema: PreflightOutputSchema
|
|
59
94
|
};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser Test Automation ā Workflow State Schema
|
|
3
|
+
*
|
|
4
|
+
* Declares the input shape that nodes in this template actually read off
|
|
5
|
+
* `state`. Wired into the graph via `graph.setStateSchema(...)` in
|
|
6
|
+
* graph.mjs so that:
|
|
7
|
+
*
|
|
8
|
+
* - `zibby workflow run <slug> -p testSpec=...` validates inputs
|
|
9
|
+
* against this schema before nodes ever execute.
|
|
10
|
+
* - The post-scaffold `Pass inputs:` cheatsheet reads top-level fields
|
|
11
|
+
* from this schema so the printed examples match what the template
|
|
12
|
+
* actually consumes (testSpec, model) instead of generic placeholders.
|
|
13
|
+
*
|
|
14
|
+
* Field provenance:
|
|
15
|
+
* - testSpec, model ā USER input (passed via -p / --input).
|
|
16
|
+
* - cwd, sessionPath, ā FRAMEWORK-injected by the workflow runner
|
|
17
|
+
* outputPath, context (NOT user input). Declared optional so a
|
|
18
|
+
* user-only payload validates cleanly.
|
|
19
|
+
* - preflight, execute_live ā DOWNSTREAM node outputs (set by earlier
|
|
20
|
+
* nodes during graph execution). Same: kept
|
|
21
|
+
* optional so initial inputs validate.
|
|
22
|
+
*
|
|
23
|
+
* `zod` is imported directly from the `zod` package (not re-exported via
|
|
24
|
+
* @zibby/core) ā same convention as code-analysis/state.js. The
|
|
25
|
+
* scaffolded user copy gets `zod` via the template's dep merge in
|
|
26
|
+
* templates/index.js.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { z } from 'zod';
|
|
30
|
+
|
|
31
|
+
export const browserTestAutomationStateSchema = z.object({
|
|
32
|
+
// ---- USER INPUT (what `-p` / `--input` populates) ----
|
|
33
|
+
testSpec: z.string().describe(
|
|
34
|
+
'Plain-English description of the browser test to analyze + execute (REQUIRED).',
|
|
35
|
+
),
|
|
36
|
+
model: z.string().default('auto').describe(
|
|
37
|
+
'Agent model override (e.g. "auto", "opus-4.6"). Defaults to "auto".',
|
|
38
|
+
),
|
|
39
|
+
|
|
40
|
+
// ---- FRAMEWORK-INJECTED (set by the workflow runner, not the user) ----
|
|
41
|
+
cwd: z.string().optional().describe(
|
|
42
|
+
'Working directory the workflow runs in. Injected by the runner.',
|
|
43
|
+
),
|
|
44
|
+
sessionPath: z.string().optional().describe(
|
|
45
|
+
'Per-run session directory under .zibby/output/. Injected by the runner.',
|
|
46
|
+
),
|
|
47
|
+
outputPath: z.string().optional().describe(
|
|
48
|
+
'Target path for the generated Playwright test file. Injected by the runner.',
|
|
49
|
+
),
|
|
50
|
+
context: z.any().optional().describe(
|
|
51
|
+
'Run context bag (project config, env, ā¦). Injected by the runner.',
|
|
52
|
+
),
|
|
53
|
+
|
|
54
|
+
// ---- DOWNSTREAM NODE OUTPUTS (populated mid-graph, not by the user) ----
|
|
55
|
+
preflight: z.any().optional().describe(
|
|
56
|
+
'Output of the preflight node (assertion checklist + title). Set during the run.',
|
|
57
|
+
),
|
|
58
|
+
execute_live: z.any().optional().describe(
|
|
59
|
+
'Output of the execute_live node (recorded actions + steps). Set during the run.',
|
|
60
|
+
),
|
|
61
|
+
});
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Code Analysis Template
|
|
2
|
+
|
|
3
|
+
Multi-node workflow that analyzes a Jira ticket against a code repository
|
|
4
|
+
and emits structured analysis + generated code + test cases.
|
|
5
|
+
|
|
6
|
+
## Nodes
|
|
7
|
+
|
|
8
|
+
- `setup` ā clone repos into the workspace, snapshot baseline commit
|
|
9
|
+
- `analyze_ticket` ā LLM reads ticket + code, produces a validated
|
|
10
|
+
analysis (canProceed flag + reasoning)
|
|
11
|
+
- `validation_check` ā conditional decision: route to `generate_code`
|
|
12
|
+
if `canProceed`, else `finalize`
|
|
13
|
+
- `generate_code` ā LLM generates code changes scoped to the ticket
|
|
14
|
+
- `generate_test_cases` ā LLM generates test cases covering the changes
|
|
15
|
+
- `finalize` ā write outputs, mark complete
|
|
16
|
+
|
|
17
|
+
## Required state inputs
|
|
18
|
+
|
|
19
|
+
```js
|
|
20
|
+
{
|
|
21
|
+
workspace: '/abs/path/to/workspace', // local clone target
|
|
22
|
+
repos: [{
|
|
23
|
+
name: 'my-app',
|
|
24
|
+
url: 'https://github.com/org/my-app.git',
|
|
25
|
+
branch: 'main',
|
|
26
|
+
isPrimary: true,
|
|
27
|
+
}],
|
|
28
|
+
ticketContext: {
|
|
29
|
+
ticketKey: 'PROJ-123',
|
|
30
|
+
summary: 'short title',
|
|
31
|
+
description: 'long description',
|
|
32
|
+
acceptanceCriteria: 'optional',
|
|
33
|
+
},
|
|
34
|
+
githubToken: process.env.GITHUB_TOKEN,
|
|
35
|
+
}
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
See `state.js` for the full Zod schema.
|
|
39
|
+
|
|
40
|
+
## Customizing prompts
|
|
41
|
+
|
|
42
|
+
The three LLM nodes load their prompts from `prompts/`:
|
|
43
|
+
|
|
44
|
+
- `prompts/analyze-ticket.md`
|
|
45
|
+
- `prompts/generate-code.md`
|
|
46
|
+
- `prompts/generate-test-cases.md`
|
|
47
|
+
|
|
48
|
+
Edit those files in your scaffolded copy to change agent behavior ā no
|
|
49
|
+
code changes needed. The graph reloads them at module-init time.
|
|
50
|
+
|
|
51
|
+
## Cloud deployment
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
zibby workflow deploy <your-slug>
|
|
55
|
+
zibby workflow trigger <uuid> --input '{"workspace": "...", ...}'
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Cloud runs need network access for `git clone` (the `setup` node shells
|
|
59
|
+
out to `git`). Default cloud egress works; pin to a specific IP via the
|
|
60
|
+
dedicated-egress addon if you have firewalled git hosts.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* code-analysis template ā scaffoldable WorkflowAgent wrapper.
|
|
3
|
+
*
|
|
4
|
+
* Why this file exists alongside graph.js:
|
|
5
|
+
* - graph.js exports `buildAnalysisGraph(graph)` ā a builder
|
|
6
|
+
* function the existing `zibby analyze-graph` cli + backend
|
|
7
|
+
* analysis handlers consume directly. Its shape can't change
|
|
8
|
+
* without breaking those callers.
|
|
9
|
+
* - To make code-analysis scaffoldable via
|
|
10
|
+
* `zibby workflow new <slug> -t code-analysis` and runnable via
|
|
11
|
+
* `zibby workflow run <slug>` / `zibby workflow deploy <slug>`,
|
|
12
|
+
* the template needs to expose a WorkflowAgent class ā same
|
|
13
|
+
* contract as browser-test-automation.
|
|
14
|
+
*
|
|
15
|
+
* This .mjs is the entry the user-facing scaffold uses; graph.js
|
|
16
|
+
* stays put for internal callers.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { WorkflowAgent, WorkflowGraph } from '@zibby/core';
|
|
20
|
+
import { buildAnalysisGraph } from './graph.js';
|
|
21
|
+
|
|
22
|
+
export class CodeAnalysisAgent extends WorkflowAgent {
|
|
23
|
+
buildGraph() {
|
|
24
|
+
const graph = new WorkflowGraph();
|
|
25
|
+
buildAnalysisGraph(graph);
|
|
26
|
+
return graph;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async onComplete(result) {
|
|
30
|
+
const ok = result.success !== false;
|
|
31
|
+
console.log(`[code-analysis] complete ā success: ${ok}`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -2,7 +2,7 @@ import { existsSync, readFileSync } from 'fs';
|
|
|
2
2
|
import { join } from 'path';
|
|
3
3
|
import { randomBytes } from 'crypto';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
-
import { adfToText } from '
|
|
5
|
+
import { adfToText } from '@zibby/core/utils/adf-converter.js';
|
|
6
6
|
|
|
7
7
|
const generateId = () => randomBytes(16).toString('hex');
|
|
8
8
|
|
|
@@ -10,7 +10,7 @@ import { existsSync, readFileSync } from 'fs';
|
|
|
10
10
|
import Handlebars from 'handlebars';
|
|
11
11
|
import { invokeAgent } from '@zibby/core';
|
|
12
12
|
import { generatePRMeta } from './services/prMetaService.js';
|
|
13
|
-
import { adfToText } from '
|
|
13
|
+
import { adfToText } from '@zibby/core/utils/adf-converter.js';
|
|
14
14
|
import { z } from 'zod';
|
|
15
15
|
|
|
16
16
|
const CodeImplementationOutputSchema = z.object({
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { invokeAgent } from '@zibby/core';
|
|
12
12
|
import { z } from 'zod';
|
|
13
13
|
import { randomBytes } from 'crypto';
|
|
14
|
-
import { adfToText } from '
|
|
14
|
+
import { adfToText } from '@zibby/core/utils/adf-converter.js';
|
|
15
15
|
|
|
16
16
|
// Generate a simple unique ID
|
|
17
17
|
const generateId = () => randomBytes(8).toString('hex');
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { invokeAgent } from '@zibby/core';
|
|
7
|
-
import { adfToText } from '
|
|
7
|
+
import { adfToText } from '@zibby/core/utils/adf-converter.js';
|
|
8
8
|
|
|
9
9
|
const PRMetaSchema = z.object({
|
|
10
10
|
prTitle: z.string().describe('Short PR title that includes the ticket key'),
|