@canivel/ralph 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/ralph/PROMPT_build.md +126 -0
- package/.agents/ralph/agents.sh +15 -0
- package/.agents/ralph/config.sh +25 -0
- package/.agents/ralph/log-activity.sh +15 -0
- package/.agents/ralph/loop.sh +1001 -0
- package/.agents/ralph/references/CONTEXT_ENGINEERING.md +126 -0
- package/.agents/ralph/references/GUARDRAILS.md +174 -0
- package/AGENTS.md +20 -0
- package/README.md +266 -0
- package/bin/ralph +766 -0
- package/diagram.svg +55 -0
- package/examples/commands.md +46 -0
- package/package.json +39 -0
- package/ralph.webp +0 -0
- package/skills/commit/SKILL.md +219 -0
- package/skills/commit/references/commit_examples.md +292 -0
- package/skills/dev-browser/SKILL.md +211 -0
- package/skills/dev-browser/bun.lock +443 -0
- package/skills/dev-browser/package-lock.json +2988 -0
- package/skills/dev-browser/package.json +31 -0
- package/skills/dev-browser/references/scraping.md +155 -0
- package/skills/dev-browser/scripts/start-relay.ts +32 -0
- package/skills/dev-browser/scripts/start-server.ts +117 -0
- package/skills/dev-browser/server.sh +24 -0
- package/skills/dev-browser/src/client.ts +474 -0
- package/skills/dev-browser/src/index.ts +287 -0
- package/skills/dev-browser/src/relay.ts +731 -0
- package/skills/dev-browser/src/snapshot/__tests__/snapshot.test.ts +223 -0
- package/skills/dev-browser/src/snapshot/browser-script.ts +877 -0
- package/skills/dev-browser/src/snapshot/index.ts +14 -0
- package/skills/dev-browser/src/snapshot/inject.ts +13 -0
- package/skills/dev-browser/src/types.ts +34 -0
- package/skills/dev-browser/tsconfig.json +36 -0
- package/skills/dev-browser/vitest.config.ts +12 -0
- package/skills/prd/SKILL.md +235 -0
- package/tests/agent-loops.mjs +79 -0
- package/tests/agent-ping.mjs +39 -0
- package/tests/audit.md +56 -0
- package/tests/cli-smoke.mjs +47 -0
- package/tests/real-agents.mjs +127 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ARIA Snapshot module for dev-browser.
|
|
3
|
+
*
|
|
4
|
+
* Provides Playwright-compatible ARIA snapshots with cross-connection ref persistence.
|
|
5
|
+
* Refs are stored on window.__devBrowserRefs and survive across Playwright reconnections.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* import { getSnapshotScript } from './snapshot';
|
|
9
|
+
* const script = getSnapshotScript();
|
|
10
|
+
* await page.evaluate(script);
|
|
11
|
+
* // Now window.__devBrowser_getAISnapshot() and window.__devBrowser_selectSnapshotRef(ref) are available
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export { getSnapshotScript, clearSnapshotScriptCache } from "./browser-script";
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Injectable snapshot script for browser context.
|
|
3
|
+
*
|
|
4
|
+
* This module provides the getSnapshotScript function that returns a
|
|
5
|
+
* self-contained JavaScript string for injection into browser contexts.
|
|
6
|
+
*
|
|
7
|
+
* The script is injected via page.evaluate() and exposes:
|
|
8
|
+
* - window.__devBrowser_getAISnapshot(): Returns ARIA snapshot YAML
|
|
9
|
+
* - window.__devBrowser_selectSnapshotRef(ref): Returns element for given ref
|
|
10
|
+
* - window.__devBrowserRefs: Map of ref -> Element (persists across connections)
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
export { getSnapshotScript, clearSnapshotScriptCache } from "./browser-script";
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// API request/response types - shared between client and server
|
|
2
|
+
|
|
3
|
+
export interface ServeOptions {
|
|
4
|
+
port?: number;
|
|
5
|
+
headless?: boolean;
|
|
6
|
+
cdpPort?: number;
|
|
7
|
+
/** Directory to store persistent browser profiles (cookies, localStorage, etc.) */
|
|
8
|
+
profileDir?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface ViewportSize {
|
|
12
|
+
width: number;
|
|
13
|
+
height: number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface GetPageRequest {
|
|
17
|
+
name: string;
|
|
18
|
+
/** Optional viewport size for new pages */
|
|
19
|
+
viewport?: ViewportSize;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface GetPageResponse {
|
|
23
|
+
wsEndpoint: string;
|
|
24
|
+
name: string;
|
|
25
|
+
targetId: string; // CDP target ID for reliable page matching
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface ListPagesResponse {
|
|
29
|
+
pages: string[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface ServerInfoResponse {
|
|
33
|
+
wsEndpoint: string;
|
|
34
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
// Environment setup & latest features
|
|
4
|
+
"lib": ["ESNext"],
|
|
5
|
+
"target": "ESNext",
|
|
6
|
+
"module": "Preserve",
|
|
7
|
+
"moduleDetection": "force",
|
|
8
|
+
"jsx": "react-jsx",
|
|
9
|
+
"allowJs": true,
|
|
10
|
+
|
|
11
|
+
// Bundler mode
|
|
12
|
+
"moduleResolution": "bundler",
|
|
13
|
+
"allowImportingTsExtensions": true,
|
|
14
|
+
"verbatimModuleSyntax": true,
|
|
15
|
+
"noEmit": true,
|
|
16
|
+
|
|
17
|
+
// Path aliases
|
|
18
|
+
"baseUrl": ".",
|
|
19
|
+
"paths": {
|
|
20
|
+
"@/*": ["./src/*"]
|
|
21
|
+
},
|
|
22
|
+
|
|
23
|
+
// Best practices
|
|
24
|
+
"strict": true,
|
|
25
|
+
"skipLibCheck": true,
|
|
26
|
+
"noFallthroughCasesInSwitch": true,
|
|
27
|
+
"noUncheckedIndexedAccess": true,
|
|
28
|
+
"noImplicitOverride": true,
|
|
29
|
+
|
|
30
|
+
// Some stricter flags (disabled by default)
|
|
31
|
+
"noUnusedLocals": false,
|
|
32
|
+
"noUnusedParameters": false,
|
|
33
|
+
"noPropertyAccessFromIndexSignature": false
|
|
34
|
+
},
|
|
35
|
+
"include": ["src/**/*", "scripts/**/*"]
|
|
36
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { defineConfig } from "vitest/config";
|
|
2
|
+
|
|
3
|
+
export default defineConfig({
|
|
4
|
+
test: {
|
|
5
|
+
globals: true,
|
|
6
|
+
environment: "node",
|
|
7
|
+
include: ["src/**/*.test.ts"],
|
|
8
|
+
testTimeout: 60000, // Playwright tests can be slow
|
|
9
|
+
hookTimeout: 60000,
|
|
10
|
+
teardownTimeout: 60000,
|
|
11
|
+
},
|
|
12
|
+
});
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: prd
|
|
3
|
+
description: "Generate a Product Requirements Document (PRD) as JSON for Ralph. Triggers on: create a prd, write prd for, plan this feature, requirements for, spec out."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# PRD Generator (JSON Only)
|
|
7
|
+
|
|
8
|
+
Create a structured JSON PRD that Ralph can execute deterministically. This PRD is the **single source of truth** for stories, gates, and status.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## The Job
|
|
13
|
+
|
|
14
|
+
1. Receive a feature description from the user
|
|
15
|
+
2. Ask **5–10** clarifying questions (with lettered options), in batches of up to 5 at a time (ask 5, wait for answers, then ask the next batch if needed).
|
|
16
|
+
3. **Always ask about quality gates** (commands that must pass)
|
|
17
|
+
4. Generate a **detailed** JSON PRD and save it to the provided path
|
|
18
|
+
|
|
19
|
+
**Important:** Do NOT implement anything. Only generate JSON.
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## Step 1: Clarifying Questions
|
|
24
|
+
|
|
25
|
+
Ask questions even if the prompt seems clear. The goal is to capture missing product details, stack choices, and UI/route structure so the PRD is implementable without guesswork. Focus on:
|
|
26
|
+
|
|
27
|
+
- **Problem/Goal:** What problem does this solve?
|
|
28
|
+
- **Core Functionality:** What are the key actions?
|
|
29
|
+
- **Scope/Boundaries:** What should it NOT do?
|
|
30
|
+
- **Success Criteria:** How do we know it's done?
|
|
31
|
+
- **Stack + Environment:** frameworks, hosting, runtime, database, auth approach
|
|
32
|
+
- **UI + Routes:** key screens, navigation, route map, layout constraints
|
|
33
|
+
- **Data Model + Import Format:** entities, relationships, external data shape
|
|
34
|
+
- **Rules/Calculations:** business logic, progression rules, edge cases
|
|
35
|
+
- **Quality Gates:** tests, lint, typecheck, build/dev verification (REQUIRED)
|
|
36
|
+
|
|
37
|
+
Always ask explicitly:
|
|
38
|
+
- **Is this a new project or an existing codebase?**
|
|
39
|
+
|
|
40
|
+
### Format Questions Like This:
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
1. What is the primary goal of this feature?
|
|
44
|
+
A. Improve user onboarding experience
|
|
45
|
+
B. Increase user retention
|
|
46
|
+
C. Reduce support burden
|
|
47
|
+
D. Other: [please specify]
|
|
48
|
+
|
|
49
|
+
2. Who is the target user?
|
|
50
|
+
A. New users only
|
|
51
|
+
B. Existing users only
|
|
52
|
+
C. All users
|
|
53
|
+
D. Admin users only
|
|
54
|
+
|
|
55
|
+
3. What quality commands must pass for each story?
|
|
56
|
+
A. npm test
|
|
57
|
+
B. npm run lint
|
|
58
|
+
C. npm run typecheck
|
|
59
|
+
D. Other: [specify]
|
|
60
|
+
|
|
61
|
+
Note: All example questions and options in this section are illustrative only. Do not copy them verbatim into the PRD unless the user explicitly chooses them. Derive the final quality gates and requirements from the user's answers and the feature context.
|
|
62
|
+
|
|
63
|
+
4. What stack + hosting are we using (and any constraints)?
|
|
64
|
+
A. React + Vite (static hosting)
|
|
65
|
+
B. Next.js (Node/Edge)
|
|
66
|
+
C. TanStack Start (Cloudflare)
|
|
67
|
+
D. Other: [specify]
|
|
68
|
+
|
|
69
|
+
5. What UI screens/routes are required?
|
|
70
|
+
A. Minimal (1–2 pages)
|
|
71
|
+
B. Basic app shell (dashboard + detail pages)
|
|
72
|
+
C. Full routing map (list all routes)
|
|
73
|
+
D. Other: [specify]
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## Step 2: JSON Structure
|
|
79
|
+
|
|
80
|
+
Output a JSON file with this shape (include detailed top-level fields so the PRD is fully self-contained):
|
|
81
|
+
|
|
82
|
+
```json
|
|
83
|
+
{
|
|
84
|
+
"version": 1,
|
|
85
|
+
"project": "Feature Name",
|
|
86
|
+
"overview": "Short problem + solution summary",
|
|
87
|
+
"goals": [
|
|
88
|
+
"Goal 1",
|
|
89
|
+
"Goal 2"
|
|
90
|
+
],
|
|
91
|
+
"nonGoals": [
|
|
92
|
+
"Explicitly out of scope items"
|
|
93
|
+
],
|
|
94
|
+
"successMetrics": [
|
|
95
|
+
"How success is measured"
|
|
96
|
+
],
|
|
97
|
+
"openQuestions": [
|
|
98
|
+
"Remaining unknowns"
|
|
99
|
+
],
|
|
100
|
+
"stack": {
|
|
101
|
+
"framework": "TanStack Start",
|
|
102
|
+
"hosting": "Cloudflare",
|
|
103
|
+
"database": "D1",
|
|
104
|
+
"auth": "describe approach"
|
|
105
|
+
},
|
|
106
|
+
"routes": [
|
|
107
|
+
{ "path": "/", "name": "Home", "purpose": "..." }
|
|
108
|
+
],
|
|
109
|
+
"uiNotes": [
|
|
110
|
+
"Layout or component requirements"
|
|
111
|
+
],
|
|
112
|
+
"dataModel": [
|
|
113
|
+
{ "entity": "Workout", "fields": ["id", "userId", "date", "notes"] }
|
|
114
|
+
],
|
|
115
|
+
"importFormat": {
|
|
116
|
+
"description": "Expected JSON shape",
|
|
117
|
+
"example": { "programName": "..." }
|
|
118
|
+
},
|
|
119
|
+
"rules": [
|
|
120
|
+
"Key business rules / calculations"
|
|
121
|
+
],
|
|
122
|
+
"qualityGates": ["npm run test:ping"],
|
|
123
|
+
"stories": [
|
|
124
|
+
{
|
|
125
|
+
"id": "US-001",
|
|
126
|
+
"title": "Short story title",
|
|
127
|
+
"status": "open",
|
|
128
|
+
"dependsOn": [],
|
|
129
|
+
"description": "As a [user], I want [feature] so that [benefit].",
|
|
130
|
+
"acceptanceCriteria": [
|
|
131
|
+
"Specific verifiable criterion",
|
|
132
|
+
"Another criterion"
|
|
133
|
+
]
|
|
134
|
+
}
|
|
135
|
+
]
|
|
136
|
+
}
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### Rules
|
|
140
|
+
- **IDs**: Sequential (`US-001`, `US-002`, ...)
|
|
141
|
+
- **Status**: Always `"open"` for new stories
|
|
142
|
+
- **DependsOn**: Use IDs only; empty array if none
|
|
143
|
+
- **Quality Gates**: Only at the top-level `qualityGates`
|
|
144
|
+
- **Acceptance Criteria**: Verifiable, specific, testable
|
|
145
|
+
- **Every story must include**: at least 1 example + 1 negative case
|
|
146
|
+
- **UI stories**: include explicit routes, components, and UI states
|
|
147
|
+
- **New projects**: include initial setup stories (scaffold, env/config, local dev, deploy basics, **package installs**)
|
|
148
|
+
- **Dependencies**: any new package/library introduced must be called out with install commands in acceptance criteria (e.g., `npm install <pkg>`), plus any required config or scripts.
|
|
149
|
+
- **Ordering**: if this is a new project, the **first story must be setup** (scaffold + installs + scripts + env/config). Migrations or data model work come after setup.
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## Output Requirements
|
|
154
|
+
|
|
155
|
+
- Save the JSON to the exact path provided in the prompt (e.g., `.agents/tasks/prd-<slug>.json`)
|
|
156
|
+
- Output **only** the JSON file content (no Markdown PRD)
|
|
157
|
+
- Do not include extra commentary in the file
|
|
158
|
+
|
|
159
|
+
After saving, tell the user:
|
|
160
|
+
`PRD JSON saved to <path>. Close this chat and run \`ralph build\`.`
|
|
161
|
+
|
|
162
|
+
If the prompt provides a **directory** (not a filename), choose a short filename:
|
|
163
|
+
- `prd-<short-slug>.json` where `<short-slug>` is 1–3 meaningful words (avoid filler like “i want to”).
|
|
164
|
+
- Examples: `prd-workout-tracker.json`, `prd-usage-billing.json`
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## Story Size (Critical)
|
|
169
|
+
|
|
170
|
+
Each story must be completable in a **single Ralph iteration**.
|
|
171
|
+
If a story feels too large, split it into multiple smaller stories with dependencies.
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
## Example Output (JSON)
|
|
176
|
+
|
|
177
|
+
```json
|
|
178
|
+
{
|
|
179
|
+
"version": 1,
|
|
180
|
+
"project": "Task Priority System",
|
|
181
|
+
"overview": "Add priority levels to tasks so users can focus on what matters most.",
|
|
182
|
+
"goals": [
|
|
183
|
+
"Allow assigning priority (high/medium/low) to any task",
|
|
184
|
+
"Enable filtering by priority"
|
|
185
|
+
],
|
|
186
|
+
"nonGoals": [
|
|
187
|
+
"No automatic priority assignment"
|
|
188
|
+
],
|
|
189
|
+
"successMetrics": [
|
|
190
|
+
"Users can change priority in under 2 clicks"
|
|
191
|
+
],
|
|
192
|
+
"openQuestions": [
|
|
193
|
+
"Should priority affect ordering within a column?"
|
|
194
|
+
],
|
|
195
|
+
"stack": {
|
|
196
|
+
"framework": "React",
|
|
197
|
+
"hosting": "Cloudflare Pages",
|
|
198
|
+
"database": "D1",
|
|
199
|
+
"auth": "single shared login"
|
|
200
|
+
},
|
|
201
|
+
"routes": [
|
|
202
|
+
{ "path": "/tasks", "name": "Task List", "purpose": "View and filter tasks" },
|
|
203
|
+
{ "path": "/tasks/:id", "name": "Task Detail", "purpose": "Edit task priority" }
|
|
204
|
+
],
|
|
205
|
+
"uiNotes": [
|
|
206
|
+
"Priority badge colors: high=red, medium=yellow, low=gray"
|
|
207
|
+
],
|
|
208
|
+
"dataModel": [
|
|
209
|
+
{ "entity": "Task", "fields": ["id", "title", "priority"] }
|
|
210
|
+
],
|
|
211
|
+
"importFormat": {
|
|
212
|
+
"description": "Not applicable",
|
|
213
|
+
"example": {}
|
|
214
|
+
},
|
|
215
|
+
"rules": [
|
|
216
|
+
"Priority defaults to medium when not set"
|
|
217
|
+
],
|
|
218
|
+
"qualityGates": ["npm run test:ping"],
|
|
219
|
+
"stories": [
|
|
220
|
+
{
|
|
221
|
+
"id": "US-001",
|
|
222
|
+
"title": "Add priority field to database",
|
|
223
|
+
"status": "open",
|
|
224
|
+
"dependsOn": [],
|
|
225
|
+
"description": "As a developer, I want to store task priority so it persists across sessions.",
|
|
226
|
+
"acceptanceCriteria": [
|
|
227
|
+
"Add priority column with default 'medium'",
|
|
228
|
+
"Example: creating a task without priority -> defaults to 'medium'",
|
|
229
|
+
"Negative case: invalid priority 'urgent' -> validation error",
|
|
230
|
+
"Migration runs successfully"
|
|
231
|
+
]
|
|
232
|
+
}
|
|
233
|
+
]
|
|
234
|
+
}
|
|
235
|
+
```
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { spawnSync } from "node:child_process";
|
|
5
|
+
|
|
6
|
+
const repoRoot = path.resolve(new URL("..", import.meta.url).pathname);
|
|
7
|
+
const cliPath = path.join(repoRoot, "bin", "ralph");
|
|
8
|
+
|
|
9
|
+
function run(cmd, args, options = {}) {
|
|
10
|
+
const result = spawnSync(cmd, args, { stdio: "inherit", ...options });
|
|
11
|
+
if (result.status !== 0) {
|
|
12
|
+
console.error(`Command failed: ${cmd} ${args.join(" ")}`);
|
|
13
|
+
process.exit(result.status ?? 1);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function commandExists(cmd) {
|
|
18
|
+
const result = spawnSync(`command -v ${cmd}`, { shell: true, stdio: "ignore" });
|
|
19
|
+
return result.status === 0;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function setupTempProject() {
|
|
23
|
+
const base = mkdtempSync(path.join(tmpdir(), "ralph-smoke-"));
|
|
24
|
+
mkdirSync(path.join(base, ".agents", "tasks"), { recursive: true });
|
|
25
|
+
mkdirSync(path.join(base, ".ralph"), { recursive: true });
|
|
26
|
+
const prd = {
|
|
27
|
+
version: 1,
|
|
28
|
+
project: "Smoke Test",
|
|
29
|
+
qualityGates: [],
|
|
30
|
+
stories: [
|
|
31
|
+
{
|
|
32
|
+
id: "US-001",
|
|
33
|
+
title: "Smoke Test Story",
|
|
34
|
+
status: "open",
|
|
35
|
+
dependsOn: [],
|
|
36
|
+
acceptanceCriteria: [
|
|
37
|
+
"Example: input -> output",
|
|
38
|
+
"Negative case: bad input -> error",
|
|
39
|
+
],
|
|
40
|
+
},
|
|
41
|
+
],
|
|
42
|
+
};
|
|
43
|
+
writeFileSync(
|
|
44
|
+
path.join(base, ".agents", "tasks", "prd.json"),
|
|
45
|
+
`${JSON.stringify(prd, null, 2)}\n`,
|
|
46
|
+
);
|
|
47
|
+
return base;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const agents = ["codex", "claude", "droid"];
|
|
51
|
+
const integration = process.env.RALPH_INTEGRATION === "1";
|
|
52
|
+
|
|
53
|
+
for (const agent of agents) {
|
|
54
|
+
const projectRoot = setupTempProject();
|
|
55
|
+
try {
|
|
56
|
+
const env = { ...process.env };
|
|
57
|
+
if (!integration) {
|
|
58
|
+
env.RALPH_DRY_RUN = "1";
|
|
59
|
+
} else if (agent === "codex" && !commandExists("codex")) {
|
|
60
|
+
console.log(`Skipping codex integration test (missing codex).`);
|
|
61
|
+
continue;
|
|
62
|
+
} else if (agent === "claude" && !commandExists("claude")) {
|
|
63
|
+
console.log(`Skipping claude integration test (missing claude).`);
|
|
64
|
+
continue;
|
|
65
|
+
} else if (agent === "droid" && !commandExists("droid")) {
|
|
66
|
+
console.log(`Skipping droid integration test (missing droid).`);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
run(process.execPath, [cliPath, "build", "1", "--no-commit", `--agent=${agent}`], {
|
|
71
|
+
cwd: projectRoot,
|
|
72
|
+
env,
|
|
73
|
+
});
|
|
74
|
+
} finally {
|
|
75
|
+
rmSync(projectRoot, { recursive: true, force: true });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
console.log("Agent loop smoke tests passed.");
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
|
|
5
|
+
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
6
|
+
const cliPath = path.join(repoRoot, "bin", "ralph");
|
|
7
|
+
|
|
8
|
+
function run(cmd, args, options = {}) {
|
|
9
|
+
const result = spawnSync(cmd, args, { stdio: "inherit", ...options });
|
|
10
|
+
if (result.status !== 0) {
|
|
11
|
+
console.error(`Command failed: ${cmd} ${args.join(" ")}`);
|
|
12
|
+
process.exit(result.status ?? 1);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const agents = [
|
|
17
|
+
{ name: "codex", bin: "codex" },
|
|
18
|
+
{ name: "claude", bin: "claude" },
|
|
19
|
+
{ name: "droid", bin: "droid" },
|
|
20
|
+
{ name: "opencode", bin: "opencode" },
|
|
21
|
+
];
|
|
22
|
+
const runnable = [];
|
|
23
|
+
const skipped = [];
|
|
24
|
+
for (const agent of agents) {
|
|
25
|
+
const check = spawnSync(`command -v ${agent.bin}`, { shell: true, stdio: "ignore" });
|
|
26
|
+
if (check.status === 0) {
|
|
27
|
+
runnable.push(agent.name);
|
|
28
|
+
} else {
|
|
29
|
+
skipped.push(agent.name);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (skipped.length) {
|
|
33
|
+
console.log(`Skipping ping for missing agents: ${skipped.join(", ")}`);
|
|
34
|
+
}
|
|
35
|
+
for (const agent of runnable) {
|
|
36
|
+
run(process.execPath, [cliPath, "ping", `--agent=${agent}`]);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
console.log("Agent ping tests passed.");
|
package/tests/audit.md
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Ralph Test Coverage Audit
|
|
2
|
+
|
|
3
|
+
## Scope reviewed
|
|
4
|
+
- `bin/ralph`
|
|
5
|
+
- `.agents/ralph/loop.sh`
|
|
6
|
+
- `.agents/ralph/PROMPT_build.md`
|
|
7
|
+
- `.agents/ralph/log-activity.sh`
|
|
8
|
+
- `.agents/ralph/references/*`
|
|
9
|
+
- `skills/*`
|
|
10
|
+
- `tests/*`
|
|
11
|
+
|
|
12
|
+
## Current test coverage
|
|
13
|
+
|
|
14
|
+
### CLI
|
|
15
|
+
- `tests/cli-smoke.mjs`
|
|
16
|
+
- Verifies `--help` output
|
|
17
|
+
- Verifies `ralph prd ... --out` creates a PRD JSON file (dry-run)
|
|
18
|
+
- Verifies `ralph overview --prd ...` creates an overview file
|
|
19
|
+
|
|
20
|
+
### Loop (dry-run)
|
|
21
|
+
- `tests/agent-loops.mjs`
|
|
22
|
+
- Runs `ralph build 1 --no-commit` with all agents using `RALPH_DRY_RUN=1`
|
|
23
|
+
- Confirms loops execute and log to stdout without real agent execution
|
|
24
|
+
|
|
25
|
+
### Agent ping (fast, real)
|
|
26
|
+
- `tests/agent-ping.mjs`
|
|
27
|
+
- Runs `ralph ping` for codex/claude/droid
|
|
28
|
+
- Verifies each agent responds with `<end>pong</end>`
|
|
29
|
+
|
|
30
|
+
### Real integration (new)
|
|
31
|
+
- `tests/real-agents.mjs`
|
|
32
|
+
- Creates a temp repo with a 2‑story JSON PRD and minimal AGENTS instructions
|
|
33
|
+
- Runs `ralph build 2 --agent=<codex|claude|droid>`
|
|
34
|
+
- Verifies:
|
|
35
|
+
- PRD stories are all marked `done`
|
|
36
|
+
- At least one git commit was created
|
|
37
|
+
- Progress log exists
|
|
38
|
+
- Cleans up the temp repo and logs after each agent run (pass or fail)
|
|
39
|
+
|
|
40
|
+
## Gaps identified
|
|
41
|
+
- **Interactive installs**: `ralph install` and `ralph install --skills` require prompts and are not exercised in automated tests.
|
|
42
|
+
- **Skill installation paths**: Local vs global skill locations are not validated in tests.
|
|
43
|
+
- **Failure logging**: `.ralph/errors.log` contents are not asserted.
|
|
44
|
+
- **Activity logging**: Activity log presence is not asserted (only implied via run completion).
|
|
45
|
+
- **Guardrails**: No test verifies that guardrails are created/updated.
|
|
46
|
+
|
|
47
|
+
## Recommendations
|
|
48
|
+
1. Keep default tests deterministic (`npm test`) and provide a separate real‑agent test (`npm run test:real`).
|
|
49
|
+
2. Add a non‑interactive flag for skill install if you want it testable in CI.
|
|
50
|
+
3. Add assertions for `.ralph/activity.log` and `.ralph/guardrails.md` creation in `tests/real-agents.mjs`.
|
|
51
|
+
4. Consider adding coverage for PRD selection when multiple JSON files exist.
|
|
52
|
+
|
|
53
|
+
## How to run
|
|
54
|
+
- **Deterministic:** `npm test`
|
|
55
|
+
- **Fast real agent check:** `npm run test:ping`
|
|
56
|
+
- **Real agents:** `npm run test:real` (requires codex/claude/droid installed and authenticated)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { mkdtempSync, existsSync, rmSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
|
|
7
|
+
function run(cmd, args, options = {}) {
|
|
8
|
+
const result = spawnSync(cmd, args, { stdio: "inherit", ...options });
|
|
9
|
+
if (result.status !== 0) {
|
|
10
|
+
console.error(`Command failed: ${cmd} ${args.join(" ")}`);
|
|
11
|
+
process.exit(result.status ?? 1);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
16
|
+
const cliPath = path.join(repoRoot, "bin", "ralph");
|
|
17
|
+
|
|
18
|
+
run(process.execPath, [cliPath, "--help"]);
|
|
19
|
+
|
|
20
|
+
const projectRoot = mkdtempSync(path.join(tmpdir(), "ralph-cli-"));
|
|
21
|
+
try {
|
|
22
|
+
const outPath = path.join(projectRoot, "prd.json");
|
|
23
|
+
run(process.execPath, [cliPath, "prd", "Smoke test PRD", "--out", outPath], {
|
|
24
|
+
cwd: projectRoot,
|
|
25
|
+
env: { ...process.env, RALPH_DRY_RUN: "1" },
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
if (!existsSync(outPath)) {
|
|
29
|
+
console.error("PRD smoke test failed: output not created.");
|
|
30
|
+
process.exit(1);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
run(process.execPath, [cliPath, "overview", "--prd", outPath], {
|
|
34
|
+
cwd: projectRoot,
|
|
35
|
+
env: { ...process.env },
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const overviewPath = outPath.replace(/\.json$/i, ".overview.md");
|
|
39
|
+
if (!existsSync(overviewPath)) {
|
|
40
|
+
console.error("Overview smoke test failed: output not created.");
|
|
41
|
+
process.exit(1);
|
|
42
|
+
}
|
|
43
|
+
} finally {
|
|
44
|
+
rmSync(projectRoot, { recursive: true, force: true });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
console.log("CLI smoke test passed.");
|