@stackmemoryai/stackmemory 1.6.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/commands/orchestrate.js +249 -2
- package/dist/src/cli/commands/orchestrator.js +206 -23
- package/package.json +1 -2
- package/scripts/gepa/.before-optimize.md +0 -112
- package/scripts/gepa/README.md +0 -275
- package/scripts/gepa/config.json +0 -59
- package/scripts/gepa/evals/coding-tasks.jsonl +0 -5
- package/scripts/gepa/evals/fixtures/api-endpoint.ts +0 -31
- package/scripts/gepa/evals/fixtures/brittle-integration.ts +0 -38
- package/scripts/gepa/evals/fixtures/buggy-loop.js +0 -18
- package/scripts/gepa/evals/fixtures/callback-hell.js +0 -53
- package/scripts/gepa/evals/fixtures/fts5-triggers.sql +0 -23
- package/scripts/gepa/evals/fixtures/leaky-service.ts +0 -70
- package/scripts/gepa/evals/fixtures/mcp-dispatch-stub.ts +0 -39
- package/scripts/gepa/evals/fixtures/pr-diff.txt +0 -24
- package/scripts/gepa/evals/fixtures/unsafe-webhook.ts +0 -34
- package/scripts/gepa/evals/fixtures/unwrapped-db-op.ts +0 -42
- package/scripts/gepa/evals/stackmemory-tasks.jsonl +0 -8
- package/scripts/gepa/generations/gen-000/baseline.md +0 -112
- package/scripts/gepa/generations/gen-001/baseline.md +0 -112
- package/scripts/gepa/generations/gen-001/variant-a.md +0 -107
- package/scripts/gepa/generations/gen-001/variant-b.md +0 -216
- package/scripts/gepa/generations/gen-001/variant-c.md +0 -83
- package/scripts/gepa/generations/gen-001/variant-d.md +0 -90
- package/scripts/gepa/hooks/auto-optimize.js +0 -494
- package/scripts/gepa/hooks/eval-tracker.js +0 -203
- package/scripts/gepa/hooks/reflect.js +0 -350
- package/scripts/gepa/optimize.js +0 -853
- package/scripts/gepa/results/eval-1-baseline.json +0 -218
- package/scripts/gepa/results/eval-1-variant-a.json +0 -218
- package/scripts/gepa/results/eval-1-variant-b.json +0 -218
- package/scripts/gepa/results/eval-1-variant-c.json +0 -218
- package/scripts/gepa/results/eval-1-variant-d.json +0 -218
- package/scripts/gepa/state.json +0 -49
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
# CLAUDE.md
|
|
2
|
-
|
|
3
|
-
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
-
|
|
5
|
-
## Project Overview
|
|
6
|
-
|
|
7
|
-
Sol is the monorepo for **Rize**, an automatic time tracking application. The stack consists of:
|
|
8
|
-
- **api/** - Rails 7.1 GraphQL backend (Ruby 3.3.5)
|
|
9
|
-
- **web/** - Next.js 14 React web app (Node 22)
|
|
10
|
-
- **electron/** - Electron desktop app (Node 22)
|
|
11
|
-
- **services/** - Bun-based TypeScript event consumers/workers
|
|
12
|
-
- **vanity/** - Webflow marketing site scripts (deprecated)
|
|
13
|
-
- **voyager/** - Marketing website for home and landing pageas (Next.js)
|
|
14
|
-
- **puppet/** - Puppeteer server for images/PDFs
|
|
15
|
-
- **chrome/** - Chrome browser extension
|
|
16
|
-
- **docs/** - Docusaurus documentation site
|
|
17
|
-
- **zapier/** - Zapier integration
|
|
18
|
-
|
|
19
|
-
## Development Commands
|
|
20
|
-
|
|
21
|
-
### Starting Development Environment
|
|
22
|
-
```bash
|
|
23
|
-
# Start all services (requires iTerm2 on macOS)
|
|
24
|
-
./scripts/run-dev.sh
|
|
25
|
-
|
|
26
|
-
# Or start individually:
|
|
27
|
-
cd api && hivemind Procfile.dev # Rails + AnyCable + Sidekiq + Clockwork
|
|
28
|
-
cd web && npm run dev # Next.js dev server
|
|
29
|
-
cd electron && npm run dev # Electron with hot reload
|
|
30
|
-
cd services && hivemind Procfile.dev # Bun services
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
### Docker Dependencies (api/docker-compose.yml)
|
|
34
|
-
```bash
|
|
35
|
-
cd api && docker-compose up -d
|
|
36
|
-
# TimescaleDB: localhost:15432
|
|
37
|
-
# Redis: localhost:16379
|
|
38
|
-
# Kafka: localhost:9092
|
|
39
|
-
# MySQL: localhost:13306
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
### Testing
|
|
43
|
-
```bash
|
|
44
|
-
# API (RSpec)
|
|
45
|
-
cd api && bundle exec rspec
|
|
46
|
-
cd api && bundle exec rspec spec/path/to/file_spec.rb # Single file
|
|
47
|
-
cd api && bundle exec rspec spec/path/to/file_spec.rb:42 # Single test at line
|
|
48
|
-
|
|
49
|
-
# Electron (Jest)
|
|
50
|
-
cd electron && npm test
|
|
51
|
-
cd electron && npm run test:watch
|
|
52
|
-
cd electron && npm run test:coverage
|
|
53
|
-
|
|
54
|
-
# Web - no active tests (exits 0)
|
|
55
|
-
```
|
|
56
|
-
|
|
57
|
-
### Building
|
|
58
|
-
```bash
|
|
59
|
-
cd api && bundle install && rake db:migrate
|
|
60
|
-
cd web && npm run build # Runs gql-gen, tailwind, next build
|
|
61
|
-
cd electron && npm run build # Electron Forge make
|
|
62
|
-
cd services && bun install
|
|
63
|
-
```
|
|
64
|
-
|
|
65
|
-
### GraphQL Code Generation
|
|
66
|
-
```bash
|
|
67
|
-
cd web && npm run build # Includes gql codegen
|
|
68
|
-
cd electron && npm run dev # Runs gql codegen as part of dev
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
## Architecture
|
|
72
|
-
|
|
73
|
-
### GraphQL API Structure
|
|
74
|
-
The API exposes two GraphQL endpoints:
|
|
75
|
-
- **api/v1** - Public API (OAuth consumers, Zapier)
|
|
76
|
-
- **private/v1** - Private API (web, electron apps)
|
|
77
|
-
|
|
78
|
-
Located at `api/app/graphql/{api,private}/v1/`
|
|
79
|
-
|
|
80
|
-
### Real-time Communication
|
|
81
|
-
- **AnyCable** WebSocket server for subscriptions
|
|
82
|
-
- ActionCable channels in `api/app/channels/`
|
|
83
|
-
- WebSocket config: `api/config/cable.yml` and `api/config/anycable.yml`
|
|
84
|
-
|
|
85
|
-
### Background Jobs
|
|
86
|
-
- **Sidekiq** for async job processing (`api/config/sidekiq.yml`)
|
|
87
|
-
- **Clockwork** for scheduled jobs (`api/config/clock.rb`)
|
|
88
|
-
|
|
89
|
-
### Event Streaming
|
|
90
|
-
- **Kafka** for event publishing/consumption
|
|
91
|
-
- Services consume events via `services/consumers/`
|
|
92
|
-
- Kafka config: `api/config/initializers/kafka.rb`
|
|
93
|
-
|
|
94
|
-
### Databases
|
|
95
|
-
- **Primary PostgreSQL** - Main application data
|
|
96
|
-
- **TimescaleDB** - Time-series data (separate connection in `database.yml`)
|
|
97
|
-
- **MySQL** - Legacy/external integrations
|
|
98
|
-
- **Redis** - Caching, ActionCable, Sidekiq
|
|
99
|
-
|
|
100
|
-
## Style Guidelines
|
|
101
|
-
|
|
102
|
-
### JavaScript/TypeScript
|
|
103
|
-
- Use `test()` instead of `it()` in tests
|
|
104
|
-
- Use `toBeCalled()` instead of `toHaveBeenCalledWith()` in jest assertions
|
|
105
|
-
|
|
106
|
-
## Key Configuration Files
|
|
107
|
-
|
|
108
|
-
- `api/config/database.yml` - Database connections (primary + timescale)
|
|
109
|
-
- `api/config/cable.yml` - AnyCable WebSocket config
|
|
110
|
-
- `api/Procfile.dev` - Development processes (rails, anycable, sidekiq, clockwork)
|
|
111
|
-
- `sol.code-workspace` - VS Code multi-folder workspace
|
|
112
|
-
- Each project requires its own `.env` file (not in repo)
|
package/scripts/gepa/README.md
DELETED
|
@@ -1,275 +0,0 @@
|
|
|
1
|
-
# Prompt Forge (GEPA)
|
|
2
|
-
|
|
3
|
-
**Genetic Eval-driven Prompt Algorithm**
|
|
4
|
-
|
|
5
|
-
Automatically evolve and optimize your `CLAUDE.md` system prompts using AI-powered evolutionary algorithms.
|
|
6
|
-
|
|
7
|
-
## Auto-Optimize on Save
|
|
8
|
-
|
|
9
|
-
```bash
|
|
10
|
-
# Start watcher - auto-optimizes when CLAUDE.md changes
|
|
11
|
-
node scripts/gepa/hooks/auto-optimize.js watch ./CLAUDE.md
|
|
12
|
-
```
|
|
13
|
-
|
|
14
|
-
Output shows before/after comparison:
|
|
15
|
-
```
|
|
16
|
-
╔════════════════════════════════════════════════════════════╗
|
|
17
|
-
║ BEFORE / AFTER COMPARISON ║
|
|
18
|
-
╠════════════════════════════════════════════════════════════╣
|
|
19
|
-
║ Metric Before After Change ║
|
|
20
|
-
╠════════════════════════════════════════════════════════════╣
|
|
21
|
-
║ Lines 125 142 +17 (+14%) ║
|
|
22
|
-
║ Est. Tokens 873 920 +47 (+5%) ║
|
|
23
|
-
║ MUST rules 1 3 +2 (+200%) ║
|
|
24
|
-
║ NEVER rules 3 5 +2 (+67%) ║
|
|
25
|
-
╚════════════════════════════════════════════════════════════╝
|
|
26
|
-
|
|
27
|
-
Section Changes:
|
|
28
|
-
Added:
|
|
29
|
-
+ Error Handling
|
|
30
|
-
+ Performance Guidelines
|
|
31
|
-
|
|
32
|
-
Summary:
|
|
33
|
-
Token budget: +47 tokens
|
|
34
|
-
Rule density: +4 explicit rules
|
|
35
|
-
```
|
|
36
|
-
|
|
37
|
-
## How It Works
|
|
38
|
-
|
|
39
|
-
```
|
|
40
|
-
┌─────────────────────────────────────────────────────────────────┐
|
|
41
|
-
│ GEPA Loop │
|
|
42
|
-
│ │
|
|
43
|
-
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
|
44
|
-
│ │ Seed │───►│ Mutate │───►│ Eval │───►│ Select │ │
|
|
45
|
-
│ │ Prompt │ │ (AI gen) │ │ (Claude) │ │ (best) │ │
|
|
46
|
-
│ └──────────┘ └──────────┘ └──────────┘ └────┬─────┘ │
|
|
47
|
-
│ ▲ │ │
|
|
48
|
-
│ │ ┌──────────┐ │ │
|
|
49
|
-
│ └──────────────┤ Reflect │◄────────────────────┘ │
|
|
50
|
-
│ │(insights)│ │
|
|
51
|
-
│ └──────────┘ │
|
|
52
|
-
└─────────────────────────────────────────────────────────────────┘
|
|
53
|
-
```
|
|
54
|
-
|
|
55
|
-
## Quick Start
|
|
56
|
-
|
|
57
|
-
```bash
|
|
58
|
-
# 1. Initialize with your current CLAUDE.md
|
|
59
|
-
node scripts/gepa/optimize.js init ./CLAUDE.md
|
|
60
|
-
|
|
61
|
-
# 2. Run full optimization (10 generations)
|
|
62
|
-
node scripts/gepa/optimize.js run
|
|
63
|
-
|
|
64
|
-
# 3. Apply the best result
|
|
65
|
-
cp scripts/gepa/generations/current ./CLAUDE.md
|
|
66
|
-
```
|
|
67
|
-
|
|
68
|
-
## Commands
|
|
69
|
-
|
|
70
|
-
| Command | Description |
|
|
71
|
-
|---------|-------------|
|
|
72
|
-
| `init [path]` | Initialize with a CLAUDE.md file |
|
|
73
|
-
| `mutate` | Generate new prompt variants |
|
|
74
|
-
| `eval [variant]` | Run evals on a specific variant |
|
|
75
|
-
| `score` | Score all variants and select best |
|
|
76
|
-
| `run [N]` | Full optimization loop for N generations |
|
|
77
|
-
| `status` | Show current optimization status |
|
|
78
|
-
| `diff [a] [b]` | Compare two variants |
|
|
79
|
-
|
|
80
|
-
## Mutation Strategies
|
|
81
|
-
|
|
82
|
-
GEPA uses 6 mutation strategies, cycling through them:
|
|
83
|
-
|
|
84
|
-
1. **rephrase** - Reword for clarity without changing meaning
|
|
85
|
-
2. **add_examples** - Add concrete examples where abstract
|
|
86
|
-
3. **remove_redundancy** - DRY up repetitive instructions
|
|
87
|
-
4. **restructure** - Reorganize for better flow
|
|
88
|
-
5. **add_constraints** - Add guardrails for failure modes
|
|
89
|
-
6. **simplify** - Break down complex rules
|
|
90
|
-
|
|
91
|
-
## Reflection (Key Innovation)
|
|
92
|
-
|
|
93
|
-
Unlike random mutations, GEPA analyzes **why** prompts fail:
|
|
94
|
-
|
|
95
|
-
```bash
|
|
96
|
-
# Analyze session patterns
|
|
97
|
-
node scripts/gepa/hooks/reflect.js analyze
|
|
98
|
-
|
|
99
|
-
# Generate targeted improvements
|
|
100
|
-
node scripts/gepa/hooks/reflect.js reflect
|
|
101
|
-
```
|
|
102
|
-
|
|
103
|
-
The reflection engine examines:
|
|
104
|
-
- Common error patterns
|
|
105
|
-
- Tool call success rates
|
|
106
|
-
- User feedback (thumbs up/down)
|
|
107
|
-
- Performance by variant
|
|
108
|
-
|
|
109
|
-
Then generates **targeted** mutation suggestions.
|
|
110
|
-
|
|
111
|
-
## Hook Integration
|
|
112
|
-
|
|
113
|
-
To track real usage for evals, add to `~/.claude/settings.json`:
|
|
114
|
-
|
|
115
|
-
```json
|
|
116
|
-
{
|
|
117
|
-
"hooks": {
|
|
118
|
-
"postToolCall": [
|
|
119
|
-
{
|
|
120
|
-
"command": "node ~/.claude/gepa/hooks/eval-tracker.js track-tool"
|
|
121
|
-
}
|
|
122
|
-
],
|
|
123
|
-
"postSession": [
|
|
124
|
-
{
|
|
125
|
-
"command": "node ~/.claude/gepa/hooks/eval-tracker.js save"
|
|
126
|
-
}
|
|
127
|
-
]
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
```
|
|
131
|
-
|
|
132
|
-
Or use the StackMemory daemon integration:
|
|
133
|
-
|
|
134
|
-
```bash
|
|
135
|
-
# Add to your .env
|
|
136
|
-
GEPA_ENABLED=true
|
|
137
|
-
GEPA_DIR=~/.claude/gepa
|
|
138
|
-
```
|
|
139
|
-
|
|
140
|
-
## Configuration
|
|
141
|
-
|
|
142
|
-
Edit `config.json`:
|
|
143
|
-
|
|
144
|
-
```json
|
|
145
|
-
{
|
|
146
|
-
"evolution": {
|
|
147
|
-
"populationSize": 4, // Variants per generation
|
|
148
|
-
"generations": 10, // Max generations
|
|
149
|
-
"selectionRate": 0.5 // Top 50% survive
|
|
150
|
-
},
|
|
151
|
-
"evals": {
|
|
152
|
-
"minSamplesPerVariant": 5, // Evals per variant
|
|
153
|
-
"timeout": 120000 // 2 min per eval
|
|
154
|
-
},
|
|
155
|
-
"scoring": {
|
|
156
|
-
"threshold": 0.8 // Stop when 80% success
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
```
|
|
160
|
-
|
|
161
|
-
## Writing Good Evals
|
|
162
|
-
|
|
163
|
-
Evals live in `evals/*.jsonl`:
|
|
164
|
-
|
|
165
|
-
```json
|
|
166
|
-
{
|
|
167
|
-
"id": "eval-001",
|
|
168
|
-
"name": "simple_function",
|
|
169
|
-
"prompt": "Write a function that checks if a string is a palindrome",
|
|
170
|
-
"expected": {
|
|
171
|
-
"has_function": true,
|
|
172
|
-
"handles_edge_cases": true
|
|
173
|
-
},
|
|
174
|
-
"weight": 1.0
|
|
175
|
-
}
|
|
176
|
-
```
|
|
177
|
-
|
|
178
|
-
### Expected Checks
|
|
179
|
-
|
|
180
|
-
| Check | What It Looks For |
|
|
181
|
-
|-------|-------------------|
|
|
182
|
-
| `has_function` | Function definition in output |
|
|
183
|
-
| `handles_edge_cases` | Null/empty/edge case handling |
|
|
184
|
-
| `uses_async` | async/await usage |
|
|
185
|
-
| `bug_fixed` | Fix-related language |
|
|
186
|
-
| `explains_fix` | Explanation of changes |
|
|
187
|
-
| Custom key | Looks for key as substring |
|
|
188
|
-
|
|
189
|
-
## Directory Structure
|
|
190
|
-
|
|
191
|
-
```
|
|
192
|
-
scripts/gepa/
|
|
193
|
-
├── config.json # Settings
|
|
194
|
-
├── state.json # Current state
|
|
195
|
-
├── optimize.js # Main optimizer
|
|
196
|
-
├── hooks/
|
|
197
|
-
│ ├── eval-tracker.js # Session tracking hook
|
|
198
|
-
│ └── reflect.js # Reflection engine
|
|
199
|
-
├── evals/
|
|
200
|
-
│ └── coding-tasks.jsonl
|
|
201
|
-
├── generations/
|
|
202
|
-
│ ├── gen-000/
|
|
203
|
-
│ │ └── baseline.md
|
|
204
|
-
│ ├── gen-001/
|
|
205
|
-
│ │ ├── variant-a.md
|
|
206
|
-
│ │ ├── variant-b.md
|
|
207
|
-
│ │ └── baseline.md
|
|
208
|
-
│ └── current -> gen-001/variant-a.md
|
|
209
|
-
└── results/
|
|
210
|
-
├── scores.jsonl
|
|
211
|
-
└── sessions/
|
|
212
|
-
```
|
|
213
|
-
|
|
214
|
-
## Best Practices
|
|
215
|
-
|
|
216
|
-
1. **Start with good evals** - Garbage in, garbage out
|
|
217
|
-
2. **Run multiple generations** - Improvements compound
|
|
218
|
-
3. **Review diffs** - Understand what changed
|
|
219
|
-
4. **Keep baseline** - Always compare against original
|
|
220
|
-
5. **Monitor for drift** - Watch for unintended changes
|
|
221
|
-
|
|
222
|
-
## Example Output
|
|
223
|
-
|
|
224
|
-
```
|
|
225
|
-
$ node optimize.js run 3
|
|
226
|
-
|
|
227
|
-
============================================================
|
|
228
|
-
GENERATION 1/3
|
|
229
|
-
============================================================
|
|
230
|
-
|
|
231
|
-
Generating 4 variants for generation 1...
|
|
232
|
-
Creating variant-a using strategy: rephrase
|
|
233
|
-
Creating variant-b using strategy: add_examples
|
|
234
|
-
Creating variant-c using strategy: remove_redundancy
|
|
235
|
-
Creating variant-d using strategy: restructure
|
|
236
|
-
|
|
237
|
-
Scoring 5 variants in generation 1...
|
|
238
|
-
Running evals on baseline... Score: 65.0%
|
|
239
|
-
Running evals on variant-a... Score: 72.0%
|
|
240
|
-
Running evals on variant-b... Score: 78.0%
|
|
241
|
-
Running evals on variant-c... Score: 70.0%
|
|
242
|
-
Running evals on variant-d... Score: 68.0%
|
|
243
|
-
|
|
244
|
-
Results:
|
|
245
|
-
1. variant-b: 78.0% <-- BEST
|
|
246
|
-
2. variant-a: 72.0%
|
|
247
|
-
3. variant-c: 70.0%
|
|
248
|
-
4. variant-d: 68.0%
|
|
249
|
-
5. baseline: 65.0%
|
|
250
|
-
|
|
251
|
-
New best: variant-b (78.0%)
|
|
252
|
-
|
|
253
|
-
============================================================
|
|
254
|
-
OPTIMIZATION COMPLETE
|
|
255
|
-
============================================================
|
|
256
|
-
Best variant: variant-b
|
|
257
|
-
Best score: 85.2%
|
|
258
|
-
Generations: 3
|
|
259
|
-
|
|
260
|
-
To apply: cp generations/current /path/to/your/CLAUDE.md
|
|
261
|
-
```
|
|
262
|
-
|
|
263
|
-
## Troubleshooting
|
|
264
|
-
|
|
265
|
-
**"claude CLI not found"**
|
|
266
|
-
Set `ANTHROPIC_API_KEY` for API fallback.
|
|
267
|
-
|
|
268
|
-
**Slow evals**
|
|
269
|
-
Reduce `minSamplesPerVariant` in config.
|
|
270
|
-
|
|
271
|
-
**Poor results**
|
|
272
|
-
Add more diverse evals covering failure modes.
|
|
273
|
-
|
|
274
|
-
**Drift from original intent**
|
|
275
|
-
Add evals that test for desired behaviors explicitly.
|
package/scripts/gepa/config.json
DELETED
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "GEPA - Genetic Eval-driven Prompt Algorithm",
|
|
3
|
-
"version": "1.0.0",
|
|
4
|
-
|
|
5
|
-
"target": {
|
|
6
|
-
"file": "CLAUDE.md",
|
|
7
|
-
"scope": "project",
|
|
8
|
-
"backup": true
|
|
9
|
-
},
|
|
10
|
-
|
|
11
|
-
"evolution": {
|
|
12
|
-
"populationSize": 4,
|
|
13
|
-
"generations": 10,
|
|
14
|
-
"selectionRate": 0.5,
|
|
15
|
-
"mutationStrategies": [
|
|
16
|
-
"rephrase",
|
|
17
|
-
"add_examples",
|
|
18
|
-
"remove_redundancy",
|
|
19
|
-
"restructure",
|
|
20
|
-
"add_constraints",
|
|
21
|
-
"simplify"
|
|
22
|
-
]
|
|
23
|
-
},
|
|
24
|
-
|
|
25
|
-
"evals": {
|
|
26
|
-
"directory": "./evals",
|
|
27
|
-
"minSamplesPerVariant": 8,
|
|
28
|
-
"timeout": 120000,
|
|
29
|
-
"metrics": [
|
|
30
|
-
"task_completion",
|
|
31
|
-
"code_quality",
|
|
32
|
-
"instruction_following",
|
|
33
|
-
"efficiency"
|
|
34
|
-
]
|
|
35
|
-
},
|
|
36
|
-
|
|
37
|
-
"judge": {
|
|
38
|
-
"model": "claude-haiku-4-5-20251001",
|
|
39
|
-
"maxOutputTokens": 2000,
|
|
40
|
-
"timeoutMs": 30000
|
|
41
|
-
},
|
|
42
|
-
|
|
43
|
-
"scoring": {
|
|
44
|
-
"weights": {
|
|
45
|
-
"task_completion": 0.4,
|
|
46
|
-
"code_quality": 0.25,
|
|
47
|
-
"instruction_following": 0.25,
|
|
48
|
-
"efficiency": 0.1
|
|
49
|
-
},
|
|
50
|
-
"threshold": 0.7
|
|
51
|
-
},
|
|
52
|
-
|
|
53
|
-
"hooks": {
|
|
54
|
-
"trackToolCalls": true,
|
|
55
|
-
"trackTokenUsage": true,
|
|
56
|
-
"trackErrors": true,
|
|
57
|
-
"trackUserFeedback": true
|
|
58
|
-
}
|
|
59
|
-
}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
{"id": "eval-001", "name": "simple_function", "prompt": "Write a function that checks if a string is a palindrome", "expected": {"has_function": true, "handles_edge_cases": true, "is_tested": false}, "weight": 1.0}
|
|
2
|
-
{"id": "eval-002", "name": "refactor_code", "prompt": "Refactor this code to use async/await instead of callbacks", "input_file": "fixtures/callback-hell.js", "expected": {"uses_async": true, "preserves_behavior": true, "no_nested_callbacks": true}, "weight": 1.5}
|
|
3
|
-
{"id": "eval-003", "name": "fix_bug", "prompt": "Fix the off-by-one error in this loop", "input_file": "fixtures/buggy-loop.js", "expected": {"bug_fixed": true, "no_new_bugs": true, "explains_fix": true}, "weight": 1.2}
|
|
4
|
-
{"id": "eval-004", "name": "add_feature", "prompt": "Add pagination to this API endpoint", "input_file": "fixtures/api-endpoint.ts", "expected": {"has_pagination": true, "has_limit": true, "has_offset": true, "backwards_compatible": true}, "weight": 1.5}
|
|
5
|
-
{"id": "eval-005", "name": "code_review", "prompt": "Review this PR and identify issues", "input_file": "fixtures/pr-diff.txt", "expected": {"identifies_security_issue": true, "identifies_performance_issue": true, "actionable_feedback": true}, "weight": 1.3}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
// Simple API endpoint that needs pagination added
|
|
2
|
-
import express from 'express';
|
|
3
|
-
|
|
4
|
-
const router = express.Router();
|
|
5
|
-
|
|
6
|
-
interface User {
|
|
7
|
-
id: number;
|
|
8
|
-
name: string;
|
|
9
|
-
email: string;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
// In-memory store
|
|
13
|
-
const users: User[] = Array.from({ length: 100 }, (_, i) => ({
|
|
14
|
-
id: i + 1,
|
|
15
|
-
name: `User ${i + 1}`,
|
|
16
|
-
email: `user${i + 1}@example.com`,
|
|
17
|
-
}));
|
|
18
|
-
|
|
19
|
-
// GET /users - returns ALL users (no pagination)
|
|
20
|
-
router.get('/users', (req, res) => {
|
|
21
|
-
res.json(users);
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
// GET /users/:id
|
|
25
|
-
router.get('/users/:id', (req, res) => {
|
|
26
|
-
const user = users.find((u) => u.id === parseInt(req.params.id));
|
|
27
|
-
if (!user) return res.status(404).json({ error: 'Not found' });
|
|
28
|
-
res.json(user);
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
export default router;
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
// Integration handler that crashes on external API failure
|
|
2
|
-
// Needs graceful degradation following the DiffMem/Greptile pattern
|
|
3
|
-
|
|
4
|
-
interface MCPResponse {
|
|
5
|
-
content: Array<{ type: string; text: string }>;
|
|
6
|
-
metadata?: Record<string, unknown>;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
// This handler crashes the entire MCP server when the API is unreachable.
|
|
10
|
-
// Refactor to degrade gracefully.
|
|
11
|
-
async function handleExternalLookup(args: {
|
|
12
|
-
query: string;
|
|
13
|
-
limit?: number;
|
|
14
|
-
}): Promise<MCPResponse> {
|
|
15
|
-
const response = await fetch('https://api.external-service.com/search', {
|
|
16
|
-
method: 'POST',
|
|
17
|
-
headers: { 'Content-Type': 'application/json' },
|
|
18
|
-
body: JSON.stringify({ query: args.query, limit: args.limit || 10 }),
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
// BUG: No error handling — fetch failures (ECONNREFUSED, DNS, timeout)
|
|
22
|
-
// will throw and crash the MCP server process.
|
|
23
|
-
// BUG: 4xx errors (bad request, auth failed) should not be retried.
|
|
24
|
-
// BUG: Logs at error level, flooding logs when service is down.
|
|
25
|
-
|
|
26
|
-
if (!response.ok) {
|
|
27
|
-
throw new Error(`API error: ${response.status}`);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
const data = await response.json();
|
|
31
|
-
|
|
32
|
-
return {
|
|
33
|
-
content: [{ type: 'text', text: JSON.stringify(data, null, 2) }],
|
|
34
|
-
metadata: { tool: 'external_lookup', resultCount: data.results?.length },
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export { handleExternalLookup };
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
// Bug: Off-by-one error - processes one too many items
|
|
2
|
-
function processItems(items) {
|
|
3
|
-
const results = [];
|
|
4
|
-
|
|
5
|
-
// BUG: should be i < items.length, not i <= items.length
|
|
6
|
-
for (let i = 0; i <= items.length; i++) {
|
|
7
|
-
const item = items[i];
|
|
8
|
-
results.push(item.toUpperCase()); // Will crash on undefined
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
return results;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
// Test
|
|
15
|
-
const items = ['apple', 'banana', 'cherry'];
|
|
16
|
-
console.log(processItems(items)); // Crashes!
|
|
17
|
-
|
|
18
|
-
module.exports = { processItems };
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
// Callback hell example - needs refactoring to async/await
|
|
2
|
-
function fetchUserData(userId, callback) {
|
|
3
|
-
getUser(userId, function (err, user) {
|
|
4
|
-
if (err) {
|
|
5
|
-
callback(err);
|
|
6
|
-
return;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
getPosts(user.id, function (err, posts) {
|
|
10
|
-
if (err) {
|
|
11
|
-
callback(err);
|
|
12
|
-
return;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
getComments(posts[0].id, function (err, comments) {
|
|
16
|
-
if (err) {
|
|
17
|
-
callback(err);
|
|
18
|
-
return;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
getLikes(comments[0].id, function (err, likes) {
|
|
22
|
-
if (err) {
|
|
23
|
-
callback(err);
|
|
24
|
-
return;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
callback(null, {
|
|
28
|
-
user: user,
|
|
29
|
-
posts: posts,
|
|
30
|
-
comments: comments,
|
|
31
|
-
likes: likes,
|
|
32
|
-
});
|
|
33
|
-
});
|
|
34
|
-
});
|
|
35
|
-
});
|
|
36
|
-
});
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
// Mock functions
|
|
40
|
-
function getUser(id, cb) {
|
|
41
|
-
cb(null, { id, name: 'Test' });
|
|
42
|
-
}
|
|
43
|
-
function getPosts(userId, cb) {
|
|
44
|
-
cb(null, [{ id: 1, title: 'Post' }]);
|
|
45
|
-
}
|
|
46
|
-
function getComments(postId, cb) {
|
|
47
|
-
cb(null, [{ id: 1, text: 'Comment' }]);
|
|
48
|
-
}
|
|
49
|
-
function getLikes(commentId, cb) {
|
|
50
|
-
cb(null, [{ id: 1 }]);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
module.exports = { fetchUserData };
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
-- FTS5 external content table for full-text search on frames
|
|
2
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS frames_fts USING fts5(
|
|
3
|
-
name, digest_text, inputs, outputs,
|
|
4
|
-
content='frames', content_rowid='rowid'
|
|
5
|
-
);
|
|
6
|
-
|
|
7
|
-
-- Sync trigger: INSERT
|
|
8
|
-
CREATE TRIGGER IF NOT EXISTS frames_ai AFTER INSERT ON frames BEGIN
|
|
9
|
-
INSERT INTO frames_fts(rowid, name, digest_text, inputs, outputs)
|
|
10
|
-
VALUES (new.rowid, new.name, new.digest_text, new.inputs, new.outputs);
|
|
11
|
-
END;
|
|
12
|
-
|
|
13
|
-
-- Sync trigger: UPDATE
|
|
14
|
-
CREATE TRIGGER IF NOT EXISTS frames_au AFTER UPDATE ON frames BEGIN
|
|
15
|
-
INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)
|
|
16
|
-
VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);
|
|
17
|
-
INSERT INTO frames_fts(rowid, name, digest_text, inputs, outputs)
|
|
18
|
-
VALUES (new.rowid, new.name, new.digest_text, new.inputs, new.outputs);
|
|
19
|
-
END;
|
|
20
|
-
|
|
21
|
-
-- BUG: Missing DELETE trigger!
|
|
22
|
-
-- When a frame is deleted, the FTS index still contains stale data.
|
|
23
|
-
-- Add the missing AFTER DELETE trigger below.
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
// Daemon service with timer leak bug
|
|
2
|
-
interface ServiceConfig {
|
|
3
|
-
enabled: boolean;
|
|
4
|
-
interval: number; // minutes
|
|
5
|
-
}
|
|
6
|
-
|
|
7
|
-
interface ServiceState {
|
|
8
|
-
isRunning: boolean;
|
|
9
|
-
intervalMs: number;
|
|
10
|
-
lastRunTime: number;
|
|
11
|
-
errorCount: number;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
class MonitorService {
|
|
15
|
-
private config: ServiceConfig;
|
|
16
|
-
private intervalId?: NodeJS.Timeout;
|
|
17
|
-
private isRunning = false;
|
|
18
|
-
private lastRunTime = 0;
|
|
19
|
-
private errorCount = 0;
|
|
20
|
-
|
|
21
|
-
constructor(config: ServiceConfig) {
|
|
22
|
-
this.config = config;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
// BUG: No guard against double-start — calling start() twice
|
|
26
|
-
// creates two intervals but only stores the second one.
|
|
27
|
-
// The first interval leaks and keeps running forever.
|
|
28
|
-
start(): void {
|
|
29
|
-
this.isRunning = true;
|
|
30
|
-
const intervalMs = this.config.interval * 60 * 1000;
|
|
31
|
-
|
|
32
|
-
this.doWork(); // initial run
|
|
33
|
-
|
|
34
|
-
this.intervalId = setInterval(() => {
|
|
35
|
-
this.doWork();
|
|
36
|
-
}, intervalMs);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
stop(): void {
|
|
40
|
-
if (this.intervalId) {
|
|
41
|
-
clearInterval(this.intervalId);
|
|
42
|
-
this.intervalId = undefined;
|
|
43
|
-
}
|
|
44
|
-
this.isRunning = false;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
// BUG: Calls stop() then start() but start() doesn't check
|
|
48
|
-
// if already running, so if stop() fails to clear the interval
|
|
49
|
-
// (e.g., intervalId is undefined), we leak.
|
|
50
|
-
updateConfig(config: Partial<ServiceConfig>): void {
|
|
51
|
-
const wasRunning = this.isRunning;
|
|
52
|
-
if (wasRunning) this.stop();
|
|
53
|
-
this.config = { ...this.config, ...config };
|
|
54
|
-
if (wasRunning && this.config.enabled) this.start();
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// TODO: Add getState() method returning ServiceState
|
|
58
|
-
|
|
59
|
-
private doWork(): void {
|
|
60
|
-
try {
|
|
61
|
-
// Simulate work
|
|
62
|
-
console.log('Running monitor check...');
|
|
63
|
-
this.lastRunTime = Date.now();
|
|
64
|
-
} catch (err) {
|
|
65
|
-
this.errorCount++;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
export { MonitorService, ServiceConfig, ServiceState };
|