fraim-framework 2.0.27 → 2.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/deploy-fraim.yml +1 -1
- package/dist/registry/scripts/build-scripts-generator.js +205 -0
- package/dist/registry/scripts/cleanup-branch.js +258 -0
- package/dist/registry/scripts/evaluate-code-quality.js +66 -0
- package/dist/registry/scripts/exec-with-timeout.js +142 -0
- package/dist/registry/scripts/fraim-config.js +61 -0
- package/dist/registry/scripts/generate-engagement-emails.js +630 -0
- package/dist/registry/scripts/generic-issues-api.js +100 -0
- package/dist/registry/scripts/newsletter-helpers.js +731 -0
- package/dist/registry/scripts/openapi-generator.js +664 -0
- package/{registry/scripts/performance/profile-server.ts → dist/registry/scripts/performance/profile-server.js} +390 -368
- package/dist/registry/scripts/run-thank-you-workflow.js +92 -0
- package/dist/registry/scripts/send-newsletter-simple.js +85 -0
- package/dist/registry/scripts/send-thank-you-emails.js +54 -0
- package/dist/registry/scripts/validate-openapi-limits.js +311 -0
- package/dist/registry/scripts/validate-test-coverage.js +262 -0
- package/dist/registry/scripts/verify-test-coverage.js +66 -0
- package/dist/src/cli/commands/init.js +42 -15
- package/dist/src/cli/commands/sync.js +22 -5
- package/dist/src/cli/fraim.js +24 -22
- package/dist/src/cli/setup/first-run.js +13 -6
- package/dist/src/fraim/db-service.js +26 -15
- package/dist/src/fraim/issues.js +67 -0
- package/dist/src/fraim-mcp-server.js +272 -18
- package/dist/src/utils/git-utils.js +1 -1
- package/dist/src/utils/script-sync-utils.js +218 -0
- package/dist/tests/debug-tools.js +80 -0
- package/dist/tests/esm-compat.js +11 -0
- package/dist/tests/test-chalk-esm-issue.js +159 -0
- package/dist/tests/test-chalk-real-world.js +265 -0
- package/dist/tests/test-chalk-regression.js +377 -0
- package/dist/tests/test-chalk-resolution-issue.js +304 -0
- package/dist/tests/test-cli.js +70 -5
- package/dist/tests/test-end-to-end-hybrid-validation.js +349 -0
- package/dist/tests/test-first-run-journey.js +43 -3
- package/dist/tests/test-fraim-install-chalk-issue.js +254 -0
- package/dist/tests/test-fraim-issues.js +59 -0
- package/dist/tests/test-genericization.js +1 -1
- package/dist/tests/test-hybrid-script-execution.js +369 -0
- package/dist/tests/test-mcp-connection.js +166 -0
- package/dist/tests/test-mcp-issue-integration.js +152 -0
- package/dist/tests/test-mcp-lifecycle-methods.js +312 -0
- package/dist/tests/test-node-compatibility.js +93 -0
- package/dist/tests/test-npm-install.js +66 -0
- package/dist/tests/test-npm-resolution-diagnostic.js +140 -0
- package/dist/tests/test-prep-issue.js +4 -1
- package/dist/tests/test-script-location-independence.js +173 -0
- package/dist/tests/test-script-sync.js +557 -0
- package/dist/tests/test-session-rehydration.js +145 -0
- package/dist/tests/test-standalone.js +5 -11
- package/dist/tests/test-sync-version-update.js +1 -1
- package/dist/tests/test-telemetry.js +190 -0
- package/dist/tests/test-user-journey.js +8 -4
- package/dist/tests/test-utils.js +13 -0
- package/dist/tests/test-wizard.js +2 -2
- package/package.json +11 -9
- package/registry/agent-guardrails.md +62 -54
- package/registry/rules/agent-success-criteria.md +52 -0
- package/registry/rules/agent-testing-guidelines.md +7 -7
- package/registry/rules/communication.md +121 -121
- package/registry/rules/continuous-learning.md +54 -54
- package/registry/rules/ephemeral-execution.md +39 -24
- package/registry/rules/hitl-ppe-record-analysis.md +302 -302
- package/registry/rules/local-development.md +7 -5
- package/registry/rules/software-development-lifecycle.md +104 -104
- package/registry/rules/successful-debugging-patterns.md +29 -16
- package/registry/rules/telemetry.md +67 -0
- package/registry/scripts/code-quality-check.sh +559 -559
- package/registry/scripts/detect-tautological-tests.sh +38 -38
- package/registry/scripts/evaluate-code-quality.ts +1 -1
- package/registry/scripts/prep-issue.sh +16 -3
- package/registry/scripts/validate-openapi-limits.ts +366 -365
- package/registry/scripts/validate-test-coverage.ts +280 -280
- package/registry/scripts/verify-pr-comments.sh +70 -70
- package/registry/scripts/verify-test-coverage.ts +1 -1
- package/registry/templates/bootstrap/ARCHITECTURE-TEMPLATE.md +53 -53
- package/registry/templates/evidence/Implementation-BugEvidence.md +85 -85
- package/registry/templates/evidence/Implementation-FeatureEvidence.md +120 -120
- package/registry/templates/marketing/HBR-ARTICLE-TEMPLATE.md +66 -0
- package/registry/workflows/bootstrap/create-architecture.md +2 -2
- package/registry/workflows/bootstrap/evaluate-code-quality.md +10 -4
- package/registry/workflows/bootstrap/verify-test-coverage.md +9 -3
- package/registry/workflows/customer-development/insight-analysis.md +156 -156
- package/registry/workflows/customer-development/interview-preparation.md +421 -421
- package/registry/workflows/customer-development/strategic-brainstorming.md +146 -146
- package/registry/workflows/customer-development/thank-customers.md +14 -2
- package/registry/workflows/customer-development/weekly-newsletter.md +27 -13
- package/registry/workflows/improve-fraim/contribute.md +32 -0
- package/registry/workflows/improve-fraim/file-issue.md +32 -0
- package/registry/workflows/marketing/hbr-article.md +73 -0
- package/registry/workflows/performance/analyze-performance.md +10 -4
- package/registry/workflows/product-building/design.md +3 -2
- package/registry/workflows/product-building/implement.md +10 -5
- package/registry/workflows/product-building/prep-issue.md +16 -18
- package/registry/workflows/product-building/resolve.md +8 -3
- package/registry/workflows/product-building/retrospect.md +3 -2
- package/registry/workflows/product-building/spec.md +5 -4
- package/registry/workflows/product-building/test.md +3 -2
- package/registry/workflows/quality-assurance/iterative-improvement-cycle.md +562 -562
- package/registry/workflows/replicate/replicate-discovery.md +336 -0
- package/registry/workflows/replicate/replicate-to-issues.md +319 -0
- package/registry/workflows/reviewer/review-implementation-vs-design-spec.md +3 -3
- package/registry/workflows/reviewer/review-implementation-vs-feature-spec.md +669 -669
- package/tsconfig.json +2 -1
- package/.windsurf/rules/windsurf-rules.md +0 -7
- package/.windsurf/workflows/resolve-issue.md +0 -6
- package/.windsurf/workflows/retrospect.md +0 -6
- package/.windsurf/workflows/start-design.md +0 -6
- package/.windsurf/workflows/start-impl.md +0 -6
- package/.windsurf/workflows/start-spec.md +0 -6
- package/.windsurf/workflows/start-tests.md +0 -6
- package/bin/fraim.js +0 -23
- package/registry/scripts/build-scripts-generator.ts +0 -215
- package/registry/scripts/cleanup-branch.ts +0 -284
- package/registry/scripts/fraim-config.ts +0 -63
- package/registry/scripts/generate-engagement-emails.ts +0 -744
- package/registry/scripts/generic-issues-api.ts +0 -150
- package/registry/scripts/newsletter-helpers.ts +0 -874
- package/registry/scripts/openapi-generator.ts +0 -693
- package/registry/scripts/run-thank-you-workflow.ts +0 -122
- package/registry/scripts/send-newsletter-simple.ts +0 -104
- package/registry/scripts/send-thank-you-emails.ts +0 -57
- package/registry/workflows/replicate/re-implementation-strategy.md +0 -226
- package/registry/workflows/replicate/use-case-extraction.md +0 -135
- package/registry/workflows/replicate/visual-analysis.md +0 -154
- package/registry/workflows/replicate/website-discovery-analysis.md +0 -231
- package/sample_package.json +0 -18
- /package/registry/scripts/{replicate/comprehensive-explorer.py → comprehensive-explorer.py} +0 -0
- /package/registry/scripts/{replicate/interactive-explorer.py → interactive-explorer.py} +0 -0
- /package/registry/scripts/{replicate/scrape-site.py → scrape-site.py} +0 -0
|
@@ -1,302 +1,302 @@
|
|
|
1
|
-
# HITL PPE Record Analysis & Debugging Patterns
|
|
2
|
-
|
|
3
|
-
## INTENT
|
|
4
|
-
To provide agents with systematic patterns for analyzing HITL (Human-In-The-Loop) records from PPE database, categorizing issues, and implementing appropriate fixes. This rule ensures consistent analysis and prevents overfitting or missing root causes.
|
|
5
|
-
|
|
6
|
-
## PRINCIPLES
|
|
7
|
-
- **Systematic Analysis**: Always follow a structured approach (query → analyze → categorize → fix)
|
|
8
|
-
- **Categorize Before Fixing**: Distinguish systemic issues from executive preferences before implementing
|
|
9
|
-
- **Leverage Existing Infrastructure**: Check if existing features can handle needs before adding new fields
|
|
10
|
-
- **Minimal Fixes**: Focus on smallest changes that solve the problem
|
|
11
|
-
- **Avoid Overfitting**: Use generic examples, never specific person names or scenarios
|
|
12
|
-
|
|
13
|
-
## CORE WORKFLOW PATTERN
|
|
14
|
-
|
|
15
|
-
### The "Query → Filter → Analyze → Categorize → Fix → Document" Pattern
|
|
16
|
-
|
|
17
|
-
```
|
|
18
|
-
1. QUERY: Connect to PPE database and query HITL records
|
|
19
|
-
2. FILTER: Exclude test/synthetic data (e.g., ChatGPT records)
|
|
20
|
-
3. ANALYZE: Extract reviewer feedback patterns and identify issues
|
|
21
|
-
4. CATEGORIZE: Divide into systemic (BAML/code) vs preferences (configurable)
|
|
22
|
-
5. FIX: Implement minimal fixes for systemic issues
|
|
23
|
-
6. DOCUMENT: Create GitHub issues for preference gaps
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
## RULE 1: Data Collection & Filtering
|
|
27
|
-
|
|
28
|
-
### When Querying HITL Records
|
|
29
|
-
|
|
30
|
-
**Always filter by source:**
|
|
31
|
-
- Exclude test/synthetic records: `Inputs.source != 'chatgpt'`
|
|
32
|
-
- Handle missing source fields: Use `$or` with `$exists: false`
|
|
33
|
-
- Filter for non-empty reviewer feedback: `ReviewerFeeback: { $exists: true, $regex: /\S/ }`
|
|
34
|
-
|
|
35
|
-
**Query Pattern:**
|
|
36
|
-
```typescript
|
|
37
|
-
const query = {
|
|
38
|
-
ReviewerFeeback: {
|
|
39
|
-
$exists: true,
|
|
40
|
-
$regex: /\S/ // At least one non-whitespace character
|
|
41
|
-
},
|
|
42
|
-
$or: [
|
|
43
|
-
{ 'Inputs.source': { $exists: false } },
|
|
44
|
-
{ 'Inputs.source': { $ne: 'chatgpt' } }
|
|
45
|
-
]
|
|
46
|
-
};
|
|
47
|
-
```
|
|
48
|
-
|
|
49
|
-
**Verification:**
|
|
50
|
-
- [ ] Verify record count makes sense (not too many/too few)
|
|
51
|
-
- [ ] Check that filtered results exclude test data
|
|
52
|
-
- [ ] Confirm reviewer feedback exists and is non-empty
|
|
53
|
-
|
|
54
|
-
## RULE 2: Issue Categorization
|
|
55
|
-
|
|
56
|
-
### Before Implementing Fixes, Categorize All Issues
|
|
57
|
-
|
|
58
|
-
**Systemic Issues (BAML/Code Fixes):**
|
|
59
|
-
- ✅ Affect all executives universally
|
|
60
|
-
- ✅ Examples: Fake conference links, date logic errors, missing calendar data extraction, accountability violations
|
|
61
|
-
- **Action**: Fix in BAML prompts or code logic
|
|
62
|
-
|
|
63
|
-
**Executive Preferences (Configurable):**
|
|
64
|
-
- ✅ Differ per executive
|
|
65
|
-
- ✅ Examples: Time preferences (avoid 6-7pm), location defaults (virtual vs in-person), verbosity level
|
|
66
|
-
- **Action**: Add to ExecutivePreferences schema OR use existing infrastructure (focus blocks)
|
|
67
|
-
|
|
68
|
-
**Hybrid Issues:**
|
|
69
|
-
- ✅ Core logic is systemic, but thresholds/preferences differ
|
|
70
|
-
- **Action**: Implement systemic fix with configurable parameters
|
|
71
|
-
|
|
72
|
-
**Categorization Checklist:**
|
|
73
|
-
- [ ] Identify which category each issue belongs to
|
|
74
|
-
- [ ] Verify categorization makes sense (would this affect all executives?)
|
|
75
|
-
- [ ] Check if existing infrastructure can handle preference needs
|
|
76
|
-
- [ ] Only add new fields after confirming no existing solution
|
|
77
|
-
|
|
78
|
-
## RULE 3: Leverage Existing Infrastructure
|
|
79
|
-
|
|
80
|
-
### Before Adding New Preference Fields
|
|
81
|
-
|
|
82
|
-
**Check if focus blocks can handle it:**
|
|
83
|
-
- Time-based preferences (avoided windows, thresholds) → Use HIGH protection focus blocks
|
|
84
|
-
- Example: "Avoid 6-7pm meetings" → Create focus block with HIGH protection for 6-7pm
|
|
85
|
-
|
|
86
|
-
**Check if rules can handle it:**
|
|
87
|
-
- Natural language rules may cover edge cases
|
|
88
|
-
- Example: "Prefer in-person for direct reports" → Could use rule with condition
|
|
89
|
-
|
|
90
|
-
**Only add new fields if:**
|
|
91
|
-
- ❌ No existing infrastructure can handle the use case
|
|
92
|
-
- ✅ The preference is structural (location type, duration defaults)
|
|
93
|
-
- ✅ It needs explicit configuration (not inferable from behavior)
|
|
94
|
-
|
|
95
|
-
**Infrastructure Check:**
|
|
96
|
-
- [ ] Check `ExecutivePreferences` schema for existing fields
|
|
97
|
-
- [ ] Check if `focus_blocks` can handle time-based preferences
|
|
98
|
-
- [ ] Check if `rules` can handle natural language preferences
|
|
99
|
-
- [ ] Only add new fields after confirming no existing solution
|
|
100
|
-
|
|
101
|
-
## RULE 4: BAML Prompt Updates
|
|
102
|
-
|
|
103
|
-
### When Fixing Systemic Issues in BAML
|
|
104
|
-
|
|
105
|
-
**Add explicit instructions:**
|
|
106
|
-
- Don't rely on implicit understanding
|
|
107
|
-
- Use phrases like "NEVER", "CRITICAL", "MUST NOT"
|
|
108
|
-
- Example: "NEVER generate conference links" not "Don't generate conference links"
|
|
109
|
-
|
|
110
|
-
**Use multiple locations:**
|
|
111
|
-
- Add rules in all relevant sections
|
|
112
|
-
- Example: Conference link prohibition → LocationGuidelines, EmailResponseGuidelines, CalendarInviteDetails
|
|
113
|
-
|
|
114
|
-
**Provide examples:**
|
|
115
|
-
- Include both correct ✅ and incorrect ❌ examples
|
|
116
|
-
- Show what NOT to do, not just what to do
|
|
117
|
-
|
|
118
|
-
**List banned phrases explicitly:**
|
|
119
|
-
- When fixing verbosity/accountability, list specific phrases to avoid
|
|
120
|
-
- Example: "DO NOT say: 'I'll note it for the team', 'I can take responsibility', etc."
|
|
121
|
-
|
|
122
|
-
**Avoid overfitting:**
|
|
123
|
-
- ❌ Never use specific person names ("Lindy", "Sarah")
|
|
124
|
-
- ✅ Use generic examples ("a person", "someone coordinating")
|
|
125
|
-
- ✅ Use placeholder variables when possible
|
|
126
|
-
|
|
127
|
-
**BAML Update Checklist:**
|
|
128
|
-
- [ ] No specific person names (generic examples only)
|
|
129
|
-
- [ ] Explicit instructions (NEVER, CRITICAL, MUST NOT)
|
|
130
|
-
- [ ] Multiple relevant sections updated if needed
|
|
131
|
-
- [ ] Banned phrases listed explicitly
|
|
132
|
-
- [ ] Examples provided (correct and incorrect)
|
|
133
|
-
- [ ] Regenerated BAML client (`npm run generate`)
|
|
134
|
-
|
|
135
|
-
## RULE 5: Iterative Refinement
|
|
136
|
-
|
|
137
|
-
### For Complex Rule Updates
|
|
138
|
-
|
|
139
|
-
**Start with core rule:**
|
|
140
|
-
- Add basic rule first
|
|
141
|
-
- Test to see if it works
|
|
142
|
-
|
|
143
|
-
**Test edge cases:**
|
|
144
|
-
- Identify problematic phrases/behaviors
|
|
145
|
-
- Look for variations of the issue
|
|
146
|
-
|
|
147
|
-
**Iterate:**
|
|
148
|
-
- Add specific exceptions and banned phrases
|
|
149
|
-
- Refine based on user feedback
|
|
150
|
-
|
|
151
|
-
**Regenerate and verify:**
|
|
152
|
-
- Always run `npm run generate` after changes
|
|
153
|
-
- Check for lint errors (note pre-existing vs new)
|
|
154
|
-
|
|
155
|
-
**Refinement Checklist:**
|
|
156
|
-
- [ ] Core rule added
|
|
157
|
-
- [ ] Edge cases identified
|
|
158
|
-
- [ ] Specific exceptions added
|
|
159
|
-
- [ ] BAML client regenerated
|
|
160
|
-
- [ ] Lint errors checked
|
|
161
|
-
|
|
162
|
-
## RULE 6: Minimal Fixes Strategy
|
|
163
|
-
|
|
164
|
-
### When Implementing Fixes
|
|
165
|
-
|
|
166
|
-
**Focus on root cause:**
|
|
167
|
-
- Don't fix symptoms, fix the underlying issue
|
|
168
|
-
- Example: Don't add workarounds, fix the prompt logic
|
|
169
|
-
|
|
170
|
-
**Smallest change:**
|
|
171
|
-
- Make the minimal change that solves the problem
|
|
172
|
-
- Avoid over-engineering
|
|
173
|
-
|
|
174
|
-
**Multiple locations if needed:**
|
|
175
|
-
- If issue appears in multiple places, fix all instances
|
|
176
|
-
- Don't partially fix
|
|
177
|
-
|
|
178
|
-
**Don't over-engineer:**
|
|
179
|
-
- Avoid adding complex logic when simple rule updates work
|
|
180
|
-
- Example: Adding validation code when prompt instruction works
|
|
181
|
-
|
|
182
|
-
**Minimal Fix Checklist:**
|
|
183
|
-
- [ ] Root cause identified (not just symptom)
|
|
184
|
-
- [ ] Smallest possible change made
|
|
185
|
-
- [ ] All instances fixed if issue appears multiple times
|
|
186
|
-
- [ ] No unnecessary complexity added
|
|
187
|
-
|
|
188
|
-
## RULE 7: Documentation
|
|
189
|
-
|
|
190
|
-
### After Analysis
|
|
191
|
-
|
|
192
|
-
**Create analysis document:**
|
|
193
|
-
- Document findings, patterns, and recommendations
|
|
194
|
-
- Include specific examples from records
|
|
195
|
-
- Link to source data
|
|
196
|
-
|
|
197
|
-
**Create GitHub issues:**
|
|
198
|
-
- For gaps that can't be fixed immediately (preferences)
|
|
199
|
-
- Include clear recommendations and implementation points
|
|
200
|
-
- Label appropriately (enhancement, area:preferences, etc.)
|
|
201
|
-
|
|
202
|
-
**Link to analysis:**
|
|
203
|
-
- Reference source documents in issues and code comments
|
|
204
|
-
- Include record IDs for traceability
|
|
205
|
-
|
|
206
|
-
**Record patterns:**
|
|
207
|
-
- Document patterns for future reference
|
|
208
|
-
- Note what worked and what didn't
|
|
209
|
-
|
|
210
|
-
**Documentation Checklist:**
|
|
211
|
-
- [ ] Analysis document created with findings
|
|
212
|
-
- [ ] GitHub issues created for preference gaps
|
|
213
|
-
- [ ] Source documents linked
|
|
214
|
-
- [ ] Patterns documented for future reference
|
|
215
|
-
|
|
216
|
-
## RULE 8: Testing & Validation
|
|
217
|
-
|
|
218
|
-
### Before Considering Fixes Complete
|
|
219
|
-
|
|
220
|
-
**Regenerate BAML client:**
|
|
221
|
-
- Always run `npm run generate` after BAML changes
|
|
222
|
-
- Verify generation succeeds
|
|
223
|
-
|
|
224
|
-
**Check lint errors:**
|
|
225
|
-
- Verify no new errors introduced
|
|
226
|
-
- Note any pre-existing errors (don't fix unrelated issues)
|
|
227
|
-
|
|
228
|
-
**Verify changes:**
|
|
229
|
-
- Confirm changes are in correct locations
|
|
230
|
-
- Check that all relevant sections updated
|
|
231
|
-
|
|
232
|
-
**Don't commit prematurely:**
|
|
233
|
-
- User may request multiple iterations
|
|
234
|
-
- Wait for explicit commit instruction
|
|
235
|
-
|
|
236
|
-
**Testing Checklist:**
|
|
237
|
-
- [ ] BAML client regenerated (`npm run generate`)
|
|
238
|
-
- [ ] Lint errors checked (no new errors)
|
|
239
|
-
- [ ] Changes verified in correct locations
|
|
240
|
-
- [ ] Not committed (wait for user instruction)
|
|
241
|
-
|
|
242
|
-
## COMMON PATTERNS
|
|
243
|
-
|
|
244
|
-
### Pattern 1: Time Window Preferences
|
|
245
|
-
**Issue**: "Executive prefers not to have 6-7pm meetings"
|
|
246
|
-
**Solution**: Use focus blocks with HIGH protection (don't add new field)
|
|
247
|
-
**Implementation**: Create focus block for 6-7pm with `protection_level: HIGH`
|
|
248
|
-
|
|
249
|
-
### Pattern 2: Location Defaults
|
|
250
|
-
**Issue**: "Location incorrectly defaulted to virtual when should be in-person"
|
|
251
|
-
**Solution**: Add `default_meeting_location_type` to ExecutivePreferences
|
|
252
|
-
**Implementation**: New field in schema, use in BAML when location not specified
|
|
253
|
-
|
|
254
|
-
### Pattern 3: Accountability Violations
|
|
255
|
-
**Issue**: "Ashley took over coordination when someone else was coordinating"
|
|
256
|
-
**Solution**: Add explicit rules to respect existing accountability
|
|
257
|
-
**Implementation**: Update AccountabilityRules template, add banned phrases list
|
|
258
|
-
|
|
259
|
-
### Pattern 4: Date Interpretation Errors
|
|
260
|
-
**Issue**: "This week included past dates"
|
|
261
|
-
**Solution**: Add validation rules to never include past dates
|
|
262
|
-
**Implementation**: Update date interpretation rules in calendar_intent.baml
|
|
263
|
-
|
|
264
|
-
## ANTI-PATTERNS TO AVOID
|
|
265
|
-
|
|
266
|
-
1. **❌ Adding new fields when focus blocks work**: Use existing infrastructure first
|
|
267
|
-
2. **❌ Using specific names in prompts**: Always use generic examples
|
|
268
|
-
3. **❌ Fixing symptoms not root causes**: Dig deeper to find underlying issue
|
|
269
|
-
4. **❌ Over-engineering solutions**: Simple prompt updates often work better than complex logic
|
|
270
|
-
5. **❌ Skipping categorization**: Always categorize before fixing
|
|
271
|
-
|
|
272
|
-
## RELATED RULES
|
|
273
|
-
|
|
274
|
-
-
|
|
275
|
-
-
|
|
276
|
-
-
|
|
277
|
-
|
|
278
|
-
## EXAMPLES
|
|
279
|
-
|
|
280
|
-
### Good: Systematic Analysis
|
|
281
|
-
```
|
|
282
|
-
1. Query PPE database with proper filters
|
|
283
|
-
2. Identify 18 records with reviewer feedback
|
|
284
|
-
3. Categorize: 7 systemic, 3 preferences
|
|
285
|
-
4. Fix systemic issues in BAML
|
|
286
|
-
5. Create GitHub issue for preferences
|
|
287
|
-
6. Document in retrospective
|
|
288
|
-
```
|
|
289
|
-
|
|
290
|
-
### Bad: Ad-hoc Fixes
|
|
291
|
-
```
|
|
292
|
-
1. See one issue
|
|
293
|
-
2. Fix it immediately
|
|
294
|
-
3. Don't check for patterns
|
|
295
|
-
4. Don't categorize
|
|
296
|
-
5. Don't document
|
|
297
|
-
```
|
|
298
|
-
|
|
299
|
-
---
|
|
300
|
-
|
|
301
|
-
**Source**: Derived from retrospective `retrospectives/ppe-hitl-feedback-analysis-debugging-patterns.mdc`
|
|
302
|
-
**Last Updated**: 2025-11-05
|
|
1
|
+
# HITL PPE Record Analysis & Debugging Patterns
|
|
2
|
+
|
|
3
|
+
## INTENT
|
|
4
|
+
To provide agents with systematic patterns for analyzing HITL (Human-In-The-Loop) records from PPE database, categorizing issues, and implementing appropriate fixes. This rule ensures consistent analysis and prevents overfitting or missing root causes.
|
|
5
|
+
|
|
6
|
+
## PRINCIPLES
|
|
7
|
+
- **Systematic Analysis**: Always follow a structured approach (query → analyze → categorize → fix)
|
|
8
|
+
- **Categorize Before Fixing**: Distinguish systemic issues from executive preferences before implementing
|
|
9
|
+
- **Leverage Existing Infrastructure**: Check if existing features can handle needs before adding new fields
|
|
10
|
+
- **Minimal Fixes**: Focus on smallest changes that solve the problem
|
|
11
|
+
- **Avoid Overfitting**: Use generic examples, never specific person names or scenarios
|
|
12
|
+
|
|
13
|
+
## CORE WORKFLOW PATTERN
|
|
14
|
+
|
|
15
|
+
### The "Query → Filter → Analyze → Categorize → Fix → Document" Pattern
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
1. QUERY: Connect to PPE database and query HITL records
|
|
19
|
+
2. FILTER: Exclude test/synthetic data (e.g., ChatGPT records)
|
|
20
|
+
3. ANALYZE: Extract reviewer feedback patterns and identify issues
|
|
21
|
+
4. CATEGORIZE: Divide into systemic (BAML/code) vs preferences (configurable)
|
|
22
|
+
5. FIX: Implement minimal fixes for systemic issues
|
|
23
|
+
6. DOCUMENT: Create GitHub issues for preference gaps
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## RULE 1: Data Collection & Filtering
|
|
27
|
+
|
|
28
|
+
### When Querying HITL Records
|
|
29
|
+
|
|
30
|
+
**Always filter by source:**
|
|
31
|
+
- Exclude test/synthetic records: `Inputs.source != 'chatgpt'`
|
|
32
|
+
- Handle missing source fields: Use `$or` with `$exists: false`
|
|
33
|
+
- Filter for non-empty reviewer feedback: `ReviewerFeeback: { $exists: true, $regex: /\S/ }`
|
|
34
|
+
|
|
35
|
+
**Query Pattern:**
|
|
36
|
+
```typescript
|
|
37
|
+
const query = {
|
|
38
|
+
ReviewerFeeback: {
|
|
39
|
+
$exists: true,
|
|
40
|
+
$regex: /\S/ // At least one non-whitespace character
|
|
41
|
+
},
|
|
42
|
+
$or: [
|
|
43
|
+
{ 'Inputs.source': { $exists: false } },
|
|
44
|
+
{ 'Inputs.source': { $ne: 'chatgpt' } }
|
|
45
|
+
]
|
|
46
|
+
};
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
**Verification:**
|
|
50
|
+
- [ ] Verify record count makes sense (not too many/too few)
|
|
51
|
+
- [ ] Check that filtered results exclude test data
|
|
52
|
+
- [ ] Confirm reviewer feedback exists and is non-empty
|
|
53
|
+
|
|
54
|
+
## RULE 2: Issue Categorization
|
|
55
|
+
|
|
56
|
+
### Before Implementing Fixes, Categorize All Issues
|
|
57
|
+
|
|
58
|
+
**Systemic Issues (BAML/Code Fixes):**
|
|
59
|
+
- ✅ Affect all executives universally
|
|
60
|
+
- ✅ Examples: Fake conference links, date logic errors, missing calendar data extraction, accountability violations
|
|
61
|
+
- **Action**: Fix in BAML prompts or code logic
|
|
62
|
+
|
|
63
|
+
**Executive Preferences (Configurable):**
|
|
64
|
+
- ✅ Differ per executive
|
|
65
|
+
- ✅ Examples: Time preferences (avoid 6-7pm), location defaults (virtual vs in-person), verbosity level
|
|
66
|
+
- **Action**: Add to ExecutivePreferences schema OR use existing infrastructure (focus blocks)
|
|
67
|
+
|
|
68
|
+
**Hybrid Issues:**
|
|
69
|
+
- ✅ Core logic is systemic, but thresholds/preferences differ
|
|
70
|
+
- **Action**: Implement systemic fix with configurable parameters
|
|
71
|
+
|
|
72
|
+
**Categorization Checklist:**
|
|
73
|
+
- [ ] Identify which category each issue belongs to
|
|
74
|
+
- [ ] Verify categorization makes sense (would this affect all executives?)
|
|
75
|
+
- [ ] Check if existing infrastructure can handle preference needs
|
|
76
|
+
- [ ] Only add new fields after confirming no existing solution
|
|
77
|
+
|
|
78
|
+
## RULE 3: Leverage Existing Infrastructure
|
|
79
|
+
|
|
80
|
+
### Before Adding New Preference Fields
|
|
81
|
+
|
|
82
|
+
**Check if focus blocks can handle it:**
|
|
83
|
+
- Time-based preferences (avoided windows, thresholds) → Use HIGH protection focus blocks
|
|
84
|
+
- Example: "Avoid 6-7pm meetings" → Create focus block with HIGH protection for 6-7pm
|
|
85
|
+
|
|
86
|
+
**Check if rules can handle it:**
|
|
87
|
+
- Natural language rules may cover edge cases
|
|
88
|
+
- Example: "Prefer in-person for direct reports" → Could use rule with condition
|
|
89
|
+
|
|
90
|
+
**Only add new fields if:**
|
|
91
|
+
- ❌ No existing infrastructure can handle the use case
|
|
92
|
+
- ✅ The preference is structural (location type, duration defaults)
|
|
93
|
+
- ✅ It needs explicit configuration (not inferable from behavior)
|
|
94
|
+
|
|
95
|
+
**Infrastructure Check:**
|
|
96
|
+
- [ ] Check `ExecutivePreferences` schema for existing fields
|
|
97
|
+
- [ ] Check if `focus_blocks` can handle time-based preferences
|
|
98
|
+
- [ ] Check if `rules` can handle natural language preferences
|
|
99
|
+
- [ ] Only add new fields after confirming no existing solution
|
|
100
|
+
|
|
101
|
+
## RULE 4: BAML Prompt Updates
|
|
102
|
+
|
|
103
|
+
### When Fixing Systemic Issues in BAML
|
|
104
|
+
|
|
105
|
+
**Add explicit instructions:**
|
|
106
|
+
- Don't rely on implicit understanding
|
|
107
|
+
- Use phrases like "NEVER", "CRITICAL", "MUST NOT"
|
|
108
|
+
- Example: "NEVER generate conference links" not "Don't generate conference links"
|
|
109
|
+
|
|
110
|
+
**Use multiple locations:**
|
|
111
|
+
- Add rules in all relevant sections
|
|
112
|
+
- Example: Conference link prohibition → LocationGuidelines, EmailResponseGuidelines, CalendarInviteDetails
|
|
113
|
+
|
|
114
|
+
**Provide examples:**
|
|
115
|
+
- Include both correct ✅ and incorrect ❌ examples
|
|
116
|
+
- Show what NOT to do, not just what to do
|
|
117
|
+
|
|
118
|
+
**List banned phrases explicitly:**
|
|
119
|
+
- When fixing verbosity/accountability, list specific phrases to avoid
|
|
120
|
+
- Example: "DO NOT say: 'I'll note it for the team', 'I can take responsibility', etc."
|
|
121
|
+
|
|
122
|
+
**Avoid overfitting:**
|
|
123
|
+
- ❌ Never use specific person names ("Lindy", "Sarah")
|
|
124
|
+
- ✅ Use generic examples ("a person", "someone coordinating")
|
|
125
|
+
- ✅ Use placeholder variables when possible
|
|
126
|
+
|
|
127
|
+
**BAML Update Checklist:**
|
|
128
|
+
- [ ] No specific person names (generic examples only)
|
|
129
|
+
- [ ] Explicit instructions (NEVER, CRITICAL, MUST NOT)
|
|
130
|
+
- [ ] Multiple relevant sections updated if needed
|
|
131
|
+
- [ ] Banned phrases listed explicitly
|
|
132
|
+
- [ ] Examples provided (correct and incorrect)
|
|
133
|
+
- [ ] Regenerated BAML client (`npm run generate`)
|
|
134
|
+
|
|
135
|
+
## RULE 5: Iterative Refinement
|
|
136
|
+
|
|
137
|
+
### For Complex Rule Updates
|
|
138
|
+
|
|
139
|
+
**Start with core rule:**
|
|
140
|
+
- Add basic rule first
|
|
141
|
+
- Test to see if it works
|
|
142
|
+
|
|
143
|
+
**Test edge cases:**
|
|
144
|
+
- Identify problematic phrases/behaviors
|
|
145
|
+
- Look for variations of the issue
|
|
146
|
+
|
|
147
|
+
**Iterate:**
|
|
148
|
+
- Add specific exceptions and banned phrases
|
|
149
|
+
- Refine based on user feedback
|
|
150
|
+
|
|
151
|
+
**Regenerate and verify:**
|
|
152
|
+
- Always run `npm run generate` after changes
|
|
153
|
+
- Check for lint errors (note pre-existing vs new)
|
|
154
|
+
|
|
155
|
+
**Refinement Checklist:**
|
|
156
|
+
- [ ] Core rule added
|
|
157
|
+
- [ ] Edge cases identified
|
|
158
|
+
- [ ] Specific exceptions added
|
|
159
|
+
- [ ] BAML client regenerated
|
|
160
|
+
- [ ] Lint errors checked
|
|
161
|
+
|
|
162
|
+
## RULE 6: Minimal Fixes Strategy
|
|
163
|
+
|
|
164
|
+
### When Implementing Fixes
|
|
165
|
+
|
|
166
|
+
**Focus on root cause:**
|
|
167
|
+
- Don't fix symptoms, fix the underlying issue
|
|
168
|
+
- Example: Don't add workarounds, fix the prompt logic
|
|
169
|
+
|
|
170
|
+
**Smallest change:**
|
|
171
|
+
- Make the minimal change that solves the problem
|
|
172
|
+
- Avoid over-engineering
|
|
173
|
+
|
|
174
|
+
**Multiple locations if needed:**
|
|
175
|
+
- If issue appears in multiple places, fix all instances
|
|
176
|
+
- Don't partially fix
|
|
177
|
+
|
|
178
|
+
**Don't over-engineer:**
|
|
179
|
+
- Avoid adding complex logic when simple rule updates work
|
|
180
|
+
- Example: Adding validation code when prompt instruction works
|
|
181
|
+
|
|
182
|
+
**Minimal Fix Checklist:**
|
|
183
|
+
- [ ] Root cause identified (not just symptom)
|
|
184
|
+
- [ ] Smallest possible change made
|
|
185
|
+
- [ ] All instances fixed if issue appears multiple times
|
|
186
|
+
- [ ] No unnecessary complexity added
|
|
187
|
+
|
|
188
|
+
## RULE 7: Documentation
|
|
189
|
+
|
|
190
|
+
### After Analysis
|
|
191
|
+
|
|
192
|
+
**Create analysis document:**
|
|
193
|
+
- Document findings, patterns, and recommendations
|
|
194
|
+
- Include specific examples from records
|
|
195
|
+
- Link to source data
|
|
196
|
+
|
|
197
|
+
**Create GitHub issues:**
|
|
198
|
+
- For gaps that can't be fixed immediately (preferences)
|
|
199
|
+
- Include clear recommendations and implementation points
|
|
200
|
+
- Label appropriately (enhancement, area:preferences, etc.)
|
|
201
|
+
|
|
202
|
+
**Link to analysis:**
|
|
203
|
+
- Reference source documents in issues and code comments
|
|
204
|
+
- Include record IDs for traceability
|
|
205
|
+
|
|
206
|
+
**Record patterns:**
|
|
207
|
+
- Document patterns for future reference
|
|
208
|
+
- Note what worked and what didn't
|
|
209
|
+
|
|
210
|
+
**Documentation Checklist:**
|
|
211
|
+
- [ ] Analysis document created with findings
|
|
212
|
+
- [ ] GitHub issues created for preference gaps
|
|
213
|
+
- [ ] Source documents linked
|
|
214
|
+
- [ ] Patterns documented for future reference
|
|
215
|
+
|
|
216
|
+
## RULE 8: Testing & Validation
|
|
217
|
+
|
|
218
|
+
### Before Considering Fixes Complete
|
|
219
|
+
|
|
220
|
+
**Regenerate BAML client:**
|
|
221
|
+
- Always run `npm run generate` after BAML changes
|
|
222
|
+
- Verify generation succeeds
|
|
223
|
+
|
|
224
|
+
**Check lint errors:**
|
|
225
|
+
- Verify no new errors introduced
|
|
226
|
+
- Note any pre-existing errors (don't fix unrelated issues)
|
|
227
|
+
|
|
228
|
+
**Verify changes:**
|
|
229
|
+
- Confirm changes are in correct locations
|
|
230
|
+
- Check that all relevant sections updated
|
|
231
|
+
|
|
232
|
+
**Don't commit prematurely:**
|
|
233
|
+
- User may request multiple iterations
|
|
234
|
+
- Wait for explicit commit instruction
|
|
235
|
+
|
|
236
|
+
**Testing Checklist:**
|
|
237
|
+
- [ ] BAML client regenerated (`npm run generate`)
|
|
238
|
+
- [ ] Lint errors checked (no new errors)
|
|
239
|
+
- [ ] Changes verified in correct locations
|
|
240
|
+
- [ ] Not committed (wait for user instruction)
|
|
241
|
+
|
|
242
|
+
## COMMON PATTERNS
|
|
243
|
+
|
|
244
|
+
### Pattern 1: Time Window Preferences
|
|
245
|
+
**Issue**: "Executive prefers not to have 6-7pm meetings"
|
|
246
|
+
**Solution**: Use focus blocks with HIGH protection (don't add new field)
|
|
247
|
+
**Implementation**: Create focus block for 6-7pm with `protection_level: HIGH`
|
|
248
|
+
|
|
249
|
+
### Pattern 2: Location Defaults
|
|
250
|
+
**Issue**: "Location incorrectly defaulted to virtual when should be in-person"
|
|
251
|
+
**Solution**: Add `default_meeting_location_type` to ExecutivePreferences
|
|
252
|
+
**Implementation**: New field in schema, use in BAML when location not specified
|
|
253
|
+
|
|
254
|
+
### Pattern 3: Accountability Violations
|
|
255
|
+
**Issue**: "Ashley took over coordination when someone else was coordinating"
|
|
256
|
+
**Solution**: Add explicit rules to respect existing accountability
|
|
257
|
+
**Implementation**: Update AccountabilityRules template, add banned phrases list
|
|
258
|
+
|
|
259
|
+
### Pattern 4: Date Interpretation Errors
|
|
260
|
+
**Issue**: "This week included past dates"
|
|
261
|
+
**Solution**: Add validation rules to never include past dates
|
|
262
|
+
**Implementation**: Update date interpretation rules in calendar_intent.baml
|
|
263
|
+
|
|
264
|
+
## ANTI-PATTERNS TO AVOID
|
|
265
|
+
|
|
266
|
+
1. **❌ Adding new fields when focus blocks work**: Use existing infrastructure first
|
|
267
|
+
2. **❌ Using specific names in prompts**: Always use generic examples
|
|
268
|
+
3. **❌ Fixing symptoms not root causes**: Dig deeper to find underlying issue
|
|
269
|
+
4. **❌ Over-engineering solutions**: Simple prompt updates often work better than complex logic
|
|
270
|
+
5. **❌ Skipping categorization**: Always categorize before fixing
|
|
271
|
+
|
|
272
|
+
## RELATED RULES
|
|
273
|
+
|
|
274
|
+
- `get_fraim_file({ path: "rules/successful-debugging-patterns.md" })` - General debugging patterns
|
|
275
|
+
- `get_fraim_file({ path: "rules/code-quality-and-debugging-patterns.md" })` - Code quality standards
|
|
276
|
+
- `get_fraim_file({ path: "rules/ashley-architecture.md" })` - BAML architecture guidelines
|
|
277
|
+
|
|
278
|
+
## EXAMPLES
|
|
279
|
+
|
|
280
|
+
### Good: Systematic Analysis
|
|
281
|
+
```
|
|
282
|
+
1. Query PPE database with proper filters
|
|
283
|
+
2. Identify 18 records with reviewer feedback
|
|
284
|
+
3. Categorize: 7 systemic, 3 preferences
|
|
285
|
+
4. Fix systemic issues in BAML
|
|
286
|
+
5. Create GitHub issue for preferences
|
|
287
|
+
6. Document in retrospective
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
### Bad: Ad-hoc Fixes
|
|
291
|
+
```
|
|
292
|
+
1. See one issue
|
|
293
|
+
2. Fix it immediately
|
|
294
|
+
3. Don't check for patterns
|
|
295
|
+
4. Don't categorize
|
|
296
|
+
5. Don't document
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
---
|
|
300
|
+
|
|
301
|
+
**Source**: Derived from retrospective `retrospectives/ppe-hitl-feedback-analysis-debugging-patterns.mdc`
|
|
302
|
+
**Last Updated**: 2025-11-05
|
|
@@ -19,7 +19,7 @@ Enable safe parallel development through strict workspace separation, preventing
|
|
|
19
19
|
All your work **MUST** be in your folder.
|
|
20
20
|
- Coordinate with other agents through GitHub issues and PRs
|
|
21
21
|
- Each agent works independently in their own folder to enable true parallel development
|
|
22
|
-
- Run every command using `npx tsx
|
|
22
|
+
- Run every command using `npx tsx ~/.fraim/scripts/exec-with-timeout.ts --timeout 30 -- <your command>` (Script is pre-synced)
|
|
23
23
|
|
|
24
24
|
## CRITICAL RULE: Absolute Workspace Separation
|
|
25
25
|
|
|
@@ -70,13 +70,15 @@ ALL commands must use the timeout wrapper:
|
|
|
70
70
|
|
|
71
71
|
```bash
|
|
72
72
|
# Standard timeout (30 seconds)
|
|
73
|
-
npx tsx
|
|
73
|
+
npx tsx ~/.fraim/scripts/exec-with-timeout.ts --timeout 30 -- npm install
|
|
74
74
|
|
|
75
75
|
# Extended timeout for long operations
|
|
76
|
-
npx tsx
|
|
76
|
+
npx tsx ~/.fraim/scripts/exec-with-timeout.ts --timeout 300 -- npm run build
|
|
77
77
|
|
|
78
78
|
# Very long operations (tests, servers)
|
|
79
|
-
npx tsx
|
|
79
|
+
npx tsx ~/.fraim/scripts/exec-with-timeout.ts --timeout 1800 -- npm run test-all
|
|
80
|
+
|
|
81
|
+
# On Windows, use %USERPROFILE% instead of ~
|
|
80
82
|
```
|
|
81
83
|
|
|
82
84
|
### Timeout Guidelines
|
|
@@ -193,7 +195,7 @@ if (updateCall.request.resourceId !== expectedId) throw new Error('Wrong resourc
|
|
|
193
195
|
✅ File Operation: edit_file("src/api-service.ts")
|
|
194
196
|
✅ Branch Check: feature/84-fix-api-sync
|
|
195
197
|
✅ Path Verification: Relative path, no "../" patterns
|
|
196
|
-
✅ Command: npx tsx
|
|
198
|
+
✅ Command: npx tsx scripts/exec-with-timeout.ts --timeout 30 -- npm test
|
|
197
199
|
```
|
|
198
200
|
|
|
199
201
|
### Bad: Workspace Violations
|