@soleri/core 9.10.0 → 9.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain/intelligence.d.ts.map +1 -1
- package/dist/brain/intelligence.js +4 -0
- package/dist/brain/intelligence.js.map +1 -1
- package/dist/brain/types.d.ts +1 -1
- package/dist/brain/types.d.ts.map +1 -1
- package/dist/planning/plan-lifecycle.d.ts.map +1 -1
- package/dist/planning/plan-lifecycle.js +4 -2
- package/dist/planning/plan-lifecycle.js.map +1 -1
- package/dist/planning/planner-types.d.ts +1 -1
- package/dist/planning/planner-types.d.ts.map +1 -1
- package/dist/runtime/orchestrate-ops.d.ts.map +1 -1
- package/dist/runtime/orchestrate-ops.js +6 -3
- package/dist/runtime/orchestrate-ops.js.map +1 -1
- package/dist/runtime/quality-signals.d.ts +6 -1
- package/dist/runtime/quality-signals.d.ts.map +1 -1
- package/dist/runtime/quality-signals.js +41 -5
- package/dist/runtime/quality-signals.js.map +1 -1
- package/package.json +1 -1
- package/src/brain/intelligence.ts +4 -0
- package/src/brain/types.ts +6 -1
- package/src/planning/plan-lifecycle.ts +5 -2
- package/src/planning/planner-types.ts +1 -1
- package/src/planning/planner.test.ts +71 -0
- package/src/runtime/orchestrate-ops.test.ts +51 -2
- package/src/runtime/orchestrate-ops.ts +12 -3
- package/src/runtime/quality-signals.test.ts +182 -8
- package/src/runtime/quality-signals.ts +44 -5
|
@@ -24,7 +24,7 @@ export interface QualityAnalysis {
|
|
|
24
24
|
/**
|
|
25
25
|
* Analyze an evidence report for quality signals.
|
|
26
26
|
*
|
|
27
|
-
* - fixIterations
|
|
27
|
+
* - fixIterations >= 2 → anti-pattern (rework)
|
|
28
28
|
* - fixIterations === 0 + verdict DONE → clean (first-pass success)
|
|
29
29
|
* - unplannedChanges → scope-creep signals
|
|
30
30
|
*/
|
|
@@ -39,4 +39,9 @@ export declare function captureQualitySignals(analysis: QualityAnalysis, vault:
|
|
|
39
39
|
skipped: number;
|
|
40
40
|
feedback: number;
|
|
41
41
|
};
|
|
42
|
+
/**
|
|
43
|
+
* Build a human-readable fix-trail summary from an evidence report.
|
|
44
|
+
* Returns `undefined` when no tasks had rework iterations.
|
|
45
|
+
*/
|
|
46
|
+
export declare function buildFixTrailSummary(report: EvidenceReport): string | undefined;
|
|
42
47
|
//# sourceMappingURL=quality-signals.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"quality-signals.d.ts","sourceRoot":"","sources":["../../src/runtime/quality-signals.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAO/C,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,cAAc,GAAG,OAAO,GAAG,aAAa,CAAC;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,aAAa,EAAE,CAAC;IAC9B,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,UAAU,EAAE,aAAa,EAAE,CAAC;CAC7B;
|
|
1
|
+
{"version":3,"file":"quality-signals.d.ts","sourceRoot":"","sources":["../../src/runtime/quality-signals.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAO/C,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,cAAc,GAAG,OAAO,GAAG,aAAa,CAAC;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,aAAa,EAAE,CAAC;IAC9B,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,UAAU,EAAE,aAAa,EAAE,CAAC;CAC7B;AAiBD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,cAAc,EACtB,KAAK,CAAC,EAAE,IAAI,GAAG,IAAI,GAClB,eAAe,CAuCjB;AAMD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,GACb;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CA+EzD;AAMD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,GAAG,SAAS,CAM/E"}
|
|
@@ -8,15 +8,19 @@
|
|
|
8
8
|
// ---------------------------------------------------------------------------
|
|
9
9
|
// Thresholds
|
|
10
10
|
// ---------------------------------------------------------------------------
|
|
11
|
-
/** Tasks with
|
|
11
|
+
/** Tasks with this many or more fix iterations are flagged as anti-patterns. */
|
|
12
12
|
const REWORK_THRESHOLD = 2;
|
|
13
|
+
/** Brain feedback confidence for clean first-try tasks. */
|
|
14
|
+
const CLEAN_TASK_CONFIDENCE = 0.9;
|
|
15
|
+
/** Brain feedback confidence for high-rework anti-pattern tasks. */
|
|
16
|
+
const REWORK_TASK_CONFIDENCE = 0.7;
|
|
13
17
|
// ---------------------------------------------------------------------------
|
|
14
18
|
// Analysis
|
|
15
19
|
// ---------------------------------------------------------------------------
|
|
16
20
|
/**
|
|
17
21
|
* Analyze an evidence report for quality signals.
|
|
18
22
|
*
|
|
19
|
-
* - fixIterations
|
|
23
|
+
* - fixIterations >= 2 → anti-pattern (rework)
|
|
20
24
|
* - fixIterations === 0 + verdict DONE → clean (first-pass success)
|
|
21
25
|
* - unplannedChanges → scope-creep signals
|
|
22
26
|
*/
|
|
@@ -26,7 +30,7 @@ export function analyzeQualitySignals(report, _plan) {
|
|
|
26
30
|
const scopeCreep = [];
|
|
27
31
|
for (const te of report.taskEvidence) {
|
|
28
32
|
const iterations = te.fixIterations ?? 0;
|
|
29
|
-
if (iterations
|
|
33
|
+
if (iterations >= REWORK_THRESHOLD) {
|
|
30
34
|
antiPatterns.push({
|
|
31
35
|
taskId: te.taskId,
|
|
32
36
|
taskTitle: te.taskTitle,
|
|
@@ -102,7 +106,19 @@ export function captureQualitySignals(analysis, vault, brain, planId) {
|
|
|
102
106
|
// Record negative brain feedback for rework tasks
|
|
103
107
|
for (const ap of analysis.antiPatterns) {
|
|
104
108
|
try {
|
|
105
|
-
brain.recordFeedback(
|
|
109
|
+
brain.recordFeedback({
|
|
110
|
+
query: ap.taskTitle,
|
|
111
|
+
entryId: planId,
|
|
112
|
+
action: 'dismissed',
|
|
113
|
+
confidence: REWORK_TASK_CONFIDENCE,
|
|
114
|
+
source: 'evidence-quality',
|
|
115
|
+
reason: `Task needed ${ap.fixIterations} fix iterations — high rework`,
|
|
116
|
+
context: JSON.stringify({
|
|
117
|
+
taskId: ap.taskId,
|
|
118
|
+
reworkCount: ap.fixIterations,
|
|
119
|
+
verdict: ap.verdict,
|
|
120
|
+
}),
|
|
121
|
+
});
|
|
106
122
|
feedback++;
|
|
107
123
|
}
|
|
108
124
|
catch {
|
|
@@ -112,7 +128,14 @@ export function captureQualitySignals(analysis, vault, brain, planId) {
|
|
|
112
128
|
// Record positive brain feedback for clean tasks
|
|
113
129
|
for (const ct of analysis.cleanTasks) {
|
|
114
130
|
try {
|
|
115
|
-
brain.recordFeedback(
|
|
131
|
+
brain.recordFeedback({
|
|
132
|
+
query: ct.taskTitle,
|
|
133
|
+
entryId: planId,
|
|
134
|
+
action: 'accepted',
|
|
135
|
+
confidence: CLEAN_TASK_CONFIDENCE,
|
|
136
|
+
source: 'evidence-quality',
|
|
137
|
+
reason: 'Clean first-try completion — no rework needed',
|
|
138
|
+
});
|
|
116
139
|
feedback++;
|
|
117
140
|
}
|
|
118
141
|
catch {
|
|
@@ -121,4 +144,17 @@ export function captureQualitySignals(analysis, vault, brain, planId) {
|
|
|
121
144
|
}
|
|
122
145
|
return { captured, skipped, feedback };
|
|
123
146
|
}
|
|
147
|
+
// ---------------------------------------------------------------------------
|
|
148
|
+
// Fix-trail summary for knowledge extraction
|
|
149
|
+
// ---------------------------------------------------------------------------
|
|
150
|
+
/**
|
|
151
|
+
* Build a human-readable fix-trail summary from an evidence report.
|
|
152
|
+
* Returns `undefined` when no tasks had rework iterations.
|
|
153
|
+
*/
|
|
154
|
+
export function buildFixTrailSummary(report) {
|
|
155
|
+
const entries = report.taskEvidence
|
|
156
|
+
.filter((te) => (te.fixIterations ?? 0) > 0)
|
|
157
|
+
.map((te) => `${te.taskTitle}: ${te.fixIterations} fix iterations`);
|
|
158
|
+
return entries.length > 0 ? entries.join('; ') : undefined;
|
|
159
|
+
}
|
|
124
160
|
//# sourceMappingURL=quality-signals.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"quality-signals.js","sourceRoot":"","sources":["../../src/runtime/quality-signals.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AA0BH,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,
|
|
1
|
+
{"version":3,"file":"quality-signals.js","sourceRoot":"","sources":["../../src/runtime/quality-signals.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AA0BH,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,gFAAgF;AAChF,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,2DAA2D;AAC3D,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,oEAAoE;AACpE,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,8EAA8E;AAC9E,WAAW;AACX,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAAsB,EACtB,KAAmB;IAEnB,MAAM,YAAY,GAAoB,EAAE,CAAC;IACzC,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,MAAM,UAAU,GAAoB,EAAE,CAAC;IAEvC,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,EAAE,CAAC,aAAa,IAAI,CAAC,CAAC;QAEzC,IAAI,UAAU,IAAI,gBAAgB,EAAE,CAAC;YACnC,YAAY,CAAC,IAAI,CAAC;gBAChB,MAAM,EAAE,EAAE,CAAC,MAAM;gBACjB,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,IAAI,EAAE,cAAc;gBACpB,aAAa,EAAE,UAAU;gBACzB,OAAO,EAAE,EAAE,CAAC,OAAO;aACpB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,UAAU,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YACrD,UAAU,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,EAAE,CAAC,MAAM;gBACjB,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,IAAI,EAAE,OAAO;gBACb,aAAa,EAAE,CAAC;gBAChB,OAAO,EAAE,EAAE,CAAC,OAAO;aACpB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,uCAAuC;IACvC,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QACzC,UAAU,CAAC,IAAI,CAAC;YACd,MAAM,EAAE,WAAW;YACnB,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI;YACvB,IAAI,EAAE,aAAa;YACnB,aAAa,EAAE,CAAC;YAChB,OAAO,EAAE,EAAE,CAAC,cAAc;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AAClD,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CACnC,QAAyB,EACzB,KAAY,EACZ,KAAY,EACZ,MAAc;IAEd,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,+CAA+C;IAC/C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,oBAAoB,EAAE,CAAC,SAAS,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACzE,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;YAExD,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YAED,MAAM,QAAQ,GAAG,EAAE,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/D,MAAM,KAAK,GAAsB;gBAC/B,EAAE,EAAE,SAAS,MAAM,IAAI,EAAE,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;gBAChD,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,oBAAoB,EAAE,CAAC,SAAS,EAAE;gBACzC,QAAQ;gBACR,WAAW,EACT,SAAS,EAAE,CAAC,SAAS,cAAc,EAAE,CAAC,aAAa,kBAAkB;oBACrE,eAAe,gBAAgB,8BAA8B;oBAC7D,mEAAmE;gBACrE,IAAI,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,eAAe,CAAC;gBAC9C,MAAM,EAAE,OAAO;aAChB,CAAC;YAEF,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACjB,QAAQ,EAAE,CAAC;QACb,CAAC;QAAC,MAAM,CAAC;YACP,iCAAiC;QACnC,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,KAAK,CAAC,cAAc,CAAC;gBACnB,KAAK,EAAE,EAAE,CAAC,SAAS;gBACnB,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,WAAW;gBACnB,UAAU,EAAE,sBAAsB;gBAClC,MAAM,EAAE,kBAAkB;gBAC1B,MAAM,EAAE,eAAe,EAAE,CAAC,aAAa,+BAA+B;gBACtE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;oBACtB,MAAM,EAAE,EAAE,CAAC,MAAM;oBACjB,WAAW,EAAE,EAAE,CAAC,aAAa;oBAC7B,OAAO,EAAE,EAAE,CAAC,OAAO;iBACpB,CAAC;aACH,CAAC,CAAC;YACH,QAAQ,EAAE,CAAC;QACb,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;IAED,iDAAiD;IACjD,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,KAAK,CAAC,cAAc,CAAC;gBACnB,KAAK,EAAE,EAAE,CAAC,SAAS;gBACnB,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,UAAU;gBAClB,UAAU,EAAE,qBAAqB;gBACjC,MAAM,EAAE,kBAAkB;gBAC1B,MAAM,EAAE,+CAA+C;aACxD,CAAC,CAAC;YACH,QAAQ,EAAE,CAAC;QACb,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC;AAED,8EAA8E;AAC9E,6CAA6C;AAC7C,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAsB;IACzD,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY;SAChC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;SAC3C,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,SAAS,KAAK,EAAE,CAAC,aAAa,iBAAiB,CAAC,CAAC;IAEtE,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7D,CAAC"}
|
package/package.json
CHANGED
|
@@ -181,6 +181,10 @@ export class BrainIntelligence {
|
|
|
181
181
|
updates.push('plan_outcome = ?');
|
|
182
182
|
values.push(input.planOutcome);
|
|
183
183
|
}
|
|
184
|
+
if (input.context) {
|
|
185
|
+
updates.push("context = COALESCE(context, '') || ?");
|
|
186
|
+
values.push(' | ' + input.context);
|
|
187
|
+
}
|
|
184
188
|
|
|
185
189
|
values.push(sessionId);
|
|
186
190
|
this.provider.run(`UPDATE brain_sessions SET ${updates.join(', ')} WHERE id = ?`, values);
|
package/src/brain/types.ts
CHANGED
|
@@ -74,7 +74,12 @@ export interface QueryContext {
|
|
|
74
74
|
// ─── Feedback Types ───────────────────────────────────────────────
|
|
75
75
|
|
|
76
76
|
export type FeedbackType = 'accepted' | 'dismissed' | 'modified' | 'failed';
|
|
77
|
-
export type FeedbackSource =
|
|
77
|
+
export type FeedbackSource =
|
|
78
|
+
| 'search'
|
|
79
|
+
| 'recommendation'
|
|
80
|
+
| 'tool-execution'
|
|
81
|
+
| 'explicit'
|
|
82
|
+
| 'evidence-quality';
|
|
78
83
|
|
|
79
84
|
export interface FeedbackInput {
|
|
80
85
|
query: string;
|
|
@@ -338,8 +338,11 @@ export function applyIteration(plan: Plan, changes: IterateChanges): number {
|
|
|
338
338
|
*/
|
|
339
339
|
export function applyTaskStatusUpdate(task: PlanTask, status: TaskStatus): void {
|
|
340
340
|
const now = Date.now();
|
|
341
|
-
// Rework detection: completed/failed → in_progress means a fix iteration
|
|
342
|
-
|
|
341
|
+
// Rework detection: completed/failed → in_progress/pending means a fix iteration
|
|
342
|
+
const isRework =
|
|
343
|
+
(task.status === 'completed' || task.status === 'failed') &&
|
|
344
|
+
(status === 'in_progress' || status === 'pending');
|
|
345
|
+
if (isRework) {
|
|
343
346
|
task.fixIterations = (task.fixIterations ?? 0) + 1;
|
|
344
347
|
task.completedAt = undefined;
|
|
345
348
|
}
|
|
@@ -88,7 +88,7 @@ export interface PlanTask {
|
|
|
88
88
|
deliverables?: TaskDeliverable[];
|
|
89
89
|
/** Verification findings for tasks that modify existing code. Advisory only. */
|
|
90
90
|
verification?: TaskVerification;
|
|
91
|
-
/** Number of
|
|
91
|
+
/** Number of rework cycles. 0 = clean first pass. Incremented when task reverts from completed/failed back to in_progress/pending. */
|
|
92
92
|
fixIterations?: number;
|
|
93
93
|
updatedAt: number;
|
|
94
94
|
}
|
|
@@ -831,4 +831,75 @@ describe('Planner', () => {
|
|
|
831
831
|
expect(archived[0].status).toBe('archived');
|
|
832
832
|
});
|
|
833
833
|
});
|
|
834
|
+
|
|
835
|
+
describe('fixIterations tracking', () => {
|
|
836
|
+
it('increments fixIterations when task goes completed → in_progress', () => {
|
|
837
|
+
const plan = planner.create({
|
|
838
|
+
objective: 'Rework test',
|
|
839
|
+
scope: 'test',
|
|
840
|
+
tasks: [{ title: 'Task A', description: 'A task' }],
|
|
841
|
+
});
|
|
842
|
+
planner.approve(plan.id);
|
|
843
|
+
planner.startExecution(plan.id);
|
|
844
|
+
|
|
845
|
+
planner.updateTask(plan.id, 'task-1', 'in_progress');
|
|
846
|
+
planner.updateTask(plan.id, 'task-1', 'completed');
|
|
847
|
+
planner.updateTask(plan.id, 'task-1', 'in_progress');
|
|
848
|
+
|
|
849
|
+
const task = planner.get(plan.id)!.tasks[0];
|
|
850
|
+
expect(task.fixIterations).toBe(1);
|
|
851
|
+
});
|
|
852
|
+
|
|
853
|
+
it('increments fixIterations when task goes failed → in_progress', () => {
|
|
854
|
+
const plan = planner.create({
|
|
855
|
+
objective: 'Failed rework test',
|
|
856
|
+
scope: 'test',
|
|
857
|
+
tasks: [{ title: 'Task A', description: 'A task' }],
|
|
858
|
+
});
|
|
859
|
+
planner.approve(plan.id);
|
|
860
|
+
planner.startExecution(plan.id);
|
|
861
|
+
|
|
862
|
+
planner.updateTask(plan.id, 'task-1', 'in_progress');
|
|
863
|
+
planner.updateTask(plan.id, 'task-1', 'failed');
|
|
864
|
+
planner.updateTask(plan.id, 'task-1', 'in_progress');
|
|
865
|
+
|
|
866
|
+
const task = planner.get(plan.id)!.tasks[0];
|
|
867
|
+
expect(task.fixIterations).toBe(1);
|
|
868
|
+
});
|
|
869
|
+
|
|
870
|
+
it('does NOT increment on forward transitions', () => {
|
|
871
|
+
const plan = planner.create({
|
|
872
|
+
objective: 'Forward transition test',
|
|
873
|
+
scope: 'test',
|
|
874
|
+
tasks: [{ title: 'Task A', description: 'A task' }],
|
|
875
|
+
});
|
|
876
|
+
planner.approve(plan.id);
|
|
877
|
+
planner.startExecution(plan.id);
|
|
878
|
+
|
|
879
|
+
planner.updateTask(plan.id, 'task-1', 'in_progress');
|
|
880
|
+
planner.updateTask(plan.id, 'task-1', 'completed');
|
|
881
|
+
|
|
882
|
+
const task = planner.get(plan.id)!.tasks[0];
|
|
883
|
+
expect(task.fixIterations ?? 0).toBe(0);
|
|
884
|
+
});
|
|
885
|
+
|
|
886
|
+
it('accumulates across multiple rework cycles', () => {
|
|
887
|
+
const plan = planner.create({
|
|
888
|
+
objective: 'Multi-rework test',
|
|
889
|
+
scope: 'test',
|
|
890
|
+
tasks: [{ title: 'Task A', description: 'A task' }],
|
|
891
|
+
});
|
|
892
|
+
planner.approve(plan.id);
|
|
893
|
+
planner.startExecution(plan.id);
|
|
894
|
+
|
|
895
|
+
planner.updateTask(plan.id, 'task-1', 'in_progress');
|
|
896
|
+
planner.updateTask(plan.id, 'task-1', 'completed');
|
|
897
|
+
planner.updateTask(plan.id, 'task-1', 'in_progress'); // rework 1
|
|
898
|
+
planner.updateTask(plan.id, 'task-1', 'completed');
|
|
899
|
+
planner.updateTask(plan.id, 'task-1', 'in_progress'); // rework 2
|
|
900
|
+
|
|
901
|
+
const task = planner.get(plan.id)!.tasks[0];
|
|
902
|
+
expect(task.fixIterations).toBe(2);
|
|
903
|
+
});
|
|
904
|
+
});
|
|
834
905
|
});
|
|
@@ -363,10 +363,10 @@ describe('createOrchestrateOps', () => {
|
|
|
363
363
|
outcome: 'completed',
|
|
364
364
|
})) as Record<string, unknown>;
|
|
365
365
|
|
|
366
|
-
// Should complete successfully
|
|
366
|
+
// Should complete successfully with evidenceReport: null
|
|
367
367
|
expect(result).toHaveProperty('plan');
|
|
368
368
|
expect(result).toHaveProperty('session');
|
|
369
|
-
expect(result).
|
|
369
|
+
expect(result.evidenceReport).toBeNull();
|
|
370
370
|
});
|
|
371
371
|
|
|
372
372
|
it('adds warning when evidence accuracy is below 50%', async () => {
|
|
@@ -395,6 +395,55 @@ describe('createOrchestrateOps', () => {
|
|
|
395
395
|
const warnings = result.warnings as string[];
|
|
396
396
|
expect(warnings.some((w) => w.includes('Low evidence accuracy (30%)'))).toBe(true);
|
|
397
397
|
});
|
|
398
|
+
|
|
399
|
+
it('runs evidence collection for abandoned plans too', async () => {
|
|
400
|
+
const { collectGitEvidence } = await import('../planning/evidence-collector.js');
|
|
401
|
+
vi.mocked(collectGitEvidence).mockReturnValueOnce({
|
|
402
|
+
planId: 'plan-1',
|
|
403
|
+
planObjective: 'test',
|
|
404
|
+
accuracy: 60,
|
|
405
|
+
evidenceSources: ['git'],
|
|
406
|
+
taskEvidence: [
|
|
407
|
+
{
|
|
408
|
+
taskId: 't1',
|
|
409
|
+
taskTitle: 'Task 1',
|
|
410
|
+
plannedStatus: 'pending',
|
|
411
|
+
matchedFiles: [],
|
|
412
|
+
verdict: 'MISSING',
|
|
413
|
+
},
|
|
414
|
+
],
|
|
415
|
+
unplannedChanges: [],
|
|
416
|
+
missingWork: [],
|
|
417
|
+
verificationGaps: [],
|
|
418
|
+
summary: '0/1 tasks verified by git evidence',
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
const op = findOp(ops, 'orchestrate_complete');
|
|
422
|
+
const result = (await op.handler({
|
|
423
|
+
planId: 'plan-1',
|
|
424
|
+
sessionId: 'session-1',
|
|
425
|
+
outcome: 'abandoned',
|
|
426
|
+
projectPath: '.',
|
|
427
|
+
})) as Record<string, unknown>;
|
|
428
|
+
|
|
429
|
+
expect(collectGitEvidence).toHaveBeenCalled();
|
|
430
|
+
expect(result).toHaveProperty('evidenceReport');
|
|
431
|
+
const report = result.evidenceReport as Record<string, unknown>;
|
|
432
|
+
expect(report.accuracy).toBe(60);
|
|
433
|
+
expect(Array.isArray(report.taskEvidence)).toBe(true);
|
|
434
|
+
});
|
|
435
|
+
|
|
436
|
+
it('returns evidenceReport as null when no plan is provided', async () => {
|
|
437
|
+
const op = findOp(ops, 'orchestrate_complete');
|
|
438
|
+
const result = (await op.handler({
|
|
439
|
+
sessionId: 'session-1',
|
|
440
|
+
outcome: 'completed',
|
|
441
|
+
summary: 'Direct task without a plan',
|
|
442
|
+
})) as Record<string, unknown>;
|
|
443
|
+
|
|
444
|
+
expect(result).toHaveProperty('evidenceReport');
|
|
445
|
+
expect(result.evidenceReport).toBeNull();
|
|
446
|
+
});
|
|
398
447
|
});
|
|
399
448
|
|
|
400
449
|
// ─── orchestrate_status ───────────────────────────────────────
|
|
@@ -43,7 +43,11 @@ import type { ImpactReport } from '../planning/impact-analyzer.js';
|
|
|
43
43
|
import { collectGitEvidence } from '../planning/evidence-collector.js';
|
|
44
44
|
import type { EvidenceReport } from '../planning/evidence-collector.js';
|
|
45
45
|
import { recordPlanFeedback } from './plan-feedback-helper.js';
|
|
46
|
-
import {
|
|
46
|
+
import {
|
|
47
|
+
analyzeQualitySignals,
|
|
48
|
+
captureQualitySignals,
|
|
49
|
+
buildFixTrailSummary,
|
|
50
|
+
} from './quality-signals.js';
|
|
47
51
|
|
|
48
52
|
// ---------------------------------------------------------------------------
|
|
49
53
|
// Intent detection — keyword-based mapping from prompt to intent
|
|
@@ -832,7 +836,7 @@ export function createOrchestrateOps(
|
|
|
832
836
|
|
|
833
837
|
// Evidence-based reconciliation: cross-reference plan tasks against git diff
|
|
834
838
|
let evidenceReport: EvidenceReport | null = null;
|
|
835
|
-
if (planObj
|
|
839
|
+
if (planObj) {
|
|
836
840
|
try {
|
|
837
841
|
evidenceReport = collectGitEvidence(
|
|
838
842
|
planObj,
|
|
@@ -840,6 +844,9 @@ export function createOrchestrateOps(
|
|
|
840
844
|
'main',
|
|
841
845
|
);
|
|
842
846
|
if (evidenceReport.accuracy < 50) {
|
|
847
|
+
console.error(
|
|
848
|
+
`[soleri] Evidence accuracy ${evidenceReport.accuracy}% — significant drift detected between plan and git state`,
|
|
849
|
+
);
|
|
843
850
|
warnings.push(
|
|
844
851
|
`Low evidence accuracy (${evidenceReport.accuracy}%) — plan tasks may not match git changes.`,
|
|
845
852
|
);
|
|
@@ -873,6 +880,7 @@ export function createOrchestrateOps(
|
|
|
873
880
|
}
|
|
874
881
|
|
|
875
882
|
// End brain session — runs regardless of plan existence
|
|
883
|
+
const fixTrail = evidenceReport ? buildFixTrailSummary(evidenceReport) : undefined;
|
|
876
884
|
const session = brainIntelligence.lifecycle({
|
|
877
885
|
action: 'end',
|
|
878
886
|
sessionId,
|
|
@@ -880,6 +888,7 @@ export function createOrchestrateOps(
|
|
|
880
888
|
planOutcome: outcome,
|
|
881
889
|
toolsUsed,
|
|
882
890
|
filesModified,
|
|
891
|
+
...(fixTrail ? { context: `Fix trail: ${fixTrail}` } : {}),
|
|
883
892
|
});
|
|
884
893
|
|
|
885
894
|
// Record brain feedback for vault entries referenced in plan decisions
|
|
@@ -974,7 +983,7 @@ export function createOrchestrateOps(
|
|
|
974
983
|
extraction,
|
|
975
984
|
epilogue: epilogueResult,
|
|
976
985
|
...(impactReport ? { impactAnalysis: impactReport } : {}),
|
|
977
|
-
|
|
986
|
+
evidenceReport,
|
|
978
987
|
...(warnings.length > 0 ? { warnings } : {}),
|
|
979
988
|
};
|
|
980
989
|
},
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
analyzeQualitySignals,
|
|
4
|
+
captureQualitySignals,
|
|
5
|
+
buildFixTrailSummary,
|
|
6
|
+
} from './quality-signals.js';
|
|
3
7
|
import type { EvidenceReport } from '../planning/evidence-collector.js';
|
|
4
8
|
|
|
5
9
|
// ---------------------------------------------------------------------------
|
|
@@ -104,7 +108,7 @@ describe('analyzeQualitySignals', () => {
|
|
|
104
108
|
expect(result.antiPatterns).toHaveLength(0);
|
|
105
109
|
});
|
|
106
110
|
|
|
107
|
-
it('
|
|
111
|
+
it('flags task with fixIterations === 2 (at threshold)', () => {
|
|
108
112
|
const report = makeReport({
|
|
109
113
|
taskEvidence: [
|
|
110
114
|
{
|
|
@@ -120,7 +124,28 @@ describe('analyzeQualitySignals', () => {
|
|
|
120
124
|
|
|
121
125
|
const result = analyzeQualitySignals(report);
|
|
122
126
|
|
|
127
|
+
expect(result.antiPatterns).toHaveLength(1);
|
|
128
|
+
expect(result.antiPatterns[0].fixIterations).toBe(2);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('does not flag task with fixIterations === 1 (below threshold)', () => {
|
|
132
|
+
const report = makeReport({
|
|
133
|
+
taskEvidence: [
|
|
134
|
+
{
|
|
135
|
+
taskId: 't4b',
|
|
136
|
+
taskTitle: 'Single retry task',
|
|
137
|
+
plannedStatus: 'completed',
|
|
138
|
+
matchedFiles: [],
|
|
139
|
+
verdict: 'DONE',
|
|
140
|
+
fixIterations: 1,
|
|
141
|
+
},
|
|
142
|
+
],
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
const result = analyzeQualitySignals(report);
|
|
146
|
+
|
|
123
147
|
expect(result.antiPatterns).toHaveLength(0);
|
|
148
|
+
expect(result.cleanTasks).toHaveLength(0);
|
|
124
149
|
});
|
|
125
150
|
|
|
126
151
|
it('detects scope creep from unplanned changes', () => {
|
|
@@ -199,9 +224,14 @@ describe('captureQualitySignals', () => {
|
|
|
199
224
|
expect(entry.tags).toContain('auto-captured');
|
|
200
225
|
|
|
201
226
|
expect(brain.recordFeedback).toHaveBeenCalledWith(
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
227
|
+
expect.objectContaining({
|
|
228
|
+
query: 'Fix login',
|
|
229
|
+
entryId: 'plan-1',
|
|
230
|
+
action: 'dismissed',
|
|
231
|
+
confidence: 0.7,
|
|
232
|
+
source: 'evidence-quality',
|
|
233
|
+
reason: 'Task needed 3 fix iterations — high rework',
|
|
234
|
+
}),
|
|
205
235
|
);
|
|
206
236
|
|
|
207
237
|
expect(result.captured).toBe(1);
|
|
@@ -226,9 +256,14 @@ describe('captureQualitySignals', () => {
|
|
|
226
256
|
const result = captureQualitySignals(analysis, vault, brain, 'plan-1');
|
|
227
257
|
|
|
228
258
|
expect(brain.recordFeedback).toHaveBeenCalledWith(
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
259
|
+
expect.objectContaining({
|
|
260
|
+
query: 'Add feature',
|
|
261
|
+
entryId: 'plan-1',
|
|
262
|
+
action: 'accepted',
|
|
263
|
+
confidence: 0.9,
|
|
264
|
+
source: 'evidence-quality',
|
|
265
|
+
reason: 'Clean first-try completion — no rework needed',
|
|
266
|
+
}),
|
|
232
267
|
);
|
|
233
268
|
expect(result.feedback).toBe(1);
|
|
234
269
|
expect(result.captured).toBe(0);
|
|
@@ -281,6 +316,83 @@ describe('captureQualitySignals', () => {
|
|
|
281
316
|
expect(entry.severity).toBe('critical');
|
|
282
317
|
});
|
|
283
318
|
|
|
319
|
+
it('records positive feedback with evidence-quality source for clean first-try tasks', () => {
|
|
320
|
+
const analysis = {
|
|
321
|
+
antiPatterns: [],
|
|
322
|
+
cleanTasks: [
|
|
323
|
+
{
|
|
324
|
+
taskId: 'clean-1',
|
|
325
|
+
taskTitle: 'Smooth task',
|
|
326
|
+
kind: 'clean' as const,
|
|
327
|
+
fixIterations: 0,
|
|
328
|
+
verdict: 'DONE',
|
|
329
|
+
},
|
|
330
|
+
],
|
|
331
|
+
scopeCreep: [],
|
|
332
|
+
};
|
|
333
|
+
|
|
334
|
+
captureQualitySignals(analysis, vault, brain, 'plan-99');
|
|
335
|
+
|
|
336
|
+
expect(brain.recordFeedback).toHaveBeenCalledWith(
|
|
337
|
+
expect.objectContaining({
|
|
338
|
+
action: 'accepted',
|
|
339
|
+
confidence: 0.9,
|
|
340
|
+
source: 'evidence-quality',
|
|
341
|
+
entryId: 'plan-99',
|
|
342
|
+
}),
|
|
343
|
+
);
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
it('records negative feedback with evidence-quality source for high-rework tasks', () => {
|
|
347
|
+
const analysis = {
|
|
348
|
+
antiPatterns: [
|
|
349
|
+
{
|
|
350
|
+
taskId: 'rework-1',
|
|
351
|
+
taskTitle: 'Painful task',
|
|
352
|
+
kind: 'anti-pattern' as const,
|
|
353
|
+
fixIterations: 3,
|
|
354
|
+
verdict: 'DONE',
|
|
355
|
+
},
|
|
356
|
+
],
|
|
357
|
+
cleanTasks: [],
|
|
358
|
+
scopeCreep: [],
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
captureQualitySignals(analysis, vault, brain, 'plan-99');
|
|
362
|
+
|
|
363
|
+
expect(brain.recordFeedback).toHaveBeenCalledWith(
|
|
364
|
+
expect.objectContaining({
|
|
365
|
+
action: 'dismissed',
|
|
366
|
+
confidence: 0.7,
|
|
367
|
+
source: 'evidence-quality',
|
|
368
|
+
reason: 'Task needed 3 fix iterations — high rework',
|
|
369
|
+
context: JSON.stringify({ taskId: 'rework-1', reworkCount: 3, verdict: 'DONE' }),
|
|
370
|
+
}),
|
|
371
|
+
);
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
it('does not record evidence-quality feedback for tasks with 1 fix iteration', () => {
|
|
375
|
+
// 1 fix iteration = neither clean (fixIterations !== 0) nor anti-pattern (< 2)
|
|
376
|
+
const analysis = analyzeQualitySignals(
|
|
377
|
+
makeReport({
|
|
378
|
+
taskEvidence: [
|
|
379
|
+
{
|
|
380
|
+
taskId: 't-mid',
|
|
381
|
+
taskTitle: 'Single retry',
|
|
382
|
+
plannedStatus: 'completed',
|
|
383
|
+
matchedFiles: [],
|
|
384
|
+
verdict: 'DONE',
|
|
385
|
+
fixIterations: 1,
|
|
386
|
+
},
|
|
387
|
+
],
|
|
388
|
+
}),
|
|
389
|
+
);
|
|
390
|
+
|
|
391
|
+
captureQualitySignals(analysis, vault, brain, 'plan-99');
|
|
392
|
+
|
|
393
|
+
expect(brain.recordFeedback).not.toHaveBeenCalled();
|
|
394
|
+
});
|
|
395
|
+
|
|
284
396
|
it('handles mixed signals correctly', () => {
|
|
285
397
|
const analysis = {
|
|
286
398
|
antiPatterns: [
|
|
@@ -310,3 +422,65 @@ describe('captureQualitySignals', () => {
|
|
|
310
422
|
expect(result.feedback).toBe(2); // 1 dismissed + 1 accepted
|
|
311
423
|
});
|
|
312
424
|
});
|
|
425
|
+
|
|
426
|
+
// ---------------------------------------------------------------------------
|
|
427
|
+
// buildFixTrailSummary
|
|
428
|
+
// ---------------------------------------------------------------------------
|
|
429
|
+
|
|
430
|
+
describe('buildFixTrailSummary', () => {
|
|
431
|
+
it('returns summary string for tasks with rework iterations', () => {
|
|
432
|
+
const report = makeReport({
|
|
433
|
+
taskEvidence: [
|
|
434
|
+
{
|
|
435
|
+
taskId: 'a',
|
|
436
|
+
taskTitle: 'Task A',
|
|
437
|
+
plannedStatus: 'completed',
|
|
438
|
+
matchedFiles: [],
|
|
439
|
+
verdict: 'DONE',
|
|
440
|
+
fixIterations: 2,
|
|
441
|
+
},
|
|
442
|
+
{
|
|
443
|
+
taskId: 'b',
|
|
444
|
+
taskTitle: 'Task B',
|
|
445
|
+
plannedStatus: 'completed',
|
|
446
|
+
matchedFiles: [],
|
|
447
|
+
verdict: 'DONE',
|
|
448
|
+
fixIterations: 0,
|
|
449
|
+
},
|
|
450
|
+
{
|
|
451
|
+
taskId: 'c',
|
|
452
|
+
taskTitle: 'Task C',
|
|
453
|
+
plannedStatus: 'completed',
|
|
454
|
+
matchedFiles: [],
|
|
455
|
+
verdict: 'DONE',
|
|
456
|
+
fixIterations: 3,
|
|
457
|
+
},
|
|
458
|
+
],
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
const summary = buildFixTrailSummary(report);
|
|
462
|
+
expect(summary).toBe('Task A: 2 fix iterations; Task C: 3 fix iterations');
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
it('returns undefined when no tasks have rework', () => {
|
|
466
|
+
const report = makeReport({
|
|
467
|
+
taskEvidence: [
|
|
468
|
+
{
|
|
469
|
+
taskId: 'a',
|
|
470
|
+
taskTitle: 'Clean',
|
|
471
|
+
plannedStatus: 'completed',
|
|
472
|
+
matchedFiles: [],
|
|
473
|
+
verdict: 'DONE',
|
|
474
|
+
fixIterations: 0,
|
|
475
|
+
},
|
|
476
|
+
],
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
expect(buildFixTrailSummary(report)).toBeUndefined();
|
|
480
|
+
});
|
|
481
|
+
|
|
482
|
+
it('returns undefined for empty task evidence', () => {
|
|
483
|
+
const report = makeReport({ taskEvidence: [] });
|
|
484
|
+
expect(buildFixTrailSummary(report)).toBeUndefined();
|
|
485
|
+
});
|
|
486
|
+
});
|