@stackmemoryai/stackmemory 1.0.1 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/src/cli/claude-sm.js +65 -0
  2. package/dist/src/cli/commands/audit.js +134 -0
  3. package/dist/src/cli/commands/bench.js +252 -0
  4. package/dist/src/cli/commands/dashboard.js +2 -1
  5. package/dist/src/cli/commands/stats.js +118 -0
  6. package/dist/src/cli/index.js +6 -0
  7. package/dist/src/core/config/feature-flags.js +7 -1
  8. package/dist/src/core/context/enhanced-rehydration.js +24 -5
  9. package/dist/src/core/extensions/cerebras-adapter.js +28 -0
  10. package/dist/src/core/extensions/deepinfra-adapter.js +32 -0
  11. package/dist/src/core/extensions/provider-adapter.js +33 -240
  12. package/dist/src/core/models/complexity-scorer.js +154 -0
  13. package/dist/src/core/models/model-router.js +230 -36
  14. package/dist/src/core/models/provider-pricing.js +63 -0
  15. package/dist/src/core/models/sensitive-guard.js +112 -0
  16. package/dist/src/core/monitoring/feedback-loops.js +88 -0
  17. package/dist/src/hooks/schemas.js +12 -1
  18. package/dist/src/integrations/anthropic/batch-client.js +256 -0
  19. package/dist/src/integrations/anthropic/client.js +87 -72
  20. package/dist/src/integrations/claude-code/subagent-client.js +133 -12
  21. package/dist/src/integrations/graphiti/client.js +16 -4
  22. package/dist/src/integrations/mcp/handlers/index.js +25 -1
  23. package/dist/src/integrations/mcp/handlers/provider-handlers.js +227 -0
  24. package/dist/src/integrations/mcp/server.js +316 -1
  25. package/dist/src/integrations/mcp/tool-definitions.js +90 -1
  26. package/dist/src/orchestrators/multimodal/baselines.js +128 -0
  27. package/dist/src/orchestrators/multimodal/constants.js +9 -1
  28. package/dist/src/orchestrators/multimodal/harness.js +86 -6
  29. package/dist/src/orchestrators/multimodal/providers.js +113 -2
  30. package/dist/src/skills/recursive-agent-orchestrator.js +15 -10
  31. package/dist/src/utils/fuzzy-edit.js +162 -0
  32. package/dist/src/utils/hook-installer.js +155 -0
  33. package/package.json +6 -2
  34. package/scripts/gepa/.before-optimize.md +159 -0
  35. package/scripts/gepa/generations/gen-000/baseline.md +159 -124
  36. package/scripts/gepa/generations/gen-001/baseline.md +159 -0
  37. package/scripts/gepa/generations/gen-001/variant-a.md +166 -0
  38. package/scripts/gepa/generations/gen-001/variant-b.md +237 -0
  39. package/scripts/gepa/generations/gen-001/variant-c.md +61 -0
  40. package/scripts/gepa/generations/gen-001/variant-d.md +119 -0
  41. package/scripts/gepa/results/eval-1-baseline.json +41 -0
  42. package/scripts/gepa/results/eval-1-variant-a.json +41 -0
  43. package/scripts/gepa/results/eval-1-variant-b.json +41 -0
  44. package/scripts/gepa/results/eval-1-variant-c.json +41 -0
  45. package/scripts/gepa/results/eval-1-variant-d.json +41 -0
  46. package/scripts/gepa/state.json +41 -2
  47. package/scripts/install-claude-hooks-auto.js +176 -44
  48. package/templates/claude-hooks/auto-checkpoint.js +174 -0
  49. package/templates/claude-hooks/chime-on-stop.sh +22 -0
  50. package/templates/claude-hooks/session-rescue.sh +15 -0
  51. package/templates/claude-hooks/stop-checkpoint.js +120 -0
@@ -30,6 +30,8 @@ import { TraceDetector } from "../../core/trace/trace-detector.js";
30
30
  import { LLMContextRetrieval } from "../../core/retrieval/index.js";
31
31
  import { DiscoveryHandlers } from "./handlers/discovery-handlers.js";
32
32
  import { DiffMemHandlers } from "./handlers/diffmem-handlers.js";
33
+ import { GraphitiClient } from "../graphiti/client.js";
34
+ import { fuzzyEdit } from "../../utils/fuzzy-edit.js";
33
35
  import { v4 as uuidv4 } from "uuid";
34
36
  import {
35
37
  DEFAULT_PLANNER_MODEL,
@@ -62,6 +64,8 @@ class LocalStackMemoryMCP {
62
64
  contextRetrieval;
63
65
  discoveryHandlers;
64
66
  diffMemHandlers;
67
+ providerHandlers = null;
68
+ graphitiClient = null;
65
69
  pendingPlans = /* @__PURE__ */ new Map();
66
70
  constructor() {
67
71
  this.projectRoot = this.findProjectRoot();
@@ -91,6 +95,18 @@ class LocalStackMemoryMCP {
91
95
  influence_score REAL,
92
96
  timestamp INTEGER DEFAULT (unixepoch())
93
97
  );
98
+
99
+ CREATE TABLE IF NOT EXISTS edit_telemetry (
100
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
101
+ timestamp INTEGER NOT NULL DEFAULT (unixepoch()),
102
+ session_id TEXT,
103
+ tool_name TEXT NOT NULL,
104
+ file_path TEXT,
105
+ success INTEGER NOT NULL DEFAULT 1,
106
+ error_type TEXT,
107
+ error_message TEXT
108
+ );
109
+ CREATE INDEX IF NOT EXISTS idx_edit_telemetry_ts ON edit_telemetry(timestamp);
94
110
  `);
95
111
  this.frameManager = new FrameManager(this.db, this.projectId);
96
112
  this.initLinearIfEnabled();
@@ -122,6 +138,16 @@ class LocalStackMemoryMCP {
122
138
  projectRoot: this.projectRoot
123
139
  });
124
140
  this.diffMemHandlers = new DiffMemHandlers();
141
+ this.initProviderHandlers();
142
+ if (process.env.GRAPHITI_ENDPOINT) {
143
+ this.graphitiClient = new GraphitiClient({
144
+ endpoint: process.env.GRAPHITI_ENDPOINT,
145
+ projectNamespace: process.env.STACKMEMORY_PROJECT_ID || this.projectId
146
+ });
147
+ logger.info("Graphiti client initialized", {
148
+ endpoint: process.env.GRAPHITI_ENDPOINT
149
+ });
150
+ }
125
151
  this.setupHandlers();
126
152
  this.loadInitialContext();
127
153
  this.loadPendingPlans();
@@ -243,6 +269,16 @@ ${summary}...`, 0.8);
243
269
  this.contexts.set(ctx.id, ctx);
244
270
  });
245
271
  }
272
+ async initProviderHandlers() {
273
+ if (!isFeatureEnabled("multiProvider")) return;
274
+ try {
275
+ const { ProviderHandlers } = await import("./handlers/provider-handlers.js");
276
+ this.providerHandlers = new ProviderHandlers();
277
+ logger.info("Provider handlers initialized (multiProvider enabled)");
278
+ } catch (error) {
279
+ logger.warn("Failed to initialize provider handlers", { error });
280
+ }
281
+ }
246
282
  setupHandlers() {
247
283
  this.server.setRequestHandler(
248
284
  z.object({
@@ -1047,7 +1083,153 @@ ${summary}...`, 0.8);
1047
1083
  type: "object",
1048
1084
  properties: {}
1049
1085
  }
1050
- }
1086
+ },
1087
+ // Graphiti tools (only active when GRAPHITI_ENDPOINT is set)
1088
+ ...this.graphitiClient ? [
1089
+ {
1090
+ name: "graphiti_status",
1091
+ description: "Check Graphiti temporal knowledge graph connection status",
1092
+ inputSchema: {
1093
+ type: "object",
1094
+ properties: {}
1095
+ }
1096
+ },
1097
+ {
1098
+ name: "graphiti_query",
1099
+ description: "Query the Graphiti temporal knowledge graph for entities, relations, and episodes",
1100
+ inputSchema: {
1101
+ type: "object",
1102
+ properties: {
1103
+ query: {
1104
+ type: "string",
1105
+ description: "Semantic text query"
1106
+ },
1107
+ entityTypes: {
1108
+ type: "array",
1109
+ items: { type: "string" },
1110
+ description: 'Entity types to filter (e.g., ["Person", "File", "Issue"])'
1111
+ },
1112
+ validFrom: {
1113
+ type: "number",
1114
+ description: "Start of time window (epoch ms)"
1115
+ },
1116
+ validTo: {
1117
+ type: "number",
1118
+ description: "End of time window (epoch ms)"
1119
+ },
1120
+ maxHops: {
1121
+ type: "number",
1122
+ description: "Graph traversal depth (default 2)"
1123
+ },
1124
+ k: {
1125
+ type: "number",
1126
+ description: "Top-k results (default 20)"
1127
+ }
1128
+ }
1129
+ }
1130
+ }
1131
+ ] : [],
1132
+ // Provider tools (only active when STACKMEMORY_MULTI_PROVIDER=true)
1133
+ ...isFeatureEnabled("multiProvider") ? [
1134
+ {
1135
+ name: "delegate_to_model",
1136
+ description: "Route a prompt to a specific provider/model. Uses smart cost-based routing by default.",
1137
+ inputSchema: {
1138
+ type: "object",
1139
+ properties: {
1140
+ prompt: {
1141
+ type: "string",
1142
+ description: "The prompt to send"
1143
+ },
1144
+ provider: {
1145
+ type: "string",
1146
+ enum: [
1147
+ "anthropic",
1148
+ "cerebras",
1149
+ "deepinfra",
1150
+ "openai",
1151
+ "openrouter"
1152
+ ],
1153
+ description: "Override provider (auto-routes if omitted)"
1154
+ },
1155
+ model: {
1156
+ type: "string",
1157
+ description: "Override model name"
1158
+ },
1159
+ taskType: {
1160
+ type: "string",
1161
+ enum: [
1162
+ "linting",
1163
+ "context",
1164
+ "code",
1165
+ "testing",
1166
+ "review",
1167
+ "plan"
1168
+ ],
1169
+ description: "Task type for auto-routing"
1170
+ },
1171
+ maxTokens: {
1172
+ type: "number",
1173
+ description: "Max tokens"
1174
+ },
1175
+ temperature: { type: "number" },
1176
+ system: {
1177
+ type: "string",
1178
+ description: "System prompt"
1179
+ }
1180
+ },
1181
+ required: ["prompt"]
1182
+ }
1183
+ },
1184
+ {
1185
+ name: "batch_submit",
1186
+ description: "Submit prompts to Anthropic Batch API (50% discount, async)",
1187
+ inputSchema: {
1188
+ type: "object",
1189
+ properties: {
1190
+ prompts: {
1191
+ type: "array",
1192
+ items: {
1193
+ type: "object",
1194
+ properties: {
1195
+ id: { type: "string" },
1196
+ prompt: { type: "string" },
1197
+ model: { type: "string" },
1198
+ maxTokens: { type: "number" },
1199
+ system: { type: "string" }
1200
+ },
1201
+ required: ["id", "prompt"]
1202
+ },
1203
+ description: "Array of prompts to batch"
1204
+ },
1205
+ description: {
1206
+ type: "string",
1207
+ description: "Batch job description"
1208
+ }
1209
+ },
1210
+ required: ["prompts"]
1211
+ }
1212
+ },
1213
+ {
1214
+ name: "batch_check",
1215
+ description: "Check status or retrieve results for a batch job",
1216
+ inputSchema: {
1217
+ type: "object",
1218
+ properties: {
1219
+ batchId: {
1220
+ type: "string",
1221
+ description: "Batch job ID"
1222
+ },
1223
+ retrieve: {
1224
+ type: "boolean",
1225
+ description: "Retrieve results if complete",
1226
+ default: false
1227
+ }
1228
+ },
1229
+ required: ["batchId"]
1230
+ }
1231
+ }
1232
+ ] : []
1051
1233
  ]
1052
1234
  };
1053
1235
  }
@@ -1195,6 +1377,111 @@ ${summary}...`, 0.8);
1195
1377
  case "diffmem_status":
1196
1378
  result = await this.diffMemHandlers.handleStatus();
1197
1379
  break;
1380
+ case "sm_edit":
1381
+ result = await this.handleSmEdit(args);
1382
+ break;
1383
+ // Provider tools
1384
+ case "delegate_to_model":
1385
+ if (!this.providerHandlers) {
1386
+ result = {
1387
+ content: [
1388
+ {
1389
+ type: "text",
1390
+ text: "Multi-provider routing is disabled."
1391
+ }
1392
+ ]
1393
+ };
1394
+ } else {
1395
+ result = await this.providerHandlers.handleDelegateToModel(
1396
+ args
1397
+ );
1398
+ }
1399
+ break;
1400
+ case "batch_submit":
1401
+ if (!this.providerHandlers) {
1402
+ result = {
1403
+ content: [
1404
+ {
1405
+ type: "text",
1406
+ text: "Multi-provider routing is disabled."
1407
+ }
1408
+ ]
1409
+ };
1410
+ } else {
1411
+ result = await this.providerHandlers.handleBatchSubmit(
1412
+ args
1413
+ );
1414
+ }
1415
+ break;
1416
+ case "batch_check":
1417
+ if (!this.providerHandlers) {
1418
+ result = {
1419
+ content: [
1420
+ {
1421
+ type: "text",
1422
+ text: "Multi-provider routing is disabled."
1423
+ }
1424
+ ]
1425
+ };
1426
+ } else {
1427
+ result = await this.providerHandlers.handleBatchCheck(
1428
+ args
1429
+ );
1430
+ }
1431
+ break;
1432
+ // Graphiti tools
1433
+ case "graphiti_status":
1434
+ if (!this.graphitiClient) {
1435
+ result = {
1436
+ content: [
1437
+ {
1438
+ type: "text",
1439
+ text: JSON.stringify({
1440
+ connected: false,
1441
+ message: "Graphiti integration disabled (GRAPHITI_ENDPOINT not set)"
1442
+ })
1443
+ }
1444
+ ]
1445
+ };
1446
+ } else {
1447
+ const status = await this.graphitiClient.getStatus();
1448
+ result = {
1449
+ content: [
1450
+ { type: "text", text: JSON.stringify(status, null, 2) }
1451
+ ]
1452
+ };
1453
+ }
1454
+ break;
1455
+ case "graphiti_query":
1456
+ if (!this.graphitiClient) {
1457
+ result = {
1458
+ content: [
1459
+ {
1460
+ type: "text",
1461
+ text: "Graphiti integration disabled (GRAPHITI_ENDPOINT not set)"
1462
+ }
1463
+ ]
1464
+ };
1465
+ } else {
1466
+ const gCtx = await this.graphitiClient.queryTemporal({
1467
+ query: args.query,
1468
+ entityTypes: args.entityTypes,
1469
+ validFrom: args.validFrom,
1470
+ validTo: args.validTo,
1471
+ maxHops: args.maxHops,
1472
+ k: args.k
1473
+ });
1474
+ const text = gCtx.chunks.map((c) => c.text).join("\n\n");
1475
+ result = {
1476
+ content: [
1477
+ {
1478
+ type: "text",
1479
+ text: text || `No results found (${gCtx.totalTokens} tokens searched)`
1480
+ }
1481
+ ]
1482
+ };
1483
+ }
1484
+ break;
1198
1485
  default:
1199
1486
  throw new Error(`Unknown tool: ${name}`);
1200
1487
  }
@@ -2639,6 +2926,34 @@ ${typeBreakdown}`
2639
2926
  };
2640
2927
  }
2641
2928
  }
2929
+ async handleSmEdit(args) {
2930
+ const {
2931
+ file_path: filePath,
2932
+ old_string: oldString,
2933
+ new_string: newString,
2934
+ threshold = 0.85
2935
+ } = args;
2936
+ if (!filePath || !oldString || newString === void 0) {
2937
+ throw new Error("file_path, old_string, and new_string are required");
2938
+ }
2939
+ const { readFileSync: readFileSync2, writeFileSync: writeFileSync2 } = await import("fs");
2940
+ const content = readFileSync2(filePath, "utf-8");
2941
+ const editResult = fuzzyEdit(content, oldString, newString, threshold);
2942
+ if (!editResult) {
2943
+ return {
2944
+ success: false,
2945
+ error: "No match found above threshold",
2946
+ threshold
2947
+ };
2948
+ }
2949
+ writeFileSync2(filePath, editResult.content, "utf-8");
2950
+ return {
2951
+ success: true,
2952
+ method: editResult.match.method,
2953
+ confidence: editResult.match.confidence,
2954
+ matchedText: editResult.match.matchedText.length > 200 ? editResult.match.matchedText.slice(0, 200) + "..." : editResult.match.matchedText
2955
+ };
2956
+ }
2642
2957
  async start() {
2643
2958
  const transport = new StdioServerTransport();
2644
2959
  await this.server.connect(transport);
@@ -12,7 +12,9 @@ class MCPToolDefinitions {
12
12
  ...this.getTaskTools(),
13
13
  ...this.getLinearTools(),
14
14
  ...this.getTraceTools(),
15
- ...this.getDiscoveryTools()
15
+ ...this.getDiscoveryTools(),
16
+ ...this.getEditTools(),
17
+ ...this.getGraphitiTools()
16
18
  ];
17
19
  }
18
20
  /**
@@ -670,6 +672,89 @@ class MCPToolDefinitions {
670
672
  }
671
673
  ];
672
674
  }
675
+ /**
676
+ * Edit tools (fuzzy edit fallback)
677
+ */
678
+ getEditTools() {
679
+ return [
680
+ {
681
+ name: "sm_edit",
682
+ description: "Fuzzy file edit \u2014 fallback when Claude Code's Edit tool fails on whitespace or indentation mismatches. Uses four-tier matching: exact, whitespace-normalized, indentation-insensitive, and line-level fuzzy (Levenshtein).",
683
+ inputSchema: {
684
+ type: "object",
685
+ properties: {
686
+ file_path: {
687
+ type: "string",
688
+ description: "Absolute path to the file to edit"
689
+ },
690
+ old_string: {
691
+ type: "string",
692
+ description: "The text to find and replace"
693
+ },
694
+ new_string: {
695
+ type: "string",
696
+ description: "The replacement text"
697
+ },
698
+ threshold: {
699
+ type: "number",
700
+ default: 0.85,
701
+ description: "Minimum similarity threshold for fuzzy matching (0-1). Default 0.85."
702
+ }
703
+ },
704
+ required: ["file_path", "old_string", "new_string"]
705
+ }
706
+ }
707
+ ];
708
+ }
709
+ /**
710
+ * Graphiti knowledge graph tools
711
+ */
712
+ getGraphitiTools() {
713
+ return [
714
+ {
715
+ name: "graphiti_status",
716
+ description: "Check Graphiti temporal knowledge graph connection status",
717
+ inputSchema: {
718
+ type: "object",
719
+ properties: {}
720
+ }
721
+ },
722
+ {
723
+ name: "graphiti_query",
724
+ description: "Query the Graphiti temporal knowledge graph for entities, relations, and episodes",
725
+ inputSchema: {
726
+ type: "object",
727
+ properties: {
728
+ query: {
729
+ type: "string",
730
+ description: "Semantic text query"
731
+ },
732
+ entityTypes: {
733
+ type: "array",
734
+ items: { type: "string" },
735
+ description: 'Entity types to filter (e.g., ["Person", "File", "Issue"])'
736
+ },
737
+ validFrom: {
738
+ type: "number",
739
+ description: "Start of time window (epoch ms)"
740
+ },
741
+ validTo: {
742
+ type: "number",
743
+ description: "End of time window (epoch ms)"
744
+ },
745
+ maxHops: {
746
+ type: "number",
747
+ description: "Graph traversal depth (default 2)"
748
+ },
749
+ k: {
750
+ type: "number",
751
+ description: "Top-k results (default 20)"
752
+ }
753
+ }
754
+ }
755
+ }
756
+ ];
757
+ }
673
758
  /**
674
759
  * Get tool definition by name
675
760
  */
@@ -691,6 +776,10 @@ class MCPToolDefinitions {
691
776
  return this.getTraceTools();
692
777
  case "discovery":
693
778
  return this.getDiscoveryTools();
779
+ case "edit":
780
+ return this.getEditTools();
781
+ case "graphiti":
782
+ return this.getGraphitiTools();
694
783
  default:
695
784
  return [];
696
785
  }
@@ -0,0 +1,128 @@
1
+ import { fileURLToPath as __fileURLToPath } from 'url';
2
+ import { dirname as __pathDirname } from 'path';
3
+ const __filename = __fileURLToPath(import.meta.url);
4
+ const __dirname = __pathDirname(__filename);
5
+ const SWE_BENCH_BASELINES = [
6
+ {
7
+ agent: "Claude Code",
8
+ model: "claude-sonnet-4",
9
+ benchmark: "swe-bench-verified",
10
+ resolveRate: 0.704,
11
+ date: "2025-12-01",
12
+ source: "https://www.swebench.com/"
13
+ },
14
+ {
15
+ agent: "Devin",
16
+ model: "mixed",
17
+ benchmark: "swe-bench-verified",
18
+ resolveRate: 0.551,
19
+ date: "2025-10-01",
20
+ source: "https://www.swebench.com/"
21
+ },
22
+ {
23
+ agent: "OpenHands",
24
+ model: "claude-sonnet-4",
25
+ benchmark: "swe-bench-verified",
26
+ resolveRate: 0.535,
27
+ date: "2025-09-01",
28
+ source: "https://www.swebench.com/"
29
+ },
30
+ {
31
+ agent: "Aider",
32
+ model: "claude-sonnet-4",
33
+ benchmark: "swe-bench-verified",
34
+ resolveRate: 0.489,
35
+ date: "2025-10-01",
36
+ source: "https://www.swebench.com/"
37
+ }
38
+ ];
39
+ const HARNESS_TARGETS = {
40
+ /** Plan generation should complete within 10s */
41
+ planLatencyP95Ms: 1e4,
42
+ /**
43
+ * Full cycle (plan + implement + critique) within 5 minutes.
44
+ * Codex execution benchmarks show 89-231s for real runs (1-2 iterations).
45
+ * 300s allows for 2-iteration runs with margin.
46
+ */
47
+ totalLatencyP95Ms: 3e5,
48
+ /**
49
+ * Single-iteration (first-pass) latency ceiling: 2.5 minutes.
50
+ * Based on observed single-pass runs of 89-115s with headroom.
51
+ */
52
+ singleIterLatencyP95Ms: 15e4,
53
+ /** First-pass approval rate (no retries needed) */
54
+ firstPassApprovalRate: 0.7,
55
+ /** Edit success rate (exact + fuzzy combined) */
56
+ editSuccessRate: 0.9,
57
+ /** Edit fuzzy fallback rate (lower = better, means exact match works) */
58
+ editFuzzyFallbackRate: 0.15,
59
+ /** Context overhead should be < 6000 tokens */
60
+ contextTokenBudget: 6e3
61
+ };
62
+ function summarizeRuns(runs) {
63
+ if (runs.length === 0) {
64
+ return {
65
+ totalRuns: 0,
66
+ approvalRate: 0,
67
+ firstPassRate: 0,
68
+ avgIterations: 0,
69
+ avgPlanLatencyMs: 0,
70
+ avgTotalLatencyMs: 0,
71
+ p95PlanLatencyMs: 0,
72
+ p95TotalLatencyMs: 0,
73
+ p95SingleIterLatencyMs: 0,
74
+ editSuccessRate: 0,
75
+ editFuzzyRate: 0,
76
+ avgContextTokens: 0,
77
+ passesTargets: {}
78
+ };
79
+ }
80
+ const approvedRuns = runs.filter((r) => r.approved);
81
+ const firstPassRuns = runs.filter((r) => r.approved && r.iterations <= 1);
82
+ const totalEdits = runs.reduce((s, r) => s + r.editAttempts, 0);
83
+ const totalEditSuccesses = runs.reduce((s, r) => s + r.editSuccesses, 0);
84
+ const totalFuzzy = runs.reduce((s, r) => s + r.editFuzzyFallbacks, 0);
85
+ const planLatencies = runs.map((r) => r.planLatencyMs).sort((a, b) => a - b);
86
+ const totalLatencies = runs.map((r) => r.totalLatencyMs).sort((a, b) => a - b);
87
+ const p95Idx = Math.min(Math.ceil(runs.length * 0.95) - 1, runs.length - 1);
88
+ const approvalRate = approvedRuns.length / runs.length;
89
+ const firstPassRate = firstPassRuns.length / runs.length;
90
+ const editSuccessRate = totalEdits > 0 ? totalEditSuccesses / totalEdits : 1;
91
+ const editFuzzyRate = totalEditSuccesses > 0 ? totalFuzzy / totalEditSuccesses : 0;
92
+ const avgContextTokens = runs.reduce((s, r) => s + r.contextTokens, 0) / runs.length;
93
+ const p95Plan = planLatencies[p95Idx];
94
+ const p95Total = totalLatencies[p95Idx];
95
+ const singleIterLatencies = runs.filter((r) => r.approved && r.iterations <= 1).map((r) => r.totalLatencyMs).sort((a, b) => a - b);
96
+ const p95SingleIter = singleIterLatencies.length > 0 ? singleIterLatencies[Math.min(
97
+ Math.ceil(singleIterLatencies.length * 0.95) - 1,
98
+ singleIterLatencies.length - 1
99
+ )] : 0;
100
+ return {
101
+ totalRuns: runs.length,
102
+ approvalRate,
103
+ firstPassRate,
104
+ avgIterations: runs.reduce((s, r) => s + r.iterations, 0) / runs.length,
105
+ avgPlanLatencyMs: runs.reduce((s, r) => s + r.planLatencyMs, 0) / runs.length,
106
+ avgTotalLatencyMs: runs.reduce((s, r) => s + r.totalLatencyMs, 0) / runs.length,
107
+ p95PlanLatencyMs: p95Plan,
108
+ p95TotalLatencyMs: p95Total,
109
+ p95SingleIterLatencyMs: p95SingleIter,
110
+ editSuccessRate,
111
+ editFuzzyRate,
112
+ avgContextTokens,
113
+ passesTargets: {
114
+ planLatency: p95Plan <= HARNESS_TARGETS.planLatencyP95Ms,
115
+ totalLatency: p95Total <= HARNESS_TARGETS.totalLatencyP95Ms,
116
+ singleIterLatency: singleIterLatencies.length === 0 || p95SingleIter <= HARNESS_TARGETS.singleIterLatencyP95Ms,
117
+ firstPassApproval: firstPassRate >= HARNESS_TARGETS.firstPassApprovalRate,
118
+ editSuccess: editSuccessRate >= HARNESS_TARGETS.editSuccessRate,
119
+ editFuzzyRate: editFuzzyRate <= HARNESS_TARGETS.editFuzzyFallbackRate,
120
+ contextBudget: avgContextTokens <= HARNESS_TARGETS.contextTokenBudget
121
+ }
122
+ };
123
+ }
124
+ export {
125
+ HARNESS_TARGETS,
126
+ SWE_BENCH_BASELINES,
127
+ summarizeRuns
128
+ };
@@ -8,9 +8,17 @@ const DEFAULT_IMPLEMENTER = process.env.STACKMEMORY_MM_IMPLEMENTER || "codex";
8
8
  const DEFAULT_MAX_ITERS = Number(
9
9
  process.env.STACKMEMORY_MM_MAX_ITERS || 2
10
10
  );
11
+ const STRUCTURED_RESPONSE_SUFFIX = `
12
+
13
+ Response Format:
14
+ - When asking a question, ALWAYS end with structured choices: Yes/No, or numbered options (1, 2, 3, 4), or lettered options (A, B, C, D).
15
+ - When completing a task, end with "Done." followed by suggested next steps as numbered options (1, 2, 3, 4).
16
+ - Never leave responses open-ended. Always provide explicit options the user can select.
17
+ - Keep options concise (one line each). Use the format that best fits: Yes/No for confirmations, 1-4 for action choices, A-D for category selections.`;
11
18
  export {
12
19
  DEFAULT_IMPLEMENTER,
13
20
  DEFAULT_MAX_ITERS,
14
21
  DEFAULT_PLANNER_MODEL,
15
- DEFAULT_REVIEWER_MODEL
22
+ DEFAULT_REVIEWER_MODEL,
23
+ STRUCTURED_RESPONSE_SUFFIX
16
24
  };