@stackmemoryai/stackmemory 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/commands/audit.js +134 -0
- package/dist/src/cli/commands/bench.js +252 -0
- package/dist/src/cli/commands/dashboard.js +2 -1
- package/dist/src/cli/commands/stats.js +118 -0
- package/dist/src/cli/index.js +6 -0
- package/dist/src/core/config/feature-flags.js +7 -1
- package/dist/src/core/context/enhanced-rehydration.js +24 -5
- package/dist/src/core/extensions/cerebras-adapter.js +28 -0
- package/dist/src/core/extensions/deepinfra-adapter.js +32 -0
- package/dist/src/core/extensions/provider-adapter.js +33 -240
- package/dist/src/core/models/complexity-scorer.js +154 -0
- package/dist/src/core/models/model-router.js +230 -36
- package/dist/src/core/models/provider-pricing.js +63 -0
- package/dist/src/core/models/sensitive-guard.js +112 -0
- package/dist/src/core/monitoring/feedback-loops.js +88 -0
- package/dist/src/hooks/schemas.js +12 -1
- package/dist/src/integrations/anthropic/batch-client.js +256 -0
- package/dist/src/integrations/anthropic/client.js +87 -72
- package/dist/src/integrations/claude-code/subagent-client.js +133 -12
- package/dist/src/integrations/graphiti/client.js +16 -4
- package/dist/src/integrations/mcp/handlers/index.js +25 -1
- package/dist/src/integrations/mcp/handlers/provider-handlers.js +227 -0
- package/dist/src/integrations/mcp/server.js +207 -1
- package/dist/src/integrations/mcp/tool-definitions.js +38 -1
- package/dist/src/orchestrators/multimodal/baselines.js +128 -0
- package/dist/src/orchestrators/multimodal/constants.js +9 -1
- package/dist/src/orchestrators/multimodal/harness.js +86 -6
- package/dist/src/orchestrators/multimodal/providers.js +113 -2
- package/dist/src/skills/recursive-agent-orchestrator.js +15 -10
- package/dist/src/utils/fuzzy-edit.js +162 -0
- package/package.json +5 -1
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { fileURLToPath as __fileURLToPath } from 'url';
|
|
2
|
+
import { dirname as __pathDirname } from 'path';
|
|
3
|
+
const __filename = __fileURLToPath(import.meta.url);
|
|
4
|
+
const __dirname = __pathDirname(__filename);
|
|
5
|
+
import { Command } from "commander";
|
|
6
|
+
import { existsSync, readFileSync } from "fs";
|
|
7
|
+
import { join } from "path";
|
|
8
|
+
import { homedir } from "os";
|
|
9
|
+
let countTokens;
|
|
10
|
+
try {
|
|
11
|
+
const tokenizer = await import("@anthropic-ai/tokenizer");
|
|
12
|
+
countTokens = tokenizer.countTokens;
|
|
13
|
+
} catch {
|
|
14
|
+
countTokens = (text) => Math.ceil(text.length / 3.5);
|
|
15
|
+
}
|
|
16
|
+
function readFileSafe(filePath) {
|
|
17
|
+
try {
|
|
18
|
+
if (existsSync(filePath)) {
|
|
19
|
+
return readFileSync(filePath, "utf-8");
|
|
20
|
+
}
|
|
21
|
+
} catch {
|
|
22
|
+
}
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
function createAuditCommand() {
|
|
26
|
+
const audit = new Command("audit").description(
|
|
27
|
+
"Measure context overhead (tokens injected before first message)"
|
|
28
|
+
).option("--json", "Output as JSON", false).action(async (options) => {
|
|
29
|
+
const projectRoot = process.cwd();
|
|
30
|
+
const home = homedir();
|
|
31
|
+
const entries = [];
|
|
32
|
+
const globalClaudeMd = readFileSafe(join(home, ".claude", "CLAUDE.md"));
|
|
33
|
+
if (globalClaudeMd) {
|
|
34
|
+
entries.push({
|
|
35
|
+
source: "~/.claude/CLAUDE.md",
|
|
36
|
+
tokens: countTokens(globalClaudeMd),
|
|
37
|
+
percent: 0
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
const projectClaudeMd = readFileSafe(join(projectRoot, "CLAUDE.md"));
|
|
41
|
+
if (projectClaudeMd) {
|
|
42
|
+
entries.push({
|
|
43
|
+
source: "./CLAUDE.md",
|
|
44
|
+
tokens: countTokens(projectClaudeMd),
|
|
45
|
+
percent: 0
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
const projectSlug = projectRoot.replace(/\//g, "-");
|
|
49
|
+
const memoryPath = join(
|
|
50
|
+
home,
|
|
51
|
+
".claude",
|
|
52
|
+
"projects",
|
|
53
|
+
projectSlug,
|
|
54
|
+
"memory",
|
|
55
|
+
"MEMORY.md"
|
|
56
|
+
);
|
|
57
|
+
const memoryMd = readFileSafe(memoryPath);
|
|
58
|
+
if (memoryMd) {
|
|
59
|
+
entries.push({
|
|
60
|
+
source: "auto-memory/MEMORY.md",
|
|
61
|
+
tokens: countTokens(memoryMd),
|
|
62
|
+
percent: 0
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
const handoffPath = join(projectRoot, ".stackmemory", "handoff.md");
|
|
66
|
+
const handoffMd = readFileSafe(handoffPath);
|
|
67
|
+
if (handoffMd) {
|
|
68
|
+
entries.push({
|
|
69
|
+
source: ".stackmemory/handoff.md",
|
|
70
|
+
tokens: countTokens(handoffMd),
|
|
71
|
+
percent: 0
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
const { MCPToolDefinitions } = await import("../../integrations/mcp/tool-definitions.js");
|
|
76
|
+
const defs = new MCPToolDefinitions();
|
|
77
|
+
const allTools = defs.getAllToolDefinitions();
|
|
78
|
+
const schemasJson = JSON.stringify(allTools);
|
|
79
|
+
entries.push({
|
|
80
|
+
source: "MCP tool schemas",
|
|
81
|
+
tokens: countTokens(schemasJson),
|
|
82
|
+
percent: 0
|
|
83
|
+
});
|
|
84
|
+
} catch {
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const dbPath = join(projectRoot, ".stackmemory", "context.db");
|
|
88
|
+
if (existsSync(dbPath)) {
|
|
89
|
+
const { default: Database } = await import("better-sqlite3");
|
|
90
|
+
const { FrameManager } = await import("../../core/context/index.js");
|
|
91
|
+
const db = new Database(dbPath);
|
|
92
|
+
const fm = new FrameManager(db, "cli-project");
|
|
93
|
+
const hotStack = fm.getHotStackContext();
|
|
94
|
+
if (hotStack) {
|
|
95
|
+
entries.push({
|
|
96
|
+
source: "Active frames (hot stack)",
|
|
97
|
+
tokens: countTokens(hotStack),
|
|
98
|
+
percent: 0
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
db.close();
|
|
102
|
+
}
|
|
103
|
+
} catch {
|
|
104
|
+
}
|
|
105
|
+
const totalTokens = entries.reduce((sum, e) => sum + e.tokens, 0);
|
|
106
|
+
for (const entry of entries) {
|
|
107
|
+
entry.percent = totalTokens > 0 ? Math.round(entry.tokens / totalTokens * 1e3) / 10 : 0;
|
|
108
|
+
}
|
|
109
|
+
if (options.json) {
|
|
110
|
+
console.log(JSON.stringify({ entries, totalTokens }, null, 2));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
console.log("\nContext Overhead Audit");
|
|
114
|
+
console.log("\u2500".repeat(60));
|
|
115
|
+
console.log(
|
|
116
|
+
`${"Source".padEnd(32)} ${"Tokens".padStart(8)} ${"%".padStart(7)}`
|
|
117
|
+
);
|
|
118
|
+
console.log("\u2500".repeat(60));
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
console.log(
|
|
121
|
+
`${entry.source.padEnd(32)} ${String(entry.tokens).padStart(8)} ${(entry.percent + "%").padStart(7)}`
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
console.log("\u2500".repeat(60));
|
|
125
|
+
console.log(
|
|
126
|
+
`${"TOTAL".padEnd(32)} ${String(totalTokens).padStart(8)} ${"100%".padStart(7)}`
|
|
127
|
+
);
|
|
128
|
+
console.log("");
|
|
129
|
+
});
|
|
130
|
+
return audit;
|
|
131
|
+
}
|
|
132
|
+
export {
|
|
133
|
+
createAuditCommand
|
|
134
|
+
};
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import { fileURLToPath as __fileURLToPath } from 'url';
|
|
2
|
+
import { dirname as __pathDirname } from 'path';
|
|
3
|
+
const __filename = __fileURLToPath(import.meta.url);
|
|
4
|
+
const __dirname = __pathDirname(__filename);
|
|
5
|
+
import { Command } from "commander";
|
|
6
|
+
import { existsSync, readFileSync, readdirSync } from "fs";
|
|
7
|
+
import { join } from "path";
|
|
8
|
+
import {
|
|
9
|
+
SWE_BENCH_BASELINES,
|
|
10
|
+
HARNESS_TARGETS,
|
|
11
|
+
summarizeRuns
|
|
12
|
+
} from "../../orchestrators/multimodal/baselines.js";
|
|
13
|
+
import {
|
|
14
|
+
feedbackLoops
|
|
15
|
+
} from "../../core/monitoring/feedback-loops.js";
|
|
16
|
+
function loadRunMetrics(projectRoot) {
|
|
17
|
+
const metricsFile = join(
|
|
18
|
+
projectRoot,
|
|
19
|
+
".stackmemory",
|
|
20
|
+
"build",
|
|
21
|
+
"harness-metrics.jsonl"
|
|
22
|
+
);
|
|
23
|
+
if (!existsSync(metricsFile)) return [];
|
|
24
|
+
const lines = readFileSync(metricsFile, "utf-8").split("\n").filter((l) => l.trim());
|
|
25
|
+
const runs = [];
|
|
26
|
+
for (const line of lines) {
|
|
27
|
+
try {
|
|
28
|
+
runs.push(JSON.parse(line));
|
|
29
|
+
} catch {
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return runs;
|
|
33
|
+
}
|
|
34
|
+
function loadSpikeAudits(projectRoot) {
|
|
35
|
+
const dir = join(projectRoot, ".stackmemory", "build");
|
|
36
|
+
if (!existsSync(dir)) return [];
|
|
37
|
+
return readdirSync(dir).filter((f) => f.startsWith("spike-") && f.endsWith(".json")).sort().reverse().slice(0, 20).map((f) => {
|
|
38
|
+
try {
|
|
39
|
+
return {
|
|
40
|
+
file: f,
|
|
41
|
+
data: JSON.parse(readFileSync(join(dir, f), "utf-8"))
|
|
42
|
+
};
|
|
43
|
+
} catch {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
}).filter(Boolean);
|
|
47
|
+
}
|
|
48
|
+
function createBenchCommand() {
|
|
49
|
+
const bench = new Command("bench").description(
|
|
50
|
+
"Harness benchmarks \u2014 compare local runs against SWE-bench baselines"
|
|
51
|
+
).option("--json", "Output as JSON", false).option("-d, --days <n>", "Only include runs from last N days", "30").option("--baselines", "Show online benchmark baselines only", false).action(async (options) => {
|
|
52
|
+
const projectRoot = process.cwd();
|
|
53
|
+
if (options.baselines) {
|
|
54
|
+
if (options.json) {
|
|
55
|
+
console.log(
|
|
56
|
+
JSON.stringify(
|
|
57
|
+
{ baselines: SWE_BENCH_BASELINES, targets: HARNESS_TARGETS },
|
|
58
|
+
null,
|
|
59
|
+
2
|
|
60
|
+
)
|
|
61
|
+
);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
console.log("\nOnline Benchmark Baselines (SWE-bench Verified)");
|
|
65
|
+
console.log("\u2500".repeat(60));
|
|
66
|
+
console.log(
|
|
67
|
+
`${"Agent".padEnd(20)} ${"Model".padEnd(20)} ${"Resolve".padStart(8)}`
|
|
68
|
+
);
|
|
69
|
+
console.log("\u2500".repeat(60));
|
|
70
|
+
for (const b of SWE_BENCH_BASELINES) {
|
|
71
|
+
console.log(
|
|
72
|
+
`${b.agent.padEnd(20)} ${b.model.padEnd(20)} ${(b.resolveRate * 100).toFixed(1).padStart(7)}%`
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
console.log("\u2500".repeat(60));
|
|
76
|
+
console.log("\nInternal Harness Targets");
|
|
77
|
+
console.log("\u2500".repeat(60));
|
|
78
|
+
console.log(
|
|
79
|
+
` Plan latency P95: ${HARNESS_TARGETS.planLatencyP95Ms}ms`
|
|
80
|
+
);
|
|
81
|
+
console.log(
|
|
82
|
+
` Total latency P95: ${HARNESS_TARGETS.totalLatencyP95Ms}ms`
|
|
83
|
+
);
|
|
84
|
+
console.log(
|
|
85
|
+
` Single-iter latency P95: ${HARNESS_TARGETS.singleIterLatencyP95Ms}ms`
|
|
86
|
+
);
|
|
87
|
+
console.log(
|
|
88
|
+
` First-pass approval: ${(HARNESS_TARGETS.firstPassApprovalRate * 100).toFixed(0)}%`
|
|
89
|
+
);
|
|
90
|
+
console.log(
|
|
91
|
+
` Edit success rate: ${(HARNESS_TARGETS.editSuccessRate * 100).toFixed(0)}%`
|
|
92
|
+
);
|
|
93
|
+
console.log(
|
|
94
|
+
` Fuzzy fallback rate: <${(HARNESS_TARGETS.editFuzzyFallbackRate * 100).toFixed(0)}%`
|
|
95
|
+
);
|
|
96
|
+
console.log(
|
|
97
|
+
` Context token budget: ${HARNESS_TARGETS.contextTokenBudget}`
|
|
98
|
+
);
|
|
99
|
+
console.log("");
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
const days = parseInt(options.days, 10) || 30;
|
|
103
|
+
const cutoff = Date.now() - days * 864e5;
|
|
104
|
+
const allRuns = loadRunMetrics(projectRoot);
|
|
105
|
+
const runs = allRuns.filter((r) => r.timestamp >= cutoff);
|
|
106
|
+
const audits = loadSpikeAudits(projectRoot);
|
|
107
|
+
if (options.json) {
|
|
108
|
+
const summary2 = summarizeRuns(runs);
|
|
109
|
+
console.log(
|
|
110
|
+
JSON.stringify(
|
|
111
|
+
{
|
|
112
|
+
summary: summary2,
|
|
113
|
+
baselines: SWE_BENCH_BASELINES,
|
|
114
|
+
targets: HARNESS_TARGETS,
|
|
115
|
+
runsInWindow: runs.length,
|
|
116
|
+
totalRuns: allRuns.length,
|
|
117
|
+
recentAudits: audits.length
|
|
118
|
+
},
|
|
119
|
+
null,
|
|
120
|
+
2
|
|
121
|
+
)
|
|
122
|
+
);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
console.log(`
|
|
126
|
+
Harness Benchmark Report (last ${days} days)`);
|
|
127
|
+
console.log("\u2550".repeat(60));
|
|
128
|
+
if (runs.length === 0) {
|
|
129
|
+
console.log("\nNo harness runs recorded yet.");
|
|
130
|
+
console.log('Run: stackmemory build "your task" --execute');
|
|
131
|
+
console.log('Or: stackmemory mm-spike -t "task" --execute\n');
|
|
132
|
+
console.log("Online Baselines (SWE-bench Verified):");
|
|
133
|
+
for (const b of SWE_BENCH_BASELINES.slice(0, 3)) {
|
|
134
|
+
console.log(
|
|
135
|
+
` ${b.agent.padEnd(16)} ${(b.resolveRate * 100).toFixed(1)}%`
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
console.log("");
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
const summary = summarizeRuns(runs);
|
|
142
|
+
console.log("\nHarness Metrics:");
|
|
143
|
+
console.log(` Total runs: ${summary.totalRuns}`);
|
|
144
|
+
console.log(
|
|
145
|
+
` Approval rate: ${(summary.approvalRate * 100).toFixed(1)}%`
|
|
146
|
+
);
|
|
147
|
+
console.log(
|
|
148
|
+
` First-pass rate: ${(summary.firstPassRate * 100).toFixed(1)}%`
|
|
149
|
+
);
|
|
150
|
+
console.log(
|
|
151
|
+
` Avg iterations: ${summary.avgIterations.toFixed(1)}`
|
|
152
|
+
);
|
|
153
|
+
console.log(
|
|
154
|
+
` Plan latency (avg): ${Math.round(summary.avgPlanLatencyMs)}ms`
|
|
155
|
+
);
|
|
156
|
+
console.log(
|
|
157
|
+
` Plan latency (P95): ${Math.round(summary.p95PlanLatencyMs)}ms`
|
|
158
|
+
);
|
|
159
|
+
console.log(
|
|
160
|
+
` Total latency (avg): ${Math.round(summary.avgTotalLatencyMs)}ms`
|
|
161
|
+
);
|
|
162
|
+
console.log(
|
|
163
|
+
` Total latency (P95): ${Math.round(summary.p95TotalLatencyMs)}ms`
|
|
164
|
+
);
|
|
165
|
+
console.log(
|
|
166
|
+
` Edit success rate: ${(summary.editSuccessRate * 100).toFixed(1)}%`
|
|
167
|
+
);
|
|
168
|
+
console.log(
|
|
169
|
+
` Fuzzy fallback rate: ${(summary.editFuzzyRate * 100).toFixed(1)}%`
|
|
170
|
+
);
|
|
171
|
+
console.log(
|
|
172
|
+
` Context tokens (avg): ${Math.round(summary.avgContextTokens)}`
|
|
173
|
+
);
|
|
174
|
+
console.log("\nTarget Comparison:");
|
|
175
|
+
const checks = summary.passesTargets;
|
|
176
|
+
for (const [key, passes] of Object.entries(checks)) {
|
|
177
|
+
const icon = passes ? "PASS" : "FAIL";
|
|
178
|
+
console.log(` [${icon}] ${key}`);
|
|
179
|
+
}
|
|
180
|
+
console.log("\nOnline Baselines (SWE-bench Verified):");
|
|
181
|
+
for (const b of SWE_BENCH_BASELINES.slice(0, 4)) {
|
|
182
|
+
console.log(
|
|
183
|
+
` ${b.agent.padEnd(16)} ${(b.resolveRate * 100).toFixed(1)}%`
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
if (audits.length > 0) {
|
|
187
|
+
console.log(`
|
|
188
|
+
Recent Spike Audits (${audits.length}):`);
|
|
189
|
+
for (const a of audits.slice(0, 5)) {
|
|
190
|
+
const task = a.data?.input?.task || "(unknown)";
|
|
191
|
+
const approved = a.data?.iterations?.some(
|
|
192
|
+
(it) => it.critique?.approved
|
|
193
|
+
);
|
|
194
|
+
const icon = approved ? "OK" : "--";
|
|
195
|
+
console.log(` [${icon}] ${task.slice(0, 50)}`);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
console.log("");
|
|
199
|
+
});
|
|
200
|
+
bench.command("loops").description("Show feedback loop configuration, status, and recent events").option("--json", "Output as JSON", false).action((options) => {
|
|
201
|
+
const config = feedbackLoops.getConfig();
|
|
202
|
+
const stats = feedbackLoops.getStats();
|
|
203
|
+
const history = feedbackLoops.getHistory(void 0, 20);
|
|
204
|
+
if (options.json) {
|
|
205
|
+
console.log(JSON.stringify({ config, stats, history }, null, 2));
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
console.log("\nFeedback Loops");
|
|
209
|
+
console.log("\u2550".repeat(60));
|
|
210
|
+
const loopDescriptions = {
|
|
211
|
+
contextPressure: "Context 70%+ \u2192 auto-digest old frames",
|
|
212
|
+
editRecovery: "Edit failure \u2192 sm_edit fuzzy fallback \u2192 telemetry",
|
|
213
|
+
retrievalQuality: "Empty results > 20% \u2192 switch search strategy",
|
|
214
|
+
traceErrorChain: "Same error 3x \u2192 surface anchor + memory",
|
|
215
|
+
harnessRegression: "Approval rate drops \u2192 regression alert",
|
|
216
|
+
sessionDrift: "Depth > 5 or stale frames \u2192 auto-checkpoint"
|
|
217
|
+
};
|
|
218
|
+
console.log("\nLoop Configuration:");
|
|
219
|
+
for (const [name, cfg] of Object.entries(config)) {
|
|
220
|
+
const icon = cfg.enabled ? " ON" : "OFF";
|
|
221
|
+
const desc = loopDescriptions[name] || name;
|
|
222
|
+
const cooldown = cfg.cooldownSec > 0 ? ` (cooldown ${cfg.cooldownSec}s)` : "";
|
|
223
|
+
console.log(` [${icon}] ${name.padEnd(22)} ${desc}${cooldown}`);
|
|
224
|
+
}
|
|
225
|
+
if (Object.keys(stats).length > 0) {
|
|
226
|
+
console.log("\nLoop Stats (this session):");
|
|
227
|
+
for (const [name, s] of Object.entries(stats)) {
|
|
228
|
+
const ago = s.lastFired ? `${Math.round((Date.now() - s.lastFired) / 1e3)}s ago` : "never";
|
|
229
|
+
console.log(
|
|
230
|
+
` ${name.padEnd(22)} ${s.fires} fires, ${s.successes} ok, ${s.errors} err (last: ${ago})`
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
if (history.length > 0) {
|
|
235
|
+
console.log(`
|
|
236
|
+
Recent Events (${history.length}):`);
|
|
237
|
+
for (const e of history.slice(-10)) {
|
|
238
|
+
const time = new Date(e.timestamp).toISOString().slice(11, 19);
|
|
239
|
+
console.log(
|
|
240
|
+
` ${time} [${e.loop}] ${e.trigger} \u2192 ${e.action} (${e.outcome})`
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
} else {
|
|
244
|
+
console.log("\nNo loop events fired yet this session.");
|
|
245
|
+
}
|
|
246
|
+
console.log("");
|
|
247
|
+
});
|
|
248
|
+
return bench;
|
|
249
|
+
}
|
|
250
|
+
export {
|
|
251
|
+
createBenchCommand
|
|
252
|
+
};
|
|
@@ -9,6 +9,7 @@ import { SessionManager } from "../../core/session/session-manager.js";
|
|
|
9
9
|
import Database from "better-sqlite3";
|
|
10
10
|
import { join } from "path";
|
|
11
11
|
import { existsSync } from "fs";
|
|
12
|
+
import { getModelTokenLimit } from "../../core/models/model-router.js";
|
|
12
13
|
const dashboardCommand = {
|
|
13
14
|
command: "dashboard",
|
|
14
15
|
describe: "Display monitoring dashboard in terminal",
|
|
@@ -201,7 +202,7 @@ async function estimateContextUsage(db) {
|
|
|
201
202
|
).get();
|
|
202
203
|
const totalBytes = (result?.input_size || 0) + (result?.output_size || 0);
|
|
203
204
|
const estimatedTokens = totalBytes / 4;
|
|
204
|
-
const maxTokens =
|
|
205
|
+
const maxTokens = getModelTokenLimit(process.env.ANTHROPIC_MODEL);
|
|
205
206
|
return Math.round(estimatedTokens / maxTokens * 100);
|
|
206
207
|
}
|
|
207
208
|
export {
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { fileURLToPath as __fileURLToPath } from 'url';
|
|
2
|
+
import { dirname as __pathDirname } from 'path';
|
|
3
|
+
const __filename = __fileURLToPath(import.meta.url);
|
|
4
|
+
const __dirname = __pathDirname(__filename);
|
|
5
|
+
import { Command } from "commander";
|
|
6
|
+
import { existsSync } from "fs";
|
|
7
|
+
import { join } from "path";
|
|
8
|
+
function createStatsCommand() {
|
|
9
|
+
const stats = new Command("stats").description("Show telemetry statistics").argument("[category]", "Category to show (edits)", "edits").option("-d, --days <n>", "Number of days to show trend", "7").option("--json", "Output as JSON", false).action(async (category, options) => {
|
|
10
|
+
if (category !== "edits") {
|
|
11
|
+
console.log(`Unknown stats category: ${category}`);
|
|
12
|
+
console.log("Available: edits");
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
const projectRoot = process.cwd();
|
|
16
|
+
const dbPath = join(projectRoot, ".stackmemory", "context.db");
|
|
17
|
+
if (!existsSync(dbPath)) {
|
|
18
|
+
console.log('No data. Run "stackmemory init" first.');
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
const { default: Database } = await import("better-sqlite3");
|
|
22
|
+
const db = new Database(dbPath);
|
|
23
|
+
const tableExists = db.prepare(
|
|
24
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='edit_telemetry'"
|
|
25
|
+
).get();
|
|
26
|
+
if (!tableExists) {
|
|
27
|
+
db.close();
|
|
28
|
+
console.log("No edit telemetry data yet.");
|
|
29
|
+
console.log("Edit telemetry is collected via the PostToolUse hook.");
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const days = parseInt(options.days, 10) || 7;
|
|
33
|
+
const cutoff = Math.floor(Date.now() / 1e3) - days * 86400;
|
|
34
|
+
const byTool = db.prepare(
|
|
35
|
+
`SELECT
|
|
36
|
+
tool_name,
|
|
37
|
+
COUNT(*) as total,
|
|
38
|
+
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successes,
|
|
39
|
+
SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failures
|
|
40
|
+
FROM edit_telemetry
|
|
41
|
+
WHERE timestamp >= ?
|
|
42
|
+
GROUP BY tool_name
|
|
43
|
+
ORDER BY total DESC`
|
|
44
|
+
).all(cutoff);
|
|
45
|
+
const topFails = db.prepare(
|
|
46
|
+
`SELECT file_path, COUNT(*) as fail_count
|
|
47
|
+
FROM edit_telemetry
|
|
48
|
+
WHERE success = 0 AND timestamp >= ? AND file_path IS NOT NULL
|
|
49
|
+
GROUP BY file_path
|
|
50
|
+
ORDER BY fail_count DESC
|
|
51
|
+
LIMIT 10`
|
|
52
|
+
).all(cutoff);
|
|
53
|
+
const errorTypes = db.prepare(
|
|
54
|
+
`SELECT error_type, COUNT(*) as count
|
|
55
|
+
FROM edit_telemetry
|
|
56
|
+
WHERE success = 0 AND timestamp >= ? AND error_type IS NOT NULL
|
|
57
|
+
GROUP BY error_type
|
|
58
|
+
ORDER BY count DESC`
|
|
59
|
+
).all(cutoff);
|
|
60
|
+
const trend = db.prepare(
|
|
61
|
+
`SELECT
|
|
62
|
+
date(timestamp, 'unixepoch') as day,
|
|
63
|
+
COUNT(*) as total,
|
|
64
|
+
SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failures
|
|
65
|
+
FROM edit_telemetry
|
|
66
|
+
WHERE timestamp >= ?
|
|
67
|
+
GROUP BY day
|
|
68
|
+
ORDER BY day DESC`
|
|
69
|
+
).all(cutoff);
|
|
70
|
+
db.close();
|
|
71
|
+
if (options.json) {
|
|
72
|
+
console.log(
|
|
73
|
+
JSON.stringify({ byTool, topFails, errorTypes, trend }, null, 2)
|
|
74
|
+
);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
console.log(`
|
|
78
|
+
Edit Telemetry (last ${days} days)`);
|
|
79
|
+
console.log("\u2500".repeat(50));
|
|
80
|
+
if (byTool.length === 0) {
|
|
81
|
+
console.log("No edit telemetry data in this period.");
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
console.log("\nSuccess Rate by Tool:");
|
|
85
|
+
for (const row of byTool) {
|
|
86
|
+
const rate = row.total > 0 ? Math.round(row.successes / row.total * 100) : 0;
|
|
87
|
+
console.log(
|
|
88
|
+
` ${row.tool_name.padEnd(20)} ${rate}% (${row.successes}/${row.total})`
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
if (topFails.length > 0) {
|
|
92
|
+
console.log("\nTop Failure Files:");
|
|
93
|
+
for (const row of topFails) {
|
|
94
|
+
console.log(` ${row.file_path} (${row.fail_count}x)`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (errorTypes.length > 0) {
|
|
98
|
+
console.log("\nError Types:");
|
|
99
|
+
for (const row of errorTypes) {
|
|
100
|
+
console.log(` ${row.error_type.padEnd(30)} ${row.count}x`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (trend.length > 0) {
|
|
104
|
+
console.log("\nDaily Trend:");
|
|
105
|
+
for (const row of trend) {
|
|
106
|
+
const failRate = row.total > 0 ? Math.round(row.failures / row.total * 100) : 0;
|
|
107
|
+
console.log(
|
|
108
|
+
` ${row.day} ${row.total} edits, ${row.failures} failures (${failRate}%)`
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
console.log("");
|
|
113
|
+
});
|
|
114
|
+
return stats;
|
|
115
|
+
}
|
|
116
|
+
export {
|
|
117
|
+
createStatsCommand
|
|
118
|
+
};
|
package/dist/src/cli/index.js
CHANGED
|
@@ -51,6 +51,9 @@ import { createDiscoveryCommands } from "./commands/discovery.js";
|
|
|
51
51
|
import { createModelCommand } from "./commands/model.js";
|
|
52
52
|
import { registerSetupCommands } from "./commands/setup.js";
|
|
53
53
|
import { createPingCommand } from "./commands/ping.js";
|
|
54
|
+
import { createAuditCommand } from "./commands/audit.js";
|
|
55
|
+
import { createStatsCommand } from "./commands/stats.js";
|
|
56
|
+
import { createBenchCommand } from "./commands/bench.js";
|
|
54
57
|
import chalk from "chalk";
|
|
55
58
|
import * as fs from "fs";
|
|
56
59
|
import * as path from "path";
|
|
@@ -551,6 +554,9 @@ program.addCommand(createPingCommand());
|
|
|
551
554
|
program.addCommand(createRetrievalCommands());
|
|
552
555
|
program.addCommand(createDiscoveryCommands());
|
|
553
556
|
program.addCommand(createModelCommand());
|
|
557
|
+
program.addCommand(createAuditCommand());
|
|
558
|
+
program.addCommand(createStatsCommand());
|
|
559
|
+
program.addCommand(createBenchCommand());
|
|
554
560
|
registerSetupCommands(program);
|
|
555
561
|
program.command("mm-spike").description(
|
|
556
562
|
"Run multi-agent planning/implementation spike (planner/implementer/critic)"
|
|
@@ -19,6 +19,8 @@ function isFeatureEnabled(feature) {
|
|
|
19
19
|
return process.env["STACKMEMORY_SKILLS"] === "true" || process.env["STACKMEMORY_SKILLS"] === "1";
|
|
20
20
|
case "ralph":
|
|
21
21
|
return process.env["STACKMEMORY_RALPH"] !== "false";
|
|
22
|
+
case "multiProvider":
|
|
23
|
+
return process.env["STACKMEMORY_MULTI_PROVIDER"] === "true" || process.env["STACKMEMORY_MULTI_PROVIDER"] === "1";
|
|
22
24
|
default:
|
|
23
25
|
return false;
|
|
24
26
|
}
|
|
@@ -30,7 +32,8 @@ function getFeatureFlags() {
|
|
|
30
32
|
chromadb: isFeatureEnabled("chromadb"),
|
|
31
33
|
aiSummaries: isFeatureEnabled("aiSummaries"),
|
|
32
34
|
skills: isFeatureEnabled("skills"),
|
|
33
|
-
ralph: isFeatureEnabled("ralph")
|
|
35
|
+
ralph: isFeatureEnabled("ralph"),
|
|
36
|
+
multiProvider: isFeatureEnabled("multiProvider")
|
|
34
37
|
};
|
|
35
38
|
}
|
|
36
39
|
function logFeatureStatus() {
|
|
@@ -53,6 +56,9 @@ function logFeatureStatus() {
|
|
|
53
56
|
console.log(
|
|
54
57
|
` Ralph: ${flags.ralph ? "enabled" : "disabled (set STACKMEMORY_RALPH=true)"}`
|
|
55
58
|
);
|
|
59
|
+
console.log(
|
|
60
|
+
` MultiProvider: ${flags.multiProvider ? "enabled" : "disabled (set STACKMEMORY_MULTI_PROVIDER=true)"}`
|
|
61
|
+
);
|
|
56
62
|
}
|
|
57
63
|
}
|
|
58
64
|
export {
|
|
@@ -5,22 +5,41 @@ const __dirname = __pathDirname(__filename);
|
|
|
5
5
|
import * as fs from "fs/promises";
|
|
6
6
|
import * as path from "path";
|
|
7
7
|
import { logger } from "../monitoring/logger.js";
|
|
8
|
+
import {
|
|
9
|
+
getModelTokenLimit
|
|
10
|
+
} from "../models/model-router.js";
|
|
8
11
|
class CompactionHandler {
|
|
9
12
|
frameManager;
|
|
10
13
|
metrics;
|
|
11
14
|
tokenAccumulator = 0;
|
|
12
15
|
preservedAnchors = /* @__PURE__ */ new Map();
|
|
13
|
-
|
|
16
|
+
modelTokenLimit;
|
|
17
|
+
/**
|
|
18
|
+
* @param frameManager - Frame manager instance
|
|
19
|
+
* @param modelOrLimit - Model name string (looked up in MODEL_TOKEN_LIMITS)
|
|
20
|
+
* or explicit numeric token limit.
|
|
21
|
+
* Defaults to DEFAULT_MODEL_TOKEN_LIMIT (200K).
|
|
22
|
+
*
|
|
23
|
+
* Thresholds are derived from the model limit:
|
|
24
|
+
* warning = 90% of limit
|
|
25
|
+
* critical = 95% of limit (auto-compact trigger)
|
|
26
|
+
*/
|
|
27
|
+
constructor(frameManager, modelOrLimit) {
|
|
14
28
|
this.frameManager = frameManager;
|
|
29
|
+
this.modelTokenLimit = typeof modelOrLimit === "number" ? modelOrLimit : getModelTokenLimit(modelOrLimit ?? void 0);
|
|
15
30
|
this.metrics = {
|
|
16
31
|
estimatedTokens: 0,
|
|
17
|
-
warningThreshold:
|
|
18
|
-
|
|
19
|
-
criticalThreshold: 17e4,
|
|
20
|
-
// 170K tokens
|
|
32
|
+
warningThreshold: Math.floor(this.modelTokenLimit * 0.9),
|
|
33
|
+
criticalThreshold: Math.floor(this.modelTokenLimit * 0.95),
|
|
21
34
|
anchorsPreserved: 0
|
|
22
35
|
};
|
|
23
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* Get the resolved model token limit
|
|
39
|
+
*/
|
|
40
|
+
getModelTokenLimit() {
|
|
41
|
+
return this.modelTokenLimit;
|
|
42
|
+
}
|
|
24
43
|
/**
|
|
25
44
|
* Track token usage from a message
|
|
26
45
|
*/
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { fileURLToPath as __fileURLToPath } from 'url';
|
|
2
|
+
import { dirname as __pathDirname } from 'path';
|
|
3
|
+
const __filename = __fileURLToPath(import.meta.url);
|
|
4
|
+
const __dirname = __pathDirname(__filename);
|
|
5
|
+
import {
|
|
6
|
+
GPTAdapter
|
|
7
|
+
} from "./provider-adapter.js";
|
|
8
|
+
class CerebrasAdapter extends GPTAdapter {
|
|
9
|
+
id = "cerebras";
|
|
10
|
+
name = "Cerebras";
|
|
11
|
+
version = "1.0.0";
|
|
12
|
+
extensions = {};
|
|
13
|
+
constructor(config) {
|
|
14
|
+
super({
|
|
15
|
+
apiKey: config.apiKey,
|
|
16
|
+
baseUrl: config.baseUrl || "https://api.cerebras.ai/v1"
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
supportsExtension() {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
async listModels() {
|
|
23
|
+
return ["llama-4-scout-17b-16e-instruct", "llama3.1-8b", "llama3.1-70b"];
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export {
|
|
27
|
+
CerebrasAdapter
|
|
28
|
+
};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { fileURLToPath as __fileURLToPath } from 'url';
|
|
2
|
+
import { dirname as __pathDirname } from 'path';
|
|
3
|
+
const __filename = __fileURLToPath(import.meta.url);
|
|
4
|
+
const __dirname = __pathDirname(__filename);
|
|
5
|
+
import {
|
|
6
|
+
GPTAdapter
|
|
7
|
+
} from "./provider-adapter.js";
|
|
8
|
+
class DeepInfraAdapter extends GPTAdapter {
|
|
9
|
+
id = "deepinfra";
|
|
10
|
+
name = "DeepInfra";
|
|
11
|
+
version = "1.0.0";
|
|
12
|
+
extensions = {};
|
|
13
|
+
constructor(config) {
|
|
14
|
+
super({
|
|
15
|
+
apiKey: config.apiKey,
|
|
16
|
+
baseUrl: config.baseUrl || "https://api.deepinfra.com/v1/openai"
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
supportsExtension() {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
async listModels() {
|
|
23
|
+
return [
|
|
24
|
+
"THUDM/glm-4-9b-chat",
|
|
25
|
+
"meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
26
|
+
"meta-llama/Meta-Llama-3.1-70B-Instruct"
|
|
27
|
+
];
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export {
|
|
31
|
+
DeepInfraAdapter
|
|
32
|
+
};
|