@hawon/nexus 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -38
- package/dist/cli/index.js +76 -145
- package/dist/index.js +15 -26
- package/dist/mcp/server.js +61 -32
- package/package.json +2 -1
- package/scripts/auto-skill.sh +54 -0
- package/scripts/auto-sync.sh +11 -0
- package/scripts/benchmark.ts +444 -0
- package/scripts/scan-tool-result.sh +46 -0
- package/src/cli/index.ts +79 -172
- package/src/index.ts +17 -29
- package/src/mcp/server.ts +67 -41
- package/src/memory-engine/index.ts +4 -6
- package/src/memory-engine/nexus-memory.test.ts +437 -0
- package/src/memory-engine/nexus-memory.ts +631 -0
- package/src/memory-engine/semantic.ts +380 -0
- package/src/parser/parse.ts +1 -21
- package/src/promptguard/advanced-rules.ts +129 -12
- package/src/promptguard/entropy.ts +21 -2
- package/src/promptguard/evolution/auto-update.ts +16 -6
- package/src/promptguard/multilingual-rules.ts +68 -0
- package/src/promptguard/rules.ts +87 -2
- package/src/promptguard/scanner.test.ts +262 -0
- package/src/promptguard/scanner.ts +1 -1
- package/src/promptguard/semantic.ts +19 -4
- package/src/promptguard/token-analysis.ts +17 -5
- package/src/review/analyzer.test.ts +279 -0
- package/src/review/analyzer.ts +112 -28
- package/src/shared/stop-words.ts +21 -0
- package/src/skills/index.ts +11 -27
- package/src/skills/memory-skill-engine.ts +1044 -0
- package/src/testing/health-check.ts +19 -2
- package/src/cost/index.ts +0 -3
- package/src/cost/tracker.ts +0 -290
- package/src/cost/types.ts +0 -34
- package/src/memory-engine/compressor.ts +0 -97
- package/src/memory-engine/context-window.ts +0 -113
- package/src/memory-engine/store.ts +0 -371
- package/src/memory-engine/types.ts +0 -32
- package/src/skills/context-engine.ts +0 -863
- package/src/skills/extractor.ts +0 -224
- package/src/skills/global-context.ts +0 -726
- package/src/skills/library.ts +0 -189
- package/src/skills/pattern-engine.ts +0 -712
- package/src/skills/render-evolved.ts +0 -160
- package/src/skills/skill-reconciler.ts +0 -703
- package/src/skills/smart-extractor.ts +0 -843
- package/src/skills/types.ts +0 -18
- package/src/skills/wisdom-extractor.ts +0 -737
- package/src/superdev-evolution/index.ts +0 -3
- package/src/superdev-evolution/skill-manager.ts +0 -266
- package/src/superdev-evolution/types.ts +0 -20
|
@@ -101,6 +101,22 @@ function extractMockedFunctions(content: string): string[] {
|
|
|
101
101
|
return mocks;
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
function countIndividualTests(content: string, ext: string): number {
|
|
105
|
+
let count = 0;
|
|
106
|
+
if (ext === ".ts" || ext === ".js" || ext === ".tsx" || ext === ".jsx") {
|
|
107
|
+
// Count it() and test() calls, but NOT describe() — those are suites, not tests
|
|
108
|
+
const re = /(?:^|[^.\w])(?:it|test)\s*\(\s*['"`]/gm;
|
|
109
|
+
while (re.exec(content) !== null) count++;
|
|
110
|
+
} else if (ext === ".py") {
|
|
111
|
+
const re = /def\s+test_\w+\s*\(/g;
|
|
112
|
+
while (re.exec(content) !== null) count++;
|
|
113
|
+
} else if (ext === ".go") {
|
|
114
|
+
const re = /func\s+Test\w+\s*\(/g;
|
|
115
|
+
while (re.exec(content) !== null) count++;
|
|
116
|
+
}
|
|
117
|
+
return count;
|
|
118
|
+
}
|
|
119
|
+
|
|
104
120
|
function extractTestNames(content: string): string[] {
|
|
105
121
|
const names: string[] = [];
|
|
106
122
|
// it("...", ...) / test("...", ...) / describe("...", ...)
|
|
@@ -127,7 +143,8 @@ function extractFunctionNames(content: string, ext: string): string[] {
|
|
|
127
143
|
let re: RegExp;
|
|
128
144
|
|
|
129
145
|
if (ext === ".ts" || ext === ".js" || ext === ".tsx" || ext === ".jsx") {
|
|
130
|
-
|
|
146
|
+
// Match: function declarations, const/let/var assignments, and object method shorthand
|
|
147
|
+
re = /(?:export\s+)?(?:async\s+)?function\s+(\w+)|(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?\(?|^\s+(\w+)\s*(?:<[^>]*>)?\s*\([^)]*\)\s*(?::\s*[^{]+)?\s*\{/gm;
|
|
131
148
|
} else if (ext === ".py") {
|
|
132
149
|
re = /def\s+(\w+)\s*\(/g;
|
|
133
150
|
} else if (ext === ".go") {
|
|
@@ -138,7 +155,7 @@ function extractFunctionNames(content: string, ext: string): string[] {
|
|
|
138
155
|
|
|
139
156
|
let m;
|
|
140
157
|
while ((m = re.exec(content)) !== null) {
|
|
141
|
-
names.push(m[1] || m[2]);
|
|
158
|
+
names.push(m[1] || m[2] || m[3]);
|
|
142
159
|
}
|
|
143
160
|
return names.filter(Boolean);
|
|
144
161
|
}
|
package/src/cost/index.ts
DELETED
package/src/cost/tracker.ts
DELETED
|
@@ -1,290 +0,0 @@
|
|
|
1
|
-
import { readFileSync, appendFileSync, mkdirSync, existsSync } from "node:fs";
|
|
2
|
-
import { join, dirname } from "node:path";
|
|
3
|
-
import type { CostEntry, CostReport, CostAlert, BudgetConfig } from "./types.js";
|
|
4
|
-
|
|
5
|
-
// Pricing per million tokens: [input, output] in USD
|
|
6
|
-
const MODEL_PRICING: Record<string, [number, number]> = {
|
|
7
|
-
// OpenAI
|
|
8
|
-
"gpt-4o": [2.50, 10.00],
|
|
9
|
-
"gpt-4o-mini": [0.15, 0.60],
|
|
10
|
-
"gpt-4-turbo": [10.00, 30.00],
|
|
11
|
-
"gpt-4": [30.00, 60.00],
|
|
12
|
-
"gpt-3.5-turbo": [0.50, 1.50],
|
|
13
|
-
"o1": [15.00, 60.00],
|
|
14
|
-
"o1-mini": [3.00, 12.00],
|
|
15
|
-
"o1-pro": [150.00, 600.00],
|
|
16
|
-
"o3": [10.00, 40.00],
|
|
17
|
-
"o3-mini": [1.10, 4.40],
|
|
18
|
-
"o4-mini": [1.10, 4.40],
|
|
19
|
-
// Anthropic
|
|
20
|
-
"claude-opus-4": [15.00, 75.00],
|
|
21
|
-
"claude-sonnet-4": [3.00, 15.00],
|
|
22
|
-
"claude-3.5-sonnet": [3.00, 15.00],
|
|
23
|
-
"claude-3.5-haiku": [0.80, 4.00],
|
|
24
|
-
"claude-3-opus": [15.00, 75.00],
|
|
25
|
-
"claude-3-sonnet": [3.00, 15.00],
|
|
26
|
-
"claude-3-haiku": [0.25, 1.25],
|
|
27
|
-
// Google
|
|
28
|
-
"gemini-2.5-pro": [1.25, 10.00],
|
|
29
|
-
"gemini-2.5-flash": [0.15, 0.60],
|
|
30
|
-
"gemini-2.0-flash": [0.10, 0.40],
|
|
31
|
-
"gemini-1.5-pro": [1.25, 5.00],
|
|
32
|
-
"gemini-1.5-flash": [0.075, 0.30],
|
|
33
|
-
};
|
|
34
|
-
|
|
35
|
-
function inferProvider(model: string): string {
|
|
36
|
-
if (model.startsWith("gpt-") || model.startsWith("o1") || model.startsWith("o3") || model.startsWith("o4")) return "openai";
|
|
37
|
-
if (model.startsWith("claude-")) return "anthropic";
|
|
38
|
-
if (model.startsWith("gemini-")) return "google";
|
|
39
|
-
return "unknown";
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
function computeCost(model: string, inputTokens: number, outputTokens: number): number {
|
|
43
|
-
const pricing = MODEL_PRICING[model];
|
|
44
|
-
if (!pricing) return 0;
|
|
45
|
-
const [inputPerM, outputPerM] = pricing;
|
|
46
|
-
return (inputTokens / 1_000_000) * inputPerM + (outputTokens / 1_000_000) * outputPerM;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
function todayKey(): string {
|
|
50
|
-
return new Date().toISOString().slice(0, 10);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
function daysBetween(a: Date, b: Date): number {
|
|
54
|
-
return Math.abs(a.getTime() - b.getTime()) / (24 * 60 * 60 * 1000);
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export interface CostTracker {
|
|
58
|
-
record(entry: Omit<CostEntry, "timestamp">): CostAlert[];
|
|
59
|
-
getReport(days?: number): CostReport;
|
|
60
|
-
getProjectedCost(): number;
|
|
61
|
-
checkBudget(): CostAlert[];
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export function createCostTracker(dataDir: string, budget?: BudgetConfig): CostTracker {
|
|
65
|
-
const logFile = join(dataDir, "cost-log.jsonl");
|
|
66
|
-
|
|
67
|
-
// Ensure data directory exists
|
|
68
|
-
if (!existsSync(dataDir)) {
|
|
69
|
-
mkdirSync(dataDir, { recursive: true });
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function readEntries(): CostEntry[] {
|
|
73
|
-
if (!existsSync(logFile)) return [];
|
|
74
|
-
const content = readFileSync(logFile, "utf-8").trim();
|
|
75
|
-
if (!content) return [];
|
|
76
|
-
return content
|
|
77
|
-
.split("\n")
|
|
78
|
-
.filter(Boolean)
|
|
79
|
-
.map((line) => {
|
|
80
|
-
try {
|
|
81
|
-
return JSON.parse(line) as CostEntry;
|
|
82
|
-
} catch {
|
|
83
|
-
return null;
|
|
84
|
-
}
|
|
85
|
-
})
|
|
86
|
-
.filter((e): e is CostEntry => e !== null);
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
function appendEntry(entry: CostEntry): void {
|
|
90
|
-
const dir = dirname(logFile);
|
|
91
|
-
if (!existsSync(dir)) {
|
|
92
|
-
mkdirSync(dir, { recursive: true });
|
|
93
|
-
}
|
|
94
|
-
appendFileSync(logFile, JSON.stringify(entry) + "\n", "utf-8");
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
function filterByDays(entries: CostEntry[], days: number): CostEntry[] {
|
|
98
|
-
const cutoff = new Date();
|
|
99
|
-
cutoff.setDate(cutoff.getDate() - days);
|
|
100
|
-
const cutoffStr = cutoff.toISOString();
|
|
101
|
-
return entries.filter((e) => e.timestamp >= cutoffStr);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
function buildReport(entries: CostEntry[]): CostReport {
|
|
105
|
-
const byProvider: Record<string, number> = {};
|
|
106
|
-
const byModel: Record<string, number> = {};
|
|
107
|
-
const byDay: Record<string, number> = {};
|
|
108
|
-
let totalCost = 0;
|
|
109
|
-
|
|
110
|
-
for (const e of entries) {
|
|
111
|
-
totalCost += e.cost;
|
|
112
|
-
byProvider[e.provider] = (byProvider[e.provider] ?? 0) + e.cost;
|
|
113
|
-
byModel[e.model] = (byModel[e.model] ?? 0) + e.cost;
|
|
114
|
-
const day = e.timestamp.slice(0, 10);
|
|
115
|
-
byDay[day] = (byDay[day] ?? 0) + e.cost;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
const totalRequests = entries.length;
|
|
119
|
-
const averageCostPerRequest = totalRequests > 0 ? totalCost / totalRequests : 0;
|
|
120
|
-
|
|
121
|
-
// Project monthly cost based on daily average
|
|
122
|
-
const dayKeys = Object.keys(byDay);
|
|
123
|
-
let projectedMonthlyCost = 0;
|
|
124
|
-
if (dayKeys.length > 0) {
|
|
125
|
-
const firstDay = new Date(dayKeys.sort()[0]);
|
|
126
|
-
const spanDays = Math.max(1, daysBetween(new Date(), firstDay));
|
|
127
|
-
const dailyAvg = totalCost / spanDays;
|
|
128
|
-
projectedMonthlyCost = dailyAvg * 30;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
const alerts = checkBudgetInternal(entries, totalCost, byDay);
|
|
132
|
-
|
|
133
|
-
return {
|
|
134
|
-
totalCost,
|
|
135
|
-
byProvider,
|
|
136
|
-
byModel,
|
|
137
|
-
byDay,
|
|
138
|
-
averageCostPerRequest,
|
|
139
|
-
totalRequests,
|
|
140
|
-
projectedMonthlyCost,
|
|
141
|
-
alerts,
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
function checkBudgetInternal(
|
|
146
|
-
entries: CostEntry[],
|
|
147
|
-
totalCost?: number,
|
|
148
|
-
byDay?: Record<string, number>
|
|
149
|
-
): CostAlert[] {
|
|
150
|
-
if (!budget) return [];
|
|
151
|
-
|
|
152
|
-
const alerts: CostAlert[] = [];
|
|
153
|
-
const all = entries;
|
|
154
|
-
const today = todayKey();
|
|
155
|
-
const threshold = budget.alertThreshold ?? 0.8;
|
|
156
|
-
|
|
157
|
-
// Daily limit
|
|
158
|
-
if (budget.dailyLimit !== undefined) {
|
|
159
|
-
let todayCost = 0;
|
|
160
|
-
if (byDay && byDay[today] !== undefined) {
|
|
161
|
-
todayCost = byDay[today];
|
|
162
|
-
} else {
|
|
163
|
-
todayCost = all
|
|
164
|
-
.filter((e) => e.timestamp.startsWith(today))
|
|
165
|
-
.reduce((sum, e) => sum + e.cost, 0);
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
if (todayCost > budget.dailyLimit) {
|
|
169
|
-
alerts.push({
|
|
170
|
-
type: "budget_exceeded",
|
|
171
|
-
message: `Daily budget exceeded: $${todayCost.toFixed(4)} / $${budget.dailyLimit.toFixed(2)}`,
|
|
172
|
-
severity: "critical",
|
|
173
|
-
});
|
|
174
|
-
} else if (todayCost > budget.dailyLimit * threshold) {
|
|
175
|
-
alerts.push({
|
|
176
|
-
type: "budget_exceeded",
|
|
177
|
-
message: `Approaching daily budget: $${todayCost.toFixed(4)} / $${budget.dailyLimit.toFixed(2)} (${Math.round((todayCost / budget.dailyLimit) * 100)}%)`,
|
|
178
|
-
severity: "warning",
|
|
179
|
-
});
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
// Monthly limit
|
|
184
|
-
if (budget.monthlyLimit !== undefined) {
|
|
185
|
-
const monthPrefix = today.slice(0, 7);
|
|
186
|
-
const monthCost = all
|
|
187
|
-
.filter((e) => e.timestamp.startsWith(monthPrefix))
|
|
188
|
-
.reduce((sum, e) => sum + e.cost, 0);
|
|
189
|
-
|
|
190
|
-
if (monthCost > budget.monthlyLimit) {
|
|
191
|
-
alerts.push({
|
|
192
|
-
type: "budget_exceeded",
|
|
193
|
-
message: `Monthly budget exceeded: $${monthCost.toFixed(4)} / $${budget.monthlyLimit.toFixed(2)}`,
|
|
194
|
-
severity: "critical",
|
|
195
|
-
});
|
|
196
|
-
} else if (monthCost > budget.monthlyLimit * threshold) {
|
|
197
|
-
alerts.push({
|
|
198
|
-
type: "budget_exceeded",
|
|
199
|
-
message: `Approaching monthly budget: $${monthCost.toFixed(4)} / $${budget.monthlyLimit.toFixed(2)} (${Math.round((monthCost / budget.monthlyLimit) * 100)}%)`,
|
|
200
|
-
severity: "warning",
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
// Spike detection: if today's cost is 3x the average daily cost
|
|
206
|
-
if (all.length > 0) {
|
|
207
|
-
const dayMap: Record<string, number> = byDay ?? {};
|
|
208
|
-
if (!byDay) {
|
|
209
|
-
for (const e of all) {
|
|
210
|
-
const d = e.timestamp.slice(0, 10);
|
|
211
|
-
dayMap[d] = (dayMap[d] ?? 0) + e.cost;
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
const dayValues = Object.entries(dayMap)
|
|
215
|
-
.filter(([d]) => d !== today)
|
|
216
|
-
.map(([, v]) => v);
|
|
217
|
-
if (dayValues.length > 0) {
|
|
218
|
-
const avgDaily = dayValues.reduce((a, b) => a + b, 0) / dayValues.length;
|
|
219
|
-
const todayCost = dayMap[today] ?? 0;
|
|
220
|
-
if (avgDaily > 0 && todayCost > avgDaily * 3) {
|
|
221
|
-
alerts.push({
|
|
222
|
-
type: "spike_detected",
|
|
223
|
-
message: `Cost spike detected: today's cost $${todayCost.toFixed(4)} is ${(todayCost / avgDaily).toFixed(1)}x the daily average of $${avgDaily.toFixed(4)}`,
|
|
224
|
-
severity: "warning",
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
return alerts;
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
return {
|
|
234
|
-
record(entry: Omit<CostEntry, "timestamp">): CostAlert[] {
|
|
235
|
-
// Infer provider from model name if not provided
|
|
236
|
-
const provider = entry.provider || inferProvider(entry.model);
|
|
237
|
-
|
|
238
|
-
// Auto-compute cost if not provided or zero, and model is known
|
|
239
|
-
let cost = entry.cost;
|
|
240
|
-
if (cost === 0 && MODEL_PRICING[entry.model]) {
|
|
241
|
-
cost = computeCost(entry.model, entry.inputTokens, entry.outputTokens);
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
const fullEntry: CostEntry = {
|
|
245
|
-
...entry,
|
|
246
|
-
provider,
|
|
247
|
-
cost,
|
|
248
|
-
timestamp: new Date().toISOString(),
|
|
249
|
-
};
|
|
250
|
-
|
|
251
|
-
// Per-request limit check
|
|
252
|
-
const alerts: CostAlert[] = [];
|
|
253
|
-
if (budget?.perRequestLimit !== undefined && cost > budget.perRequestLimit) {
|
|
254
|
-
alerts.push({
|
|
255
|
-
type: "budget_exceeded",
|
|
256
|
-
message: `Single request cost $${cost.toFixed(4)} exceeds per-request limit of $${budget.perRequestLimit.toFixed(4)}`,
|
|
257
|
-
severity: "critical",
|
|
258
|
-
});
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
appendEntry(fullEntry);
|
|
262
|
-
|
|
263
|
-
// Check overall budget after recording
|
|
264
|
-
const entries = readEntries();
|
|
265
|
-
alerts.push(...checkBudgetInternal(entries));
|
|
266
|
-
|
|
267
|
-
return alerts;
|
|
268
|
-
},
|
|
269
|
-
|
|
270
|
-
getReport(days = 30): CostReport {
|
|
271
|
-
const entries = filterByDays(readEntries(), days);
|
|
272
|
-
return buildReport(entries);
|
|
273
|
-
},
|
|
274
|
-
|
|
275
|
-
getProjectedCost(): number {
|
|
276
|
-
const entries = readEntries();
|
|
277
|
-
if (entries.length === 0) return 0;
|
|
278
|
-
|
|
279
|
-
const sorted = entries.map((e) => e.timestamp).sort();
|
|
280
|
-
const first = new Date(sorted[0]);
|
|
281
|
-
const spanDays = Math.max(1, daysBetween(new Date(), first));
|
|
282
|
-
const totalCost = entries.reduce((sum, e) => sum + e.cost, 0);
|
|
283
|
-
return (totalCost / spanDays) * 30;
|
|
284
|
-
},
|
|
285
|
-
|
|
286
|
-
checkBudget(): CostAlert[] {
|
|
287
|
-
return checkBudgetInternal(readEntries());
|
|
288
|
-
},
|
|
289
|
-
};
|
|
290
|
-
}
|
package/src/cost/types.ts
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
export type CostEntry = {
|
|
2
|
-
timestamp: string;
|
|
3
|
-
provider: string; // "openai" | "anthropic" | "google" | etc
|
|
4
|
-
model: string;
|
|
5
|
-
inputTokens: number;
|
|
6
|
-
outputTokens: number;
|
|
7
|
-
cost: number; // USD
|
|
8
|
-
endpoint?: string;
|
|
9
|
-
sessionId?: string;
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
export type CostReport = {
|
|
13
|
-
totalCost: number;
|
|
14
|
-
byProvider: Record<string, number>;
|
|
15
|
-
byModel: Record<string, number>;
|
|
16
|
-
byDay: Record<string, number>;
|
|
17
|
-
averageCostPerRequest: number;
|
|
18
|
-
totalRequests: number;
|
|
19
|
-
projectedMonthlyCost: number;
|
|
20
|
-
alerts: CostAlert[];
|
|
21
|
-
};
|
|
22
|
-
|
|
23
|
-
export type CostAlert = {
|
|
24
|
-
type: "budget_exceeded" | "spike_detected" | "unusual_model" | "rate_limit_approaching";
|
|
25
|
-
message: string;
|
|
26
|
-
severity: "critical" | "warning" | "info";
|
|
27
|
-
};
|
|
28
|
-
|
|
29
|
-
export type BudgetConfig = {
|
|
30
|
-
dailyLimit?: number;
|
|
31
|
-
monthlyLimit?: number;
|
|
32
|
-
perRequestLimit?: number;
|
|
33
|
-
alertThreshold?: number; // 0-1, percentage of budget
|
|
34
|
-
};
|
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
import type { MemoryEntry } from "./types.js";
|
|
2
|
-
|
|
3
|
-
const STOP_WORDS = new Set([
|
|
4
|
-
"the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
|
|
5
|
-
"have", "has", "had", "do", "does", "did", "will", "would", "could",
|
|
6
|
-
"should", "may", "might", "shall", "can", "need", "dare", "ought",
|
|
7
|
-
"used", "to", "of", "in", "for", "on", "with", "at", "by", "from",
|
|
8
|
-
"as", "into", "through", "during", "before", "after", "above", "below",
|
|
9
|
-
"between", "out", "off", "over", "under", "again", "further", "then",
|
|
10
|
-
"once", "here", "there", "when", "where", "why", "how", "all", "both",
|
|
11
|
-
"each", "few", "more", "most", "other", "some", "such", "no", "nor",
|
|
12
|
-
"not", "only", "own", "same", "so", "than", "too", "very", "just",
|
|
13
|
-
"because", "but", "and", "or", "if", "while", "that", "this", "it",
|
|
14
|
-
"its", "i", "me", "my", "we", "our", "you", "your", "he", "him",
|
|
15
|
-
"his", "she", "her", "they", "them", "their", "what", "which", "who",
|
|
16
|
-
]);
|
|
17
|
-
|
|
18
|
-
function tokenize(text: string): string[] {
|
|
19
|
-
return text
|
|
20
|
-
.toLowerCase()
|
|
21
|
-
.replace(/[^a-z0-9\s]/g, " ")
|
|
22
|
-
.split(/\s+/)
|
|
23
|
-
.filter((w) => w.length > 1 && !STOP_WORDS.has(w));
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
function scoreLineByKeywordDensity(line: string, globalFreq: Map<string, number>): number {
|
|
27
|
-
const tokens = tokenize(line);
|
|
28
|
-
if (tokens.length === 0) return 0;
|
|
29
|
-
let score = 0;
|
|
30
|
-
for (const token of tokens) {
|
|
31
|
-
score += globalFreq.get(token) ?? 0;
|
|
32
|
-
}
|
|
33
|
-
return score / tokens.length;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
export function extractKeyLines(text: string, maxLines: number): string[] {
|
|
37
|
-
const lines = text.split("\n").filter((l) => l.trim().length > 0);
|
|
38
|
-
if (lines.length <= maxLines) return lines;
|
|
39
|
-
|
|
40
|
-
const allTokens = tokenize(text);
|
|
41
|
-
const freq = new Map<string, number>();
|
|
42
|
-
for (const t of allTokens) {
|
|
43
|
-
freq.set(t, (freq.get(t) ?? 0) + 1);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
const scored = lines.map((line, idx) => ({
|
|
47
|
-
line,
|
|
48
|
-
idx,
|
|
49
|
-
score: scoreLineByKeywordDensity(line, freq),
|
|
50
|
-
}));
|
|
51
|
-
|
|
52
|
-
scored.sort((a, b) => b.score - a.score);
|
|
53
|
-
const selected = scored.slice(0, maxLines);
|
|
54
|
-
selected.sort((a, b) => a.idx - b.idx);
|
|
55
|
-
|
|
56
|
-
return selected.map((s) => s.line);
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
export function compressEntry(entry: MemoryEntry): MemoryEntry {
|
|
60
|
-
if (entry.compressed) return entry;
|
|
61
|
-
|
|
62
|
-
const lines = entry.content.split("\n").filter((l) => l.trim().length > 0);
|
|
63
|
-
let summary: string;
|
|
64
|
-
|
|
65
|
-
if (lines.length <= 3) {
|
|
66
|
-
summary = entry.content;
|
|
67
|
-
} else {
|
|
68
|
-
const firstSentence = extractFirstSentence(entry.content);
|
|
69
|
-
const keyLines = extractKeyLines(entry.content, Math.max(3, Math.ceil(lines.length * 0.2)));
|
|
70
|
-
const parts = [firstSentence];
|
|
71
|
-
for (const kl of keyLines) {
|
|
72
|
-
if (kl !== firstSentence && !parts.includes(kl)) {
|
|
73
|
-
parts.push(kl);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
summary = parts.join("\n");
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
return {
|
|
80
|
-
...entry,
|
|
81
|
-
summary,
|
|
82
|
-
content: summary,
|
|
83
|
-
compressed: true,
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function extractFirstSentence(text: string): string {
|
|
88
|
-
const match = text.match(/^[^\n]*?[.!?](?:\s|$)/);
|
|
89
|
-
if (match) return match[0].trim();
|
|
90
|
-
const firstLine = text.split("\n")[0] ?? "";
|
|
91
|
-
return firstLine.slice(0, 200);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
export function estimateSizeBytes(entry: MemoryEntry): number {
|
|
95
|
-
const json = JSON.stringify(entry);
|
|
96
|
-
return Buffer.byteLength(json, "utf-8");
|
|
97
|
-
}
|
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
import type { MemoryStore } from "./store.js";
|
|
2
|
-
import type { MemoryTier } from "./types.js";
|
|
3
|
-
|
|
4
|
-
export type ContextWindowConfig = {
|
|
5
|
-
maxTokens: number;
|
|
6
|
-
store: MemoryStore;
|
|
7
|
-
};
|
|
8
|
-
|
|
9
|
-
export type ContextWindow = {
|
|
10
|
-
load(query: string): string;
|
|
11
|
-
save(content: string, tags: string[]): void;
|
|
12
|
-
summarizeAndArchive(): void;
|
|
13
|
-
getContextSize(): number;
|
|
14
|
-
};
|
|
15
|
-
|
|
16
|
-
const TIER_LABELS: Record<MemoryTier, string> = {
|
|
17
|
-
working: "Working Memory",
|
|
18
|
-
short_term: "Short-Term Memory",
|
|
19
|
-
long_term: "Long-Term Memory",
|
|
20
|
-
archive: "Archive",
|
|
21
|
-
};
|
|
22
|
-
|
|
23
|
-
function estimateTokens(text: string): number {
|
|
24
|
-
return Math.ceil(text.length / 4);
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export function createContextWindow(config: ContextWindowConfig): ContextWindow {
|
|
28
|
-
const { maxTokens, store } = config;
|
|
29
|
-
let currentContext = "";
|
|
30
|
-
|
|
31
|
-
function formatEntries(entries: { tier: MemoryTier; content: string; tags: string[] }[]): string {
|
|
32
|
-
const sections = new Map<MemoryTier, string[]>();
|
|
33
|
-
|
|
34
|
-
for (const entry of entries) {
|
|
35
|
-
if (!sections.has(entry.tier)) {
|
|
36
|
-
sections.set(entry.tier, []);
|
|
37
|
-
}
|
|
38
|
-
sections.get(entry.tier)!.push(entry.content);
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
const parts: string[] = [];
|
|
42
|
-
const tierOrder: MemoryTier[] = ["working", "short_term", "long_term", "archive"];
|
|
43
|
-
for (const tier of tierOrder) {
|
|
44
|
-
const items = sections.get(tier);
|
|
45
|
-
if (!items || items.length === 0) continue;
|
|
46
|
-
parts.push(`=== ${TIER_LABELS[tier]} ===`);
|
|
47
|
-
for (const item of items) {
|
|
48
|
-
parts.push(item);
|
|
49
|
-
parts.push("---");
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
return parts.join("\n");
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
return {
|
|
57
|
-
load(query) {
|
|
58
|
-
const budgetTokens = maxTokens;
|
|
59
|
-
const results = store.search({ query, limit: 50 });
|
|
60
|
-
|
|
61
|
-
const selected: typeof results = [];
|
|
62
|
-
let tokenCount = 0;
|
|
63
|
-
|
|
64
|
-
for (const entry of results) {
|
|
65
|
-
const entryTokens = estimateTokens(entry.content);
|
|
66
|
-
if (tokenCount + entryTokens > budgetTokens) break;
|
|
67
|
-
selected.push(entry);
|
|
68
|
-
tokenCount += entryTokens;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
currentContext = formatEntries(selected);
|
|
72
|
-
return currentContext;
|
|
73
|
-
},
|
|
74
|
-
|
|
75
|
-
save(content, tags) {
|
|
76
|
-
store.add({
|
|
77
|
-
content,
|
|
78
|
-
tags,
|
|
79
|
-
tier: "working",
|
|
80
|
-
});
|
|
81
|
-
},
|
|
82
|
-
|
|
83
|
-
summarizeAndArchive() {
|
|
84
|
-
// Trigger pruning which auto-demotes stale entries across tiers
|
|
85
|
-
store.prune();
|
|
86
|
-
|
|
87
|
-
// Additionally, compress any working memory entries that have been accessed
|
|
88
|
-
// but are getting old relative to working memory expectations
|
|
89
|
-
const workingEntries = store.search({ query: "", tier: "working", limit: 100 });
|
|
90
|
-
for (const entry of workingEntries) {
|
|
91
|
-
const ageHours = (Date.now() - new Date(entry.accessedAt).getTime()) / (1000 * 60 * 60);
|
|
92
|
-
if (ageHours > 6 && entry.accessCount < 3) {
|
|
93
|
-
store.demote(entry.id);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
// Compress short-term entries that are aging
|
|
98
|
-
const shortTermEntries = store.search({ query: "", tier: "short_term", limit: 100 });
|
|
99
|
-
for (const entry of shortTermEntries) {
|
|
100
|
-
const ageDays = (Date.now() - new Date(entry.accessedAt).getTime()) / (1000 * 60 * 60 * 24);
|
|
101
|
-
if (ageDays > 3 && !entry.compressed) {
|
|
102
|
-
store.compress(entry.id);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
currentContext = "";
|
|
107
|
-
},
|
|
108
|
-
|
|
109
|
-
getContextSize() {
|
|
110
|
-
return estimateTokens(currentContext);
|
|
111
|
-
},
|
|
112
|
-
};
|
|
113
|
-
}
|