@grapine.ai/contextprune 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +426 -1
- package/dist/cli/commands/analyze.d.ts +2 -0
- package/dist/cli/commands/analyze.js +161 -0
- package/dist/cli/commands/compress.d.ts +2 -0
- package/dist/cli/commands/compress.js +65 -0
- package/dist/cli/commands/watch.d.ts +2 -0
- package/dist/cli/commands/watch.js +432 -0
- package/dist/cli/dashboard/index.html +720 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +19 -0
- package/dist/cli/labels.d.ts +4 -0
- package/dist/cli/labels.js +35 -0
- package/dist/cli/parse-input.d.ts +33 -0
- package/dist/cli/parse-input.js +191 -0
- package/dist/src/brief/index.d.ts +2 -0
- package/dist/src/brief/index.js +101 -0
- package/dist/src/classifier/confidence.d.ts +4 -0
- package/dist/src/classifier/confidence.js +23 -0
- package/dist/src/classifier/index.d.ts +11 -0
- package/dist/src/classifier/index.js +217 -0
- package/dist/src/classifier/patterns.d.ts +7 -0
- package/dist/src/classifier/patterns.js +81 -0
- package/dist/src/compression/engine.d.ts +23 -0
- package/dist/src/compression/engine.js +363 -0
- package/dist/src/index.d.ts +41 -0
- package/dist/src/index.js +120 -0
- package/dist/src/pipeline/index.d.ts +5 -0
- package/dist/src/pipeline/index.js +167 -0
- package/dist/src/scorer/index.d.ts +4 -0
- package/dist/src/scorer/index.js +136 -0
- package/dist/src/scorer/session-extractor.d.ts +2 -0
- package/dist/src/scorer/session-extractor.js +57 -0
- package/dist/src/strategy/selector.d.ts +3 -0
- package/dist/src/strategy/selector.js +158 -0
- package/dist/src/tokenizer/index.d.ts +18 -0
- package/dist/src/tokenizer/index.js +195 -0
- package/dist/src/types.d.ts +161 -0
- package/dist/src/types.js +5 -0
- package/dist/src/utils/index.d.ts +4 -0
- package/dist/src/utils/index.js +48 -0
- package/dist/src/validation/coherence.d.ts +3 -0
- package/dist/src/validation/coherence.js +87 -0
- package/license.md +14 -0
- package/package.json +77 -41
- package/screenshots/cp_dashboard_compression.jpg +0 -0
- package/screenshots/cp_dashboard_healthy.jpg +0 -0
- package/index.js +0 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
// cli/index.ts
|
|
4
|
+
// ContextPrune CLI entry point.
|
|
5
|
+
// Usage: npx contextprune <command> [options]
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
const commander_1 = require("commander");
|
|
8
|
+
const analyze_1 = require("./commands/analyze");
|
|
9
|
+
const compress_1 = require("./commands/compress");
|
|
10
|
+
const watch_1 = require("./commands/watch");
|
|
11
|
+
const program = new commander_1.Command();
|
|
12
|
+
program
|
|
13
|
+
.name('contextprune')
|
|
14
|
+
.description('Intelligent context window management for LLM applications')
|
|
15
|
+
.version('0.1.0');
|
|
16
|
+
program.addCommand((0, analyze_1.analyzeCommand)());
|
|
17
|
+
program.addCommand((0, compress_1.compressCommand)());
|
|
18
|
+
program.addCommand((0, watch_1.watchCommand)());
|
|
19
|
+
program.parse(process.argv);
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// cli/labels.ts
|
|
3
|
+
// Single source of truth for human-readable classification and strategy names.
|
|
4
|
+
// Used by the CLI (analyze command) and injected into the dashboard HTML by the watch server.
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.STRAT_LABELS = exports.CLS_LABELS = void 0;
|
|
7
|
+
exports.clsLabel = clsLabel;
|
|
8
|
+
exports.stratLabel = stratLabel;
|
|
9
|
+
exports.CLS_LABELS = {
|
|
10
|
+
SYSTEM_CONSTRAINT: 'System Prompt',
|
|
11
|
+
TASK_DEFINITION: 'Session Goal',
|
|
12
|
+
TOOL_OUTPUT_ACTIVE: 'Tool Result',
|
|
13
|
+
TOOL_OUTPUT_STALE: 'Outdated Tool Result',
|
|
14
|
+
ERROR_ACTIVE: 'Active Error',
|
|
15
|
+
ERROR_RESOLVED: 'Fixed Error',
|
|
16
|
+
REASONING_INTERMEDIATE: 'Chain of Thought',
|
|
17
|
+
DECISION_FINAL: 'Final Answer',
|
|
18
|
+
USER_CORRECTION: 'Your Override',
|
|
19
|
+
PROGRESS_MARKER: 'Status Update',
|
|
20
|
+
CONVERSATIONAL: 'Chat / Filler',
|
|
21
|
+
};
|
|
22
|
+
exports.STRAT_LABELS = {
|
|
23
|
+
PRESERVE: 'Keep',
|
|
24
|
+
DROP: 'Remove',
|
|
25
|
+
EXTRACT_RESULT: 'Trim to Key Output',
|
|
26
|
+
COLLAPSE_TO_MARKER: 'Collapse to 1 Line',
|
|
27
|
+
SUMMARIZE: 'Summarize',
|
|
28
|
+
DEDUPLICATE: 'Remove Duplicate',
|
|
29
|
+
};
|
|
30
|
+
function clsLabel(cls) {
|
|
31
|
+
return exports.CLS_LABELS[cls] ?? cls;
|
|
32
|
+
}
|
|
33
|
+
function stratLabel(strat) {
|
|
34
|
+
return exports.STRAT_LABELS[strat] ?? strat;
|
|
35
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { LLMMessage } from '../src/types';
|
|
2
|
+
export interface TurnCost {
|
|
3
|
+
messageIndex: number;
|
|
4
|
+
timestamp: string;
|
|
5
|
+
model: string;
|
|
6
|
+
inputTokens: number;
|
|
7
|
+
outputTokens: number;
|
|
8
|
+
cacheWriteTokens: number;
|
|
9
|
+
cacheReadTokens: number;
|
|
10
|
+
cost: number;
|
|
11
|
+
}
|
|
12
|
+
export interface SessionCost {
|
|
13
|
+
date: string;
|
|
14
|
+
label: string;
|
|
15
|
+
totalCost: number;
|
|
16
|
+
totalInputTokens: number;
|
|
17
|
+
totalOutputTokens: number;
|
|
18
|
+
turns: number;
|
|
19
|
+
}
|
|
20
|
+
export interface CostData {
|
|
21
|
+
totalCost: number;
|
|
22
|
+
currency: 'USD';
|
|
23
|
+
perTurn: TurnCost[];
|
|
24
|
+
sessions: SessionCost[];
|
|
25
|
+
}
|
|
26
|
+
export interface LoadResult {
|
|
27
|
+
messages: LLMMessage[];
|
|
28
|
+
actualInputTokens?: number;
|
|
29
|
+
costData?: CostData;
|
|
30
|
+
}
|
|
31
|
+
export declare function loadMessages(file: string): LLMMessage[];
|
|
32
|
+
export declare function loadMessagesWithUsage(file: string): LoadResult;
|
|
33
|
+
export declare function loadProjectCostHistory(projectDir: string, currentFile: string): CostData;
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// cli/parse-input.ts
|
|
3
|
+
// Shared file-loading logic for CLI commands.
|
|
4
|
+
// Supports .json (array) and .jsonl (Claude Code session transcripts or plain message-per-line).
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.loadMessages = loadMessages;
|
|
7
|
+
exports.loadMessagesWithUsage = loadMessagesWithUsage;
|
|
8
|
+
exports.loadProjectCostHistory = loadProjectCostHistory;
|
|
9
|
+
const fs_1 = require("fs");
|
|
10
|
+
const path_1 = require("path");
|
|
11
|
+
// ─── Cost pricing table (USD per million tokens) ──────────────────────────────
|
|
12
|
+
// Prices: https://www.anthropic.com/pricing
|
|
13
|
+
const PRICING = {
|
|
14
|
+
'claude-opus-4-6': { input: 15.00, output: 75.00, cacheWrite: 18.75, cacheRead: 1.50 },
|
|
15
|
+
'claude-opus-4-5': { input: 15.00, output: 75.00, cacheWrite: 18.75, cacheRead: 1.50 },
|
|
16
|
+
'claude-sonnet-4-6': { input: 3.00, output: 15.00, cacheWrite: 3.75, cacheRead: 0.30 },
|
|
17
|
+
'claude-sonnet-4-5': { input: 3.00, output: 15.00, cacheWrite: 3.75, cacheRead: 0.30 },
|
|
18
|
+
'claude-haiku-4-5': { input: 0.80, output: 4.00, cacheWrite: 1.00, cacheRead: 0.08 },
|
|
19
|
+
};
|
|
20
|
+
const DEFAULT_PRICE = { input: 3.00, output: 15.00, cacheWrite: 3.75, cacheRead: 0.30 };
|
|
21
|
+
function getPricing(model) {
|
|
22
|
+
if (!model)
|
|
23
|
+
return DEFAULT_PRICE;
|
|
24
|
+
// Exact match first, then prefix match
|
|
25
|
+
if (PRICING[model])
|
|
26
|
+
return PRICING[model];
|
|
27
|
+
for (const key of Object.keys(PRICING)) {
|
|
28
|
+
if (model.startsWith(key))
|
|
29
|
+
return PRICING[key];
|
|
30
|
+
}
|
|
31
|
+
return DEFAULT_PRICE;
|
|
32
|
+
}
|
|
33
|
+
function calcCost(p, inputTok, outputTok, cacheWriteTok, cacheReadTok) {
|
|
34
|
+
return ((inputTok * p.input / 1000000) +
|
|
35
|
+
(outputTok * p.output / 1000000) +
|
|
36
|
+
(cacheWriteTok * p.cacheWrite / 1000000) +
|
|
37
|
+
(cacheReadTok * p.cacheRead / 1000000));
|
|
38
|
+
}
|
|
39
|
+
// ─── Public API ───────────────────────────────────────────────────────────────
|
|
40
|
+
function loadMessages(file) {
|
|
41
|
+
return loadMessagesWithUsage(file).messages;
|
|
42
|
+
}
|
|
43
|
+
function loadMessagesWithUsage(file) {
|
|
44
|
+
const raw = (0, fs_1.readFileSync)(file, 'utf-8');
|
|
45
|
+
if (file.endsWith('.jsonl')) {
|
|
46
|
+
return parseJsonl(raw);
|
|
47
|
+
}
|
|
48
|
+
const parsed = JSON.parse(raw);
|
|
49
|
+
if (!Array.isArray(parsed)) {
|
|
50
|
+
throw new Error('File must contain a JSON array of messages.');
|
|
51
|
+
}
|
|
52
|
+
return { messages: parsed };
|
|
53
|
+
}
|
|
54
|
+
// ─── JSONL parser ─────────────────────────────────────────────────────────────
|
|
55
|
+
// Supports two JSONL shapes:
|
|
56
|
+
// 1. Claude Code session transcript: { type: 'user'|'assistant', message: { role, content } }
|
|
57
|
+
// 2. Plain message-per-line: { role, content }
|
|
58
|
+
function parseJsonl(raw) {
|
|
59
|
+
let messages = [];
|
|
60
|
+
let actualInputTokens;
|
|
61
|
+
const perTurn = [];
|
|
62
|
+
let postCompactTurnOffset = 0; // perTurn index where last compact happened
|
|
63
|
+
for (const line of raw.trim().split('\n')) {
|
|
64
|
+
if (!line.trim())
|
|
65
|
+
continue;
|
|
66
|
+
const record = JSON.parse(line);
|
|
67
|
+
// /compact boundary — reset live context (only post-compact messages are active)
|
|
68
|
+
if (record.type === 'system' && record.subtype === 'compact_boundary') {
|
|
69
|
+
messages = [];
|
|
70
|
+
actualInputTokens = undefined;
|
|
71
|
+
postCompactTurnOffset = perTurn.length; // cost history kept, but mark the split
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
// Claude Code session transcript
|
|
75
|
+
if ((record.type === 'user' || record.type === 'assistant') && record.message?.role) {
|
|
76
|
+
const msg = record.message;
|
|
77
|
+
// Extract actual token usage from each assistant record
|
|
78
|
+
if (record.type === 'assistant' && msg.usage) {
|
|
79
|
+
const u = msg.usage;
|
|
80
|
+
const inputTok = u.input_tokens ?? 0;
|
|
81
|
+
const outputTok = u.output_tokens ?? 0;
|
|
82
|
+
const cacheWriteTok = u.cache_creation_input_tokens ?? 0;
|
|
83
|
+
const cacheReadTok = u.cache_read_input_tokens ?? 0;
|
|
84
|
+
const total = inputTok + cacheWriteTok + cacheReadTok;
|
|
85
|
+
if (total > 0) {
|
|
86
|
+
actualInputTokens = total; // keep last for session extractor
|
|
87
|
+
const model = msg.model ?? record.model ?? '';
|
|
88
|
+
const p = getPricing(model);
|
|
89
|
+
perTurn.push({
|
|
90
|
+
messageIndex: messages.length, // index of the message about to be pushed
|
|
91
|
+
timestamp: record.timestamp ?? new Date().toISOString(),
|
|
92
|
+
model,
|
|
93
|
+
inputTokens: inputTok,
|
|
94
|
+
outputTokens: outputTok,
|
|
95
|
+
cacheWriteTokens: cacheWriteTok,
|
|
96
|
+
cacheReadTokens: cacheReadTok,
|
|
97
|
+
cost: calcCost(p, inputTok, outputTok, cacheWriteTok, cacheReadTok),
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// Keep the full content array — do NOT filter to text-only
|
|
102
|
+
const { role, content } = msg;
|
|
103
|
+
if (Array.isArray(content)) {
|
|
104
|
+
if (content.length > 0)
|
|
105
|
+
messages.push({ role, content });
|
|
106
|
+
}
|
|
107
|
+
else if (typeof content === 'string' && content.trim()) {
|
|
108
|
+
messages.push({ role, content });
|
|
109
|
+
}
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
// Plain { role, content }
|
|
113
|
+
if (record.role && record.content !== undefined) {
|
|
114
|
+
messages.push({ role: record.role, content: record.content });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (messages.length === 0) {
|
|
118
|
+
throw new Error('No valid messages found in JSONL file.');
|
|
119
|
+
}
|
|
120
|
+
const costData = buildCostData(perTurn);
|
|
121
|
+
return { messages, actualInputTokens, costData };
|
|
122
|
+
}
|
|
123
|
+
// ─── Session grouping ─────────────────────────────────────────────────────────
|
|
124
|
+
// Group consecutive turns into sessions by calendar date.
|
|
125
|
+
// A gap of >2 hours between turns also starts a new session.
|
|
126
|
+
function buildCostData(perTurn) {
|
|
127
|
+
const totalCost = perTurn.reduce((s, t) => s + t.cost, 0);
|
|
128
|
+
// Group by date
|
|
129
|
+
const byDate = {};
|
|
130
|
+
for (const turn of perTurn) {
|
|
131
|
+
const date = turn.timestamp.slice(0, 10); // YYYY-MM-DD
|
|
132
|
+
if (!byDate[date])
|
|
133
|
+
byDate[date] = [];
|
|
134
|
+
byDate[date].push(turn);
|
|
135
|
+
}
|
|
136
|
+
const sessions = Object.entries(byDate)
|
|
137
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
138
|
+
.map(([date, turns]) => ({
|
|
139
|
+
date,
|
|
140
|
+
label: formatDate(date),
|
|
141
|
+
totalCost: turns.reduce((s, t) => s + t.cost, 0),
|
|
142
|
+
totalInputTokens: turns.reduce((s, t) => s + t.inputTokens + t.cacheWriteTokens + t.cacheReadTokens, 0),
|
|
143
|
+
totalOutputTokens: turns.reduce((s, t) => s + t.outputTokens, 0),
|
|
144
|
+
turns: turns.length,
|
|
145
|
+
}));
|
|
146
|
+
return { totalCost, currency: 'USD', perTurn, sessions };
|
|
147
|
+
}
|
|
148
|
+
function formatDate(iso) {
|
|
149
|
+
// iso = "YYYY-MM-DD"
|
|
150
|
+
const [y, m, d] = iso.split('-').map(Number);
|
|
151
|
+
const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
|
|
152
|
+
return `${months[m - 1]} ${d}, ${y}`;
|
|
153
|
+
}
|
|
154
|
+
// ─── Cross-session cost aggregation ───────────────────────────────────────────
|
|
155
|
+
// Reads ALL .jsonl files in the project directory and merges their cost history.
|
|
156
|
+
// The current (active) file's perTurn entries keep their messageIndex intact so
|
|
157
|
+
// per-message cost cells still work in the dashboard. All other files' entries
|
|
158
|
+
// use messageIndex = -1 (they count toward totalCost and sessions but won't
|
|
159
|
+
// match any message row in the top-consumers table).
|
|
160
|
+
function loadProjectCostHistory(projectDir, currentFile) {
|
|
161
|
+
if (!(0, fs_1.existsSync)(projectDir))
|
|
162
|
+
return buildCostData([]);
|
|
163
|
+
let files;
|
|
164
|
+
try {
|
|
165
|
+
files = (0, fs_1.readdirSync)(projectDir)
|
|
166
|
+
.filter(f => f.endsWith('.jsonl'))
|
|
167
|
+
.map(f => (0, path_1.join)(projectDir, f));
|
|
168
|
+
}
|
|
169
|
+
catch {
|
|
170
|
+
return buildCostData([]);
|
|
171
|
+
}
|
|
172
|
+
const merged = [];
|
|
173
|
+
for (const file of files) {
|
|
174
|
+
try {
|
|
175
|
+
const result = loadMessagesWithUsage(file);
|
|
176
|
+
const turns = result.costData?.perTurn ?? [];
|
|
177
|
+
if (file === currentFile) {
|
|
178
|
+
// Keep messageIndex as-is — matches the live messages array
|
|
179
|
+
merged.push(...turns);
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
// Historical session — contribute to cost totals but no message linkage
|
|
183
|
+
merged.push(...turns.map(t => ({ ...t, messageIndex: -1 })));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
catch { /* skip unreadable files */ }
|
|
187
|
+
}
|
|
188
|
+
// Sort all turns chronologically before building sessions
|
|
189
|
+
merged.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
190
|
+
return buildCostData(merged);
|
|
191
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// src/brief/index.ts
|
|
3
|
+
// Generates a structured session brief from classified messages.
|
|
4
|
+
// No LLM calls — pure extraction. Used when context is getting full
|
|
5
|
+
// so the user can paste it into a new session and continue from where they left off.
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.generateSessionBrief = generateSessionBrief;
|
|
8
|
+
const utils_1 = require("../utils");
|
|
9
|
+
const BRIEF_CHAR_LIMIT = 300; // per message excerpt
|
|
10
|
+
const MAX_DECISIONS = 5;
|
|
11
|
+
const MAX_PROGRESS = 5;
|
|
12
|
+
const MAX_STATE = 3;
|
|
13
|
+
function generateSessionBrief(messages) {
|
|
14
|
+
const goal = extractGoal(messages);
|
|
15
|
+
const constraints = extractConstraints(messages);
|
|
16
|
+
const decisions = extractDecisions(messages);
|
|
17
|
+
const progress = extractProgress(messages);
|
|
18
|
+
const state = extractCurrentState(messages);
|
|
19
|
+
const openIssues = extractOpenIssues(messages);
|
|
20
|
+
const sections = [];
|
|
21
|
+
sections.push('# Session Brief\n');
|
|
22
|
+
sections.push('> Paste this at the start of a new session to continue from where you left off.\n');
|
|
23
|
+
if (goal) {
|
|
24
|
+
sections.push(`## Goal\n${goal}`);
|
|
25
|
+
}
|
|
26
|
+
if (constraints.length) {
|
|
27
|
+
sections.push(`## Constraints & Rules\n${constraints.map(c => `- ${c}`).join('\n')}`);
|
|
28
|
+
}
|
|
29
|
+
if (decisions.length) {
|
|
30
|
+
sections.push(`## Key Decisions Made\n${decisions.map(d => `- ${d}`).join('\n')}`);
|
|
31
|
+
}
|
|
32
|
+
if (progress.length) {
|
|
33
|
+
sections.push(`## What's Done\n${progress.map(p => `- ${p}`).join('\n')}`);
|
|
34
|
+
}
|
|
35
|
+
if (state.length) {
|
|
36
|
+
sections.push(`## Current State\n${state.join('\n\n')}`);
|
|
37
|
+
}
|
|
38
|
+
if (openIssues.length) {
|
|
39
|
+
sections.push(`## Open Issues\n${openIssues.map(i => `- ${i}`).join('\n')}`);
|
|
40
|
+
}
|
|
41
|
+
sections.push('## Continue\nPick up where we left off based on the above context.');
|
|
42
|
+
return sections.join('\n\n');
|
|
43
|
+
}
|
|
44
|
+
// ─── Extractors ───────────────────────────────────────────────────────────────
|
|
45
|
+
function extractGoal(messages) {
|
|
46
|
+
// Prefer highest-scored TASK_DEFINITION; fall back to first user message
|
|
47
|
+
const taskMsgs = messages
|
|
48
|
+
.filter(m => m.classification === 'TASK_DEFINITION')
|
|
49
|
+
.sort((a, b) => b.relevanceScore - a.relevanceScore);
|
|
50
|
+
const source = taskMsgs[0]
|
|
51
|
+
?? messages.find(m => m.original.role === 'user');
|
|
52
|
+
return source ? truncate((0, utils_1.extractText)(source.original), 400) : '';
|
|
53
|
+
}
|
|
54
|
+
function extractConstraints(messages) {
|
|
55
|
+
const out = [];
|
|
56
|
+
for (const m of messages) {
|
|
57
|
+
if (m.classification === 'SYSTEM_CONSTRAINT' || m.classification === 'USER_CORRECTION') {
|
|
58
|
+
const text = truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT);
|
|
59
|
+
if (text)
|
|
60
|
+
out.push(text);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
function extractDecisions(messages) {
|
|
66
|
+
return messages
|
|
67
|
+
.filter(m => m.classification === 'DECISION_FINAL')
|
|
68
|
+
.sort((a, b) => b.relevanceScore - a.relevanceScore)
|
|
69
|
+
.slice(0, MAX_DECISIONS)
|
|
70
|
+
.map(m => truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT))
|
|
71
|
+
.filter(Boolean);
|
|
72
|
+
}
|
|
73
|
+
function extractProgress(messages) {
|
|
74
|
+
return messages
|
|
75
|
+
.filter(m => m.classification === 'PROGRESS_MARKER')
|
|
76
|
+
.slice(-MAX_PROGRESS) // most recent progress markers
|
|
77
|
+
.map(m => truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT))
|
|
78
|
+
.filter(Boolean);
|
|
79
|
+
}
|
|
80
|
+
function extractCurrentState(messages) {
|
|
81
|
+
// Most recent active tool outputs — these represent the actual current state
|
|
82
|
+
return messages
|
|
83
|
+
.filter(m => m.classification === 'TOOL_OUTPUT_ACTIVE')
|
|
84
|
+
.slice(-MAX_STATE)
|
|
85
|
+
.map(m => {
|
|
86
|
+
const text = truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT);
|
|
87
|
+
return text ? `**Call ${m.originalIndex}:** ${text}` : '';
|
|
88
|
+
})
|
|
89
|
+
.filter(Boolean);
|
|
90
|
+
}
|
|
91
|
+
function extractOpenIssues(messages) {
|
|
92
|
+
return messages
|
|
93
|
+
.filter(m => m.classification === 'ERROR_ACTIVE')
|
|
94
|
+
.map(m => truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT))
|
|
95
|
+
.filter(Boolean);
|
|
96
|
+
}
|
|
97
|
+
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
|
98
|
+
function truncate(text, limit) {
|
|
99
|
+
const cleaned = text.replace(/\s+/g, ' ').trim();
|
|
100
|
+
return cleaned.length > limit ? cleaned.slice(0, limit) + '…' : cleaned;
|
|
101
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { ConfidenceTier, CompressionStrategy } from '../types';
|
|
2
|
+
export declare function getConfidenceTier(confidence: number): ConfidenceTier;
|
|
3
|
+
export declare const CONFIDENCE_COMPRESSION_MAP: Record<ConfidenceTier, CompressionStrategy[]>;
|
|
4
|
+
export declare function isStrategyAllowed(strategy: CompressionStrategy, confidenceTier: ConfidenceTier): boolean;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// src/classifier/confidence.ts
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.CONFIDENCE_COMPRESSION_MAP = void 0;
|
|
5
|
+
exports.getConfidenceTier = getConfidenceTier;
|
|
6
|
+
exports.isStrategyAllowed = isStrategyAllowed;
|
|
7
|
+
function getConfidenceTier(confidence) {
|
|
8
|
+
if (confidence >= 0.85)
|
|
9
|
+
return 'HIGH';
|
|
10
|
+
if (confidence >= 0.70)
|
|
11
|
+
return 'MEDIUM';
|
|
12
|
+
return 'LOW';
|
|
13
|
+
}
|
|
14
|
+
// Maps confidence tier to maximum allowed compression strategies.
|
|
15
|
+
// LOW confidence = preserve always, regardless of relevance score.
|
|
16
|
+
exports.CONFIDENCE_COMPRESSION_MAP = {
|
|
17
|
+
HIGH: ['DROP', 'EXTRACT_RESULT', 'SUMMARIZE', 'DEDUPLICATE', 'COLLAPSE_TO_MARKER', 'PRESERVE'],
|
|
18
|
+
MEDIUM: ['EXTRACT_RESULT', 'SUMMARIZE', 'DEDUPLICATE', 'PRESERVE'],
|
|
19
|
+
LOW: ['PRESERVE'],
|
|
20
|
+
};
|
|
21
|
+
function isStrategyAllowed(strategy, confidenceTier) {
|
|
22
|
+
return exports.CONFIDENCE_COMPRESSION_MAP[confidenceTier].includes(strategy);
|
|
23
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { LLMMessage, AnnotatedMessage, MessageClassification } from '../types';
|
|
2
|
+
interface ClassificationResult {
|
|
3
|
+
classification: MessageClassification;
|
|
4
|
+
confidence: number;
|
|
5
|
+
method: 'rule' | 'heuristic';
|
|
6
|
+
metadata: Record<string, unknown>;
|
|
7
|
+
}
|
|
8
|
+
export declare function isPinned(classification: MessageClassification): boolean;
|
|
9
|
+
export declare function classifyMessage(message: LLMMessage, index: number, allMessages: LLMMessage[], model: string): Promise<ClassificationResult>;
|
|
10
|
+
export declare function classifyAll(messages: LLMMessage[], model: string): Promise<AnnotatedMessage[]>;
|
|
11
|
+
export {};
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// src/classifier/index.ts
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.isPinned = isPinned;
|
|
5
|
+
exports.classifyMessage = classifyMessage;
|
|
6
|
+
exports.classifyAll = classifyAll;
|
|
7
|
+
const tokenizer_1 = require("../tokenizer");
|
|
8
|
+
const utils_1 = require("../utils");
|
|
9
|
+
const patterns_1 = require("./patterns");
|
|
10
|
+
// ─── Pinned Types ─────────────────────────────────────────────────────────────
|
|
11
|
+
function isPinned(classification) {
|
|
12
|
+
return (classification === 'SYSTEM_CONSTRAINT' ||
|
|
13
|
+
classification === 'USER_CORRECTION' ||
|
|
14
|
+
classification === 'TASK_DEFINITION' ||
|
|
15
|
+
classification === 'ERROR_ACTIVE');
|
|
16
|
+
}
|
|
17
|
+
// ─── Tool Output Sub-Classifier ───────────────────────────────────────────────
|
|
18
|
+
function detectToolType(content) {
|
|
19
|
+
const text = JSON.stringify(content).toLowerCase();
|
|
20
|
+
if (text.includes('read_file') || text.includes('cat ') || text.includes('file_content'))
|
|
21
|
+
return 'file_read';
|
|
22
|
+
if (text.includes('write_file') || text.includes('create_file') || text.includes('edit_file') || text.includes('str_replace'))
|
|
23
|
+
return 'write';
|
|
24
|
+
if (text.includes('run_command') || text.includes('bash') || text.includes('execute') || text.includes('shell'))
|
|
25
|
+
return 'command_exec';
|
|
26
|
+
if (text.includes('search') || text.includes('grep') || text.includes('find'))
|
|
27
|
+
return 'search';
|
|
28
|
+
return 'unknown';
|
|
29
|
+
}
|
|
30
|
+
function isToolOutputReferenced(message, laterMessages) {
|
|
31
|
+
const content = Array.isArray(message.content) ? message.content : [];
|
|
32
|
+
const toolUseId = content.find((b) => b.id || b.tool_use_id);
|
|
33
|
+
if (toolUseId) {
|
|
34
|
+
const id = toolUseId.id ?? toolUseId.tool_use_id;
|
|
35
|
+
if (id) {
|
|
36
|
+
return laterMessages.some((m) => JSON.stringify(m.content).includes(id));
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
function hasNewerToolCall(message, allMessages, currentIndex) {
|
|
42
|
+
const content = Array.isArray(message.content) ? message.content : [];
|
|
43
|
+
const toolName = content.find((b) => b.name)?.name;
|
|
44
|
+
if (!toolName)
|
|
45
|
+
return false;
|
|
46
|
+
const laterMessages = allMessages.slice(currentIndex + 1);
|
|
47
|
+
return laterMessages.some((m) => {
|
|
48
|
+
const laterContent = Array.isArray(m.content) ? m.content : [];
|
|
49
|
+
return laterContent.some((b) => b.name === toolName);
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
async function classifyToolOutput(message, allMessages, currentIndex, model) {
|
|
53
|
+
const content = Array.isArray(message.content) ? message.content : [];
|
|
54
|
+
const toolType = detectToolType(content);
|
|
55
|
+
const text = typeof message.content === 'string' ? message.content : JSON.stringify(message.content);
|
|
56
|
+
const tokenCount = await (0, tokenizer_1.countTokens)(text, model);
|
|
57
|
+
const laterMessages = allMessages.slice(currentIndex + 1);
|
|
58
|
+
const isReferenced = isToolOutputReferenced(message, laterMessages);
|
|
59
|
+
// Pure tool_result messages have no 'name' field, so hasNewerToolCall returns
|
|
60
|
+
// false — but their staleness should mirror the preceding tool_use message.
|
|
61
|
+
const hasToolResults = content.some((b) => b.type === 'tool_result');
|
|
62
|
+
const hasToolUse = content.some((b) => b.type === 'tool_use');
|
|
63
|
+
let hasNewer;
|
|
64
|
+
if (hasToolResults && !hasToolUse && currentIndex > 0) {
|
|
65
|
+
const preceding = allMessages[currentIndex - 1];
|
|
66
|
+
hasNewer = preceding ? hasNewerToolCall(preceding, allMessages, currentIndex - 1) : false;
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
hasNewer = hasNewerToolCall(message, allMessages, currentIndex);
|
|
70
|
+
}
|
|
71
|
+
const turnsFromEnd = allMessages.length - currentIndex;
|
|
72
|
+
const isOld = turnsFromEnd > 6;
|
|
73
|
+
const isStale = !isReferenced && (hasNewer || isOld);
|
|
74
|
+
return {
|
|
75
|
+
classification: isStale ? 'TOOL_OUTPUT_STALE' : 'TOOL_OUTPUT_ACTIVE',
|
|
76
|
+
confidence: isStale ? 0.85 : 0.75,
|
|
77
|
+
toolType,
|
|
78
|
+
isLarge: tokenCount > 500,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
// ─── Error Resolution Checker ─────────────────────────────────────────────────
|
|
82
|
+
function isErrorResolved(errorMessageIndex, allMessages) {
|
|
83
|
+
const searchWindow = allMessages.slice(errorMessageIndex + 1, errorMessageIndex + 9);
|
|
84
|
+
return searchWindow.some((m) => {
|
|
85
|
+
const text = typeof m.content === 'string' ? m.content : JSON.stringify(m.content);
|
|
86
|
+
return patterns_1.RESOLUTION_PATTERNS.some((pattern) => pattern.test(text));
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
// ─── Main Classifier ──────────────────────────────────────────────────────────
|
|
90
|
+
async function classifyMessage(message, index, allMessages, model) {
|
|
91
|
+
// Rule 1: System messages — deterministic
|
|
92
|
+
if (message.role === 'system') {
|
|
93
|
+
return { classification: 'SYSTEM_CONSTRAINT', confidence: 1.0, method: 'rule', metadata: {} };
|
|
94
|
+
}
|
|
95
|
+
const text = (0, utils_1.extractText)(message);
|
|
96
|
+
// Rule 2: Tool outputs — structural detection
|
|
97
|
+
const hasToolContent = Array.isArray(message.content) &&
|
|
98
|
+
message.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result');
|
|
99
|
+
if (hasToolContent) {
|
|
100
|
+
const toolResult = await classifyToolOutput(message, allMessages, index, model);
|
|
101
|
+
return {
|
|
102
|
+
classification: toolResult.classification,
|
|
103
|
+
confidence: toolResult.confidence,
|
|
104
|
+
method: 'rule',
|
|
105
|
+
metadata: { toolType: toolResult.toolType, isLarge: toolResult.isLarge },
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
// Rule 3: User corrections — check before task definition
|
|
109
|
+
if (message.role === 'user') {
|
|
110
|
+
const correctionMatches = patterns_1.CORRECTION_PATTERNS.filter((p) => p.test(text));
|
|
111
|
+
if (correctionMatches.length >= 1) {
|
|
112
|
+
return {
|
|
113
|
+
classification: 'USER_CORRECTION',
|
|
114
|
+
confidence: correctionMatches.length >= 2 ? 0.95 : 0.75,
|
|
115
|
+
method: 'heuristic',
|
|
116
|
+
metadata: { matchCount: correctionMatches.length },
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// Rule 4: Task definition — first user messages
|
|
121
|
+
if (message.role === 'user' && index <= 2) {
|
|
122
|
+
const taskMatches = patterns_1.TASK_DEFINITION_PATTERNS.filter((p) => p.test(text));
|
|
123
|
+
if (taskMatches.length >= 1) {
|
|
124
|
+
return {
|
|
125
|
+
classification: 'TASK_DEFINITION',
|
|
126
|
+
confidence: index === 0 ? 0.95 : 0.80,
|
|
127
|
+
method: 'heuristic',
|
|
128
|
+
metadata: {},
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// Rule 5: Errors — multiple pattern matches required to reduce false positives
|
|
133
|
+
const errorMatches = patterns_1.ERROR_PATTERNS.filter((p) => p.test(text));
|
|
134
|
+
if (errorMatches.length >= 2) {
|
|
135
|
+
const resolved = isErrorResolved(index, allMessages);
|
|
136
|
+
return {
|
|
137
|
+
classification: resolved ? 'ERROR_RESOLVED' : 'ERROR_ACTIVE',
|
|
138
|
+
confidence: errorMatches.length >= 3 ? 0.92 : 0.78,
|
|
139
|
+
method: 'heuristic',
|
|
140
|
+
metadata: { matchCount: errorMatches.length, resolved },
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
// Rule 6: Progress markers — short assistant messages with progress language
|
|
144
|
+
if (message.role === 'assistant') {
|
|
145
|
+
const progressMatches = patterns_1.PROGRESS_PATTERNS.filter((p) => p.test(text));
|
|
146
|
+
if (progressMatches.length >= 1 && text.length < 500) {
|
|
147
|
+
return {
|
|
148
|
+
classification: 'PROGRESS_MARKER',
|
|
149
|
+
confidence: 0.72,
|
|
150
|
+
method: 'heuristic',
|
|
151
|
+
metadata: {},
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
// Rule 7: Reasoning vs Decision — assistant only
|
|
156
|
+
if (message.role === 'assistant') {
|
|
157
|
+
const conclusionMatches = patterns_1.CONCLUSION_PATTERNS.filter((p) => p.test(text));
|
|
158
|
+
const tokenCount = await (0, tokenizer_1.countTokens)(text, model);
|
|
159
|
+
if (conclusionMatches.length >= 1) {
|
|
160
|
+
return {
|
|
161
|
+
classification: 'DECISION_FINAL',
|
|
162
|
+
confidence: 0.70,
|
|
163
|
+
method: 'heuristic',
|
|
164
|
+
metadata: { tokenCount },
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
if (tokenCount > 800) {
|
|
168
|
+
return {
|
|
169
|
+
classification: 'REASONING_INTERMEDIATE',
|
|
170
|
+
confidence: 0.65,
|
|
171
|
+
method: 'heuristic',
|
|
172
|
+
metadata: { tokenCount },
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
// Default — CONVERSATIONAL is the safe fallback; LOW confidence prevents aggressive compression
|
|
177
|
+
return {
|
|
178
|
+
classification: 'CONVERSATIONAL',
|
|
179
|
+
confidence: 0.60,
|
|
180
|
+
method: 'heuristic',
|
|
181
|
+
metadata: {},
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
// ─── Batch Classifier ─────────────────────────────────────────────────────────
|
|
185
|
+
async function classifyAll(messages, model) {
|
|
186
|
+
const annotated = [];
|
|
187
|
+
for (let i = 0; i < messages.length; i++) {
|
|
188
|
+
const message = messages[i];
|
|
189
|
+
if (!message)
|
|
190
|
+
continue;
|
|
191
|
+
const result = await classifyMessage(message, i, messages, model);
|
|
192
|
+
// Use full content for token count — tool messages have substantial JSON payload
|
|
193
|
+
const fullText = typeof message.content === 'string'
|
|
194
|
+
? message.content
|
|
195
|
+
: JSON.stringify(message.content);
|
|
196
|
+
const tokenCount = await (0, tokenizer_1.countTokens)(fullText, model);
|
|
197
|
+
annotated.push({
|
|
198
|
+
id: (0, utils_1.generateId)(),
|
|
199
|
+
originalIndex: i,
|
|
200
|
+
original: message,
|
|
201
|
+
classification: result.classification,
|
|
202
|
+
classificationConfidence: result.confidence,
|
|
203
|
+
classificationMethod: result.method,
|
|
204
|
+
relevanceScore: 0, // filled by scorer
|
|
205
|
+
relevanceFactors: {
|
|
206
|
+
recencyScore: 0,
|
|
207
|
+
referenceCount: 0,
|
|
208
|
+
taskProximity: 0,
|
|
209
|
+
isPinned: isPinned(result.classification),
|
|
210
|
+
},
|
|
211
|
+
tokenCount,
|
|
212
|
+
compressionStrategy: 'PRESERVE', // filled by strategy selector
|
|
213
|
+
compressionApplied: false,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
return annotated;
|
|
217
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare const ERROR_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
|
|
2
|
+
export declare const RESOLUTION_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
|
|
3
|
+
export declare const CORRECTION_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
|
|
4
|
+
export declare const PROGRESS_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
|
|
5
|
+
export declare const TASK_DEFINITION_PATTERNS: readonly [RegExp, RegExp, RegExp];
|
|
6
|
+
export declare const CONCLUSION_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp];
|
|
7
|
+
export declare const TECH_STACK_SIGNALS: Record<string, RegExp>;
|