@grapine.ai/contextprune 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +426 -1
  2. package/dist/cli/commands/analyze.d.ts +2 -0
  3. package/dist/cli/commands/analyze.js +161 -0
  4. package/dist/cli/commands/compress.d.ts +2 -0
  5. package/dist/cli/commands/compress.js +65 -0
  6. package/dist/cli/commands/watch.d.ts +2 -0
  7. package/dist/cli/commands/watch.js +432 -0
  8. package/dist/cli/dashboard/index.html +720 -0
  9. package/dist/cli/index.d.ts +2 -0
  10. package/dist/cli/index.js +19 -0
  11. package/dist/cli/labels.d.ts +4 -0
  12. package/dist/cli/labels.js +35 -0
  13. package/dist/cli/parse-input.d.ts +33 -0
  14. package/dist/cli/parse-input.js +191 -0
  15. package/dist/src/brief/index.d.ts +2 -0
  16. package/dist/src/brief/index.js +101 -0
  17. package/dist/src/classifier/confidence.d.ts +4 -0
  18. package/dist/src/classifier/confidence.js +23 -0
  19. package/dist/src/classifier/index.d.ts +11 -0
  20. package/dist/src/classifier/index.js +217 -0
  21. package/dist/src/classifier/patterns.d.ts +7 -0
  22. package/dist/src/classifier/patterns.js +81 -0
  23. package/dist/src/compression/engine.d.ts +23 -0
  24. package/dist/src/compression/engine.js +363 -0
  25. package/dist/src/index.d.ts +41 -0
  26. package/dist/src/index.js +120 -0
  27. package/dist/src/pipeline/index.d.ts +5 -0
  28. package/dist/src/pipeline/index.js +167 -0
  29. package/dist/src/scorer/index.d.ts +4 -0
  30. package/dist/src/scorer/index.js +136 -0
  31. package/dist/src/scorer/session-extractor.d.ts +2 -0
  32. package/dist/src/scorer/session-extractor.js +57 -0
  33. package/dist/src/strategy/selector.d.ts +3 -0
  34. package/dist/src/strategy/selector.js +158 -0
  35. package/dist/src/tokenizer/index.d.ts +18 -0
  36. package/dist/src/tokenizer/index.js +195 -0
  37. package/dist/src/types.d.ts +161 -0
  38. package/dist/src/types.js +5 -0
  39. package/dist/src/utils/index.d.ts +4 -0
  40. package/dist/src/utils/index.js +48 -0
  41. package/dist/src/validation/coherence.d.ts +3 -0
  42. package/dist/src/validation/coherence.js +87 -0
  43. package/license.md +14 -0
  44. package/package.json +77 -41
  45. package/screenshots/cp_dashboard_compression.jpg +0 -0
  46. package/screenshots/cp_dashboard_healthy.jpg +0 -0
  47. package/index.js +0 -1
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env node
2
+ export {};
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env node
2
+ "use strict";
3
+ // cli/index.ts
4
+ // ContextPrune CLI entry point.
5
+ // Usage: npx contextprune <command> [options]
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ const commander_1 = require("commander");
8
+ const analyze_1 = require("./commands/analyze");
9
+ const compress_1 = require("./commands/compress");
10
+ const watch_1 = require("./commands/watch");
11
+ const program = new commander_1.Command();
12
+ program
13
+ .name('contextprune')
14
+ .description('Intelligent context window management for LLM applications')
15
+ .version('0.1.0');
16
+ program.addCommand((0, analyze_1.analyzeCommand)());
17
+ program.addCommand((0, compress_1.compressCommand)());
18
+ program.addCommand((0, watch_1.watchCommand)());
19
+ program.parse(process.argv);
@@ -0,0 +1,4 @@
1
+ export declare const CLS_LABELS: Record<string, string>;
2
+ export declare const STRAT_LABELS: Record<string, string>;
3
+ export declare function clsLabel(cls: string): string;
4
+ export declare function stratLabel(strat: string): string;
@@ -0,0 +1,35 @@
1
+ "use strict";
2
+ // cli/labels.ts
3
+ // Single source of truth for human-readable classification and strategy names.
4
+ // Used by the CLI (analyze command) and injected into the dashboard HTML by the watch server.
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.STRAT_LABELS = exports.CLS_LABELS = void 0;
7
+ exports.clsLabel = clsLabel;
8
+ exports.stratLabel = stratLabel;
9
+ exports.CLS_LABELS = {
10
+ SYSTEM_CONSTRAINT: 'System Prompt',
11
+ TASK_DEFINITION: 'Session Goal',
12
+ TOOL_OUTPUT_ACTIVE: 'Tool Result',
13
+ TOOL_OUTPUT_STALE: 'Outdated Tool Result',
14
+ ERROR_ACTIVE: 'Active Error',
15
+ ERROR_RESOLVED: 'Fixed Error',
16
+ REASONING_INTERMEDIATE: 'Chain of Thought',
17
+ DECISION_FINAL: 'Final Answer',
18
+ USER_CORRECTION: 'Your Override',
19
+ PROGRESS_MARKER: 'Status Update',
20
+ CONVERSATIONAL: 'Chat / Filler',
21
+ };
22
+ exports.STRAT_LABELS = {
23
+ PRESERVE: 'Keep',
24
+ DROP: 'Remove',
25
+ EXTRACT_RESULT: 'Trim to Key Output',
26
+ COLLAPSE_TO_MARKER: 'Collapse to 1 Line',
27
+ SUMMARIZE: 'Summarize',
28
+ DEDUPLICATE: 'Remove Duplicate',
29
+ };
30
+ function clsLabel(cls) {
31
+ return exports.CLS_LABELS[cls] ?? cls;
32
+ }
33
+ function stratLabel(strat) {
34
+ return exports.STRAT_LABELS[strat] ?? strat;
35
+ }
@@ -0,0 +1,33 @@
1
+ import type { LLMMessage } from '../src/types';
2
+ export interface TurnCost {
3
+ messageIndex: number;
4
+ timestamp: string;
5
+ model: string;
6
+ inputTokens: number;
7
+ outputTokens: number;
8
+ cacheWriteTokens: number;
9
+ cacheReadTokens: number;
10
+ cost: number;
11
+ }
12
+ export interface SessionCost {
13
+ date: string;
14
+ label: string;
15
+ totalCost: number;
16
+ totalInputTokens: number;
17
+ totalOutputTokens: number;
18
+ turns: number;
19
+ }
20
+ export interface CostData {
21
+ totalCost: number;
22
+ currency: 'USD';
23
+ perTurn: TurnCost[];
24
+ sessions: SessionCost[];
25
+ }
26
+ export interface LoadResult {
27
+ messages: LLMMessage[];
28
+ actualInputTokens?: number;
29
+ costData?: CostData;
30
+ }
31
+ export declare function loadMessages(file: string): LLMMessage[];
32
+ export declare function loadMessagesWithUsage(file: string): LoadResult;
33
+ export declare function loadProjectCostHistory(projectDir: string, currentFile: string): CostData;
@@ -0,0 +1,191 @@
1
+ "use strict";
2
+ // cli/parse-input.ts
3
+ // Shared file-loading logic for CLI commands.
4
+ // Supports .json (array) and .jsonl (Claude Code session transcripts or plain message-per-line).
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.loadMessages = loadMessages;
7
+ exports.loadMessagesWithUsage = loadMessagesWithUsage;
8
+ exports.loadProjectCostHistory = loadProjectCostHistory;
9
+ const fs_1 = require("fs");
10
+ const path_1 = require("path");
11
+ // ─── Cost pricing table (USD per million tokens) ──────────────────────────────
12
+ // Prices: https://www.anthropic.com/pricing
13
+ const PRICING = {
14
+ 'claude-opus-4-6': { input: 15.00, output: 75.00, cacheWrite: 18.75, cacheRead: 1.50 },
15
+ 'claude-opus-4-5': { input: 15.00, output: 75.00, cacheWrite: 18.75, cacheRead: 1.50 },
16
+ 'claude-sonnet-4-6': { input: 3.00, output: 15.00, cacheWrite: 3.75, cacheRead: 0.30 },
17
+ 'claude-sonnet-4-5': { input: 3.00, output: 15.00, cacheWrite: 3.75, cacheRead: 0.30 },
18
+ 'claude-haiku-4-5': { input: 0.80, output: 4.00, cacheWrite: 1.00, cacheRead: 0.08 },
19
+ };
20
+ const DEFAULT_PRICE = { input: 3.00, output: 15.00, cacheWrite: 3.75, cacheRead: 0.30 };
21
+ function getPricing(model) {
22
+ if (!model)
23
+ return DEFAULT_PRICE;
24
+ // Exact match first, then prefix match
25
+ if (PRICING[model])
26
+ return PRICING[model];
27
+ for (const key of Object.keys(PRICING)) {
28
+ if (model.startsWith(key))
29
+ return PRICING[key];
30
+ }
31
+ return DEFAULT_PRICE;
32
+ }
33
+ function calcCost(p, inputTok, outputTok, cacheWriteTok, cacheReadTok) {
34
+ return ((inputTok * p.input / 1000000) +
35
+ (outputTok * p.output / 1000000) +
36
+ (cacheWriteTok * p.cacheWrite / 1000000) +
37
+ (cacheReadTok * p.cacheRead / 1000000));
38
+ }
39
+ // ─── Public API ───────────────────────────────────────────────────────────────
40
+ function loadMessages(file) {
41
+ return loadMessagesWithUsage(file).messages;
42
+ }
43
+ function loadMessagesWithUsage(file) {
44
+ const raw = (0, fs_1.readFileSync)(file, 'utf-8');
45
+ if (file.endsWith('.jsonl')) {
46
+ return parseJsonl(raw);
47
+ }
48
+ const parsed = JSON.parse(raw);
49
+ if (!Array.isArray(parsed)) {
50
+ throw new Error('File must contain a JSON array of messages.');
51
+ }
52
+ return { messages: parsed };
53
+ }
54
+ // ─── JSONL parser ─────────────────────────────────────────────────────────────
55
+ // Supports two JSONL shapes:
56
+ // 1. Claude Code session transcript: { type: 'user'|'assistant', message: { role, content } }
57
+ // 2. Plain message-per-line: { role, content }
58
+ function parseJsonl(raw) {
59
+ let messages = [];
60
+ let actualInputTokens;
61
+ const perTurn = [];
62
+ let postCompactTurnOffset = 0; // perTurn index where last compact happened
63
+ for (const line of raw.trim().split('\n')) {
64
+ if (!line.trim())
65
+ continue;
66
+ const record = JSON.parse(line);
67
+ // /compact boundary — reset live context (only post-compact messages are active)
68
+ if (record.type === 'system' && record.subtype === 'compact_boundary') {
69
+ messages = [];
70
+ actualInputTokens = undefined;
71
+ postCompactTurnOffset = perTurn.length; // cost history kept, but mark the split
72
+ continue;
73
+ }
74
+ // Claude Code session transcript
75
+ if ((record.type === 'user' || record.type === 'assistant') && record.message?.role) {
76
+ const msg = record.message;
77
+ // Extract actual token usage from each assistant record
78
+ if (record.type === 'assistant' && msg.usage) {
79
+ const u = msg.usage;
80
+ const inputTok = u.input_tokens ?? 0;
81
+ const outputTok = u.output_tokens ?? 0;
82
+ const cacheWriteTok = u.cache_creation_input_tokens ?? 0;
83
+ const cacheReadTok = u.cache_read_input_tokens ?? 0;
84
+ const total = inputTok + cacheWriteTok + cacheReadTok;
85
+ if (total > 0) {
86
+ actualInputTokens = total; // keep last for session extractor
87
+ const model = msg.model ?? record.model ?? '';
88
+ const p = getPricing(model);
89
+ perTurn.push({
90
+ messageIndex: messages.length, // index of the message about to be pushed
91
+ timestamp: record.timestamp ?? new Date().toISOString(),
92
+ model,
93
+ inputTokens: inputTok,
94
+ outputTokens: outputTok,
95
+ cacheWriteTokens: cacheWriteTok,
96
+ cacheReadTokens: cacheReadTok,
97
+ cost: calcCost(p, inputTok, outputTok, cacheWriteTok, cacheReadTok),
98
+ });
99
+ }
100
+ }
101
+ // Keep the full content array — do NOT filter to text-only
102
+ const { role, content } = msg;
103
+ if (Array.isArray(content)) {
104
+ if (content.length > 0)
105
+ messages.push({ role, content });
106
+ }
107
+ else if (typeof content === 'string' && content.trim()) {
108
+ messages.push({ role, content });
109
+ }
110
+ continue;
111
+ }
112
+ // Plain { role, content }
113
+ if (record.role && record.content !== undefined) {
114
+ messages.push({ role: record.role, content: record.content });
115
+ }
116
+ }
117
+ if (messages.length === 0) {
118
+ throw new Error('No valid messages found in JSONL file.');
119
+ }
120
+ const costData = buildCostData(perTurn);
121
+ return { messages, actualInputTokens, costData };
122
+ }
123
+ // ─── Session grouping ─────────────────────────────────────────────────────────
124
+ // Group consecutive turns into sessions by calendar date.
125
+ // A gap of >2 hours between turns also starts a new session.
126
+ function buildCostData(perTurn) {
127
+ const totalCost = perTurn.reduce((s, t) => s + t.cost, 0);
128
+ // Group by date
129
+ const byDate = {};
130
+ for (const turn of perTurn) {
131
+ const date = turn.timestamp.slice(0, 10); // YYYY-MM-DD
132
+ if (!byDate[date])
133
+ byDate[date] = [];
134
+ byDate[date].push(turn);
135
+ }
136
+ const sessions = Object.entries(byDate)
137
+ .sort(([a], [b]) => a.localeCompare(b))
138
+ .map(([date, turns]) => ({
139
+ date,
140
+ label: formatDate(date),
141
+ totalCost: turns.reduce((s, t) => s + t.cost, 0),
142
+ totalInputTokens: turns.reduce((s, t) => s + t.inputTokens + t.cacheWriteTokens + t.cacheReadTokens, 0),
143
+ totalOutputTokens: turns.reduce((s, t) => s + t.outputTokens, 0),
144
+ turns: turns.length,
145
+ }));
146
+ return { totalCost, currency: 'USD', perTurn, sessions };
147
+ }
148
+ function formatDate(iso) {
149
+ // iso = "YYYY-MM-DD"
150
+ const [y, m, d] = iso.split('-').map(Number);
151
+ const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
152
+ return `${months[m - 1]} ${d}, ${y}`;
153
+ }
154
+ // ─── Cross-session cost aggregation ───────────────────────────────────────────
155
+ // Reads ALL .jsonl files in the project directory and merges their cost history.
156
+ // The current (active) file's perTurn entries keep their messageIndex intact so
157
+ // per-message cost cells still work in the dashboard. All other files' entries
158
+ // use messageIndex = -1 (they count toward totalCost and sessions but won't
159
+ // match any message row in the top-consumers table).
160
+ function loadProjectCostHistory(projectDir, currentFile) {
161
+ if (!(0, fs_1.existsSync)(projectDir))
162
+ return buildCostData([]);
163
+ let files;
164
+ try {
165
+ files = (0, fs_1.readdirSync)(projectDir)
166
+ .filter(f => f.endsWith('.jsonl'))
167
+ .map(f => (0, path_1.join)(projectDir, f));
168
+ }
169
+ catch {
170
+ return buildCostData([]);
171
+ }
172
+ const merged = [];
173
+ for (const file of files) {
174
+ try {
175
+ const result = loadMessagesWithUsage(file);
176
+ const turns = result.costData?.perTurn ?? [];
177
+ if (file === currentFile) {
178
+ // Keep messageIndex as-is — matches the live messages array
179
+ merged.push(...turns);
180
+ }
181
+ else {
182
+ // Historical session — contribute to cost totals but no message linkage
183
+ merged.push(...turns.map(t => ({ ...t, messageIndex: -1 })));
184
+ }
185
+ }
186
+ catch { /* skip unreadable files */ }
187
+ }
188
+ // Sort all turns chronologically before building sessions
189
+ merged.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
190
+ return buildCostData(merged);
191
+ }
@@ -0,0 +1,2 @@
1
+ import type { AnnotatedMessage } from '../types';
2
+ export declare function generateSessionBrief(messages: AnnotatedMessage[]): string;
@@ -0,0 +1,101 @@
1
+ "use strict";
2
+ // src/brief/index.ts
3
+ // Generates a structured session brief from classified messages.
4
+ // No LLM calls — pure extraction. Used when context is getting full
5
+ // so the user can paste it into a new session and continue from where they left off.
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ exports.generateSessionBrief = generateSessionBrief;
8
+ const utils_1 = require("../utils");
9
+ const BRIEF_CHAR_LIMIT = 300; // per message excerpt
10
+ const MAX_DECISIONS = 5;
11
+ const MAX_PROGRESS = 5;
12
+ const MAX_STATE = 3;
13
+ function generateSessionBrief(messages) {
14
+ const goal = extractGoal(messages);
15
+ const constraints = extractConstraints(messages);
16
+ const decisions = extractDecisions(messages);
17
+ const progress = extractProgress(messages);
18
+ const state = extractCurrentState(messages);
19
+ const openIssues = extractOpenIssues(messages);
20
+ const sections = [];
21
+ sections.push('# Session Brief\n');
22
+ sections.push('> Paste this at the start of a new session to continue from where you left off.\n');
23
+ if (goal) {
24
+ sections.push(`## Goal\n${goal}`);
25
+ }
26
+ if (constraints.length) {
27
+ sections.push(`## Constraints & Rules\n${constraints.map(c => `- ${c}`).join('\n')}`);
28
+ }
29
+ if (decisions.length) {
30
+ sections.push(`## Key Decisions Made\n${decisions.map(d => `- ${d}`).join('\n')}`);
31
+ }
32
+ if (progress.length) {
33
+ sections.push(`## What's Done\n${progress.map(p => `- ${p}`).join('\n')}`);
34
+ }
35
+ if (state.length) {
36
+ sections.push(`## Current State\n${state.join('\n\n')}`);
37
+ }
38
+ if (openIssues.length) {
39
+ sections.push(`## Open Issues\n${openIssues.map(i => `- ${i}`).join('\n')}`);
40
+ }
41
+ sections.push('## Continue\nPick up where we left off based on the above context.');
42
+ return sections.join('\n\n');
43
+ }
44
+ // ─── Extractors ───────────────────────────────────────────────────────────────
45
+ function extractGoal(messages) {
46
+ // Prefer highest-scored TASK_DEFINITION; fall back to first user message
47
+ const taskMsgs = messages
48
+ .filter(m => m.classification === 'TASK_DEFINITION')
49
+ .sort((a, b) => b.relevanceScore - a.relevanceScore);
50
+ const source = taskMsgs[0]
51
+ ?? messages.find(m => m.original.role === 'user');
52
+ return source ? truncate((0, utils_1.extractText)(source.original), 400) : '';
53
+ }
54
+ function extractConstraints(messages) {
55
+ const out = [];
56
+ for (const m of messages) {
57
+ if (m.classification === 'SYSTEM_CONSTRAINT' || m.classification === 'USER_CORRECTION') {
58
+ const text = truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT);
59
+ if (text)
60
+ out.push(text);
61
+ }
62
+ }
63
+ return out;
64
+ }
65
+ function extractDecisions(messages) {
66
+ return messages
67
+ .filter(m => m.classification === 'DECISION_FINAL')
68
+ .sort((a, b) => b.relevanceScore - a.relevanceScore)
69
+ .slice(0, MAX_DECISIONS)
70
+ .map(m => truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT))
71
+ .filter(Boolean);
72
+ }
73
+ function extractProgress(messages) {
74
+ return messages
75
+ .filter(m => m.classification === 'PROGRESS_MARKER')
76
+ .slice(-MAX_PROGRESS) // most recent progress markers
77
+ .map(m => truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT))
78
+ .filter(Boolean);
79
+ }
80
+ function extractCurrentState(messages) {
81
+ // Most recent active tool outputs — these represent the actual current state
82
+ return messages
83
+ .filter(m => m.classification === 'TOOL_OUTPUT_ACTIVE')
84
+ .slice(-MAX_STATE)
85
+ .map(m => {
86
+ const text = truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT);
87
+ return text ? `**Call ${m.originalIndex}:** ${text}` : '';
88
+ })
89
+ .filter(Boolean);
90
+ }
91
+ function extractOpenIssues(messages) {
92
+ return messages
93
+ .filter(m => m.classification === 'ERROR_ACTIVE')
94
+ .map(m => truncate((0, utils_1.extractText)(m.original), BRIEF_CHAR_LIMIT))
95
+ .filter(Boolean);
96
+ }
97
+ // ─── Helpers ──────────────────────────────────────────────────────────────────
98
+ function truncate(text, limit) {
99
+ const cleaned = text.replace(/\s+/g, ' ').trim();
100
+ return cleaned.length > limit ? cleaned.slice(0, limit) + '…' : cleaned;
101
+ }
@@ -0,0 +1,4 @@
1
+ import type { ConfidenceTier, CompressionStrategy } from '../types';
2
+ export declare function getConfidenceTier(confidence: number): ConfidenceTier;
3
+ export declare const CONFIDENCE_COMPRESSION_MAP: Record<ConfidenceTier, CompressionStrategy[]>;
4
+ export declare function isStrategyAllowed(strategy: CompressionStrategy, confidenceTier: ConfidenceTier): boolean;
@@ -0,0 +1,23 @@
1
+ "use strict";
2
+ // src/classifier/confidence.ts
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.CONFIDENCE_COMPRESSION_MAP = void 0;
5
+ exports.getConfidenceTier = getConfidenceTier;
6
+ exports.isStrategyAllowed = isStrategyAllowed;
7
+ function getConfidenceTier(confidence) {
8
+ if (confidence >= 0.85)
9
+ return 'HIGH';
10
+ if (confidence >= 0.70)
11
+ return 'MEDIUM';
12
+ return 'LOW';
13
+ }
14
+ // Maps confidence tier to maximum allowed compression strategies.
15
+ // LOW confidence = preserve always, regardless of relevance score.
16
+ exports.CONFIDENCE_COMPRESSION_MAP = {
17
+ HIGH: ['DROP', 'EXTRACT_RESULT', 'SUMMARIZE', 'DEDUPLICATE', 'COLLAPSE_TO_MARKER', 'PRESERVE'],
18
+ MEDIUM: ['EXTRACT_RESULT', 'SUMMARIZE', 'DEDUPLICATE', 'PRESERVE'],
19
+ LOW: ['PRESERVE'],
20
+ };
21
+ function isStrategyAllowed(strategy, confidenceTier) {
22
+ return exports.CONFIDENCE_COMPRESSION_MAP[confidenceTier].includes(strategy);
23
+ }
@@ -0,0 +1,11 @@
1
+ import type { LLMMessage, AnnotatedMessage, MessageClassification } from '../types';
2
+ interface ClassificationResult {
3
+ classification: MessageClassification;
4
+ confidence: number;
5
+ method: 'rule' | 'heuristic';
6
+ metadata: Record<string, unknown>;
7
+ }
8
+ export declare function isPinned(classification: MessageClassification): boolean;
9
+ export declare function classifyMessage(message: LLMMessage, index: number, allMessages: LLMMessage[], model: string): Promise<ClassificationResult>;
10
+ export declare function classifyAll(messages: LLMMessage[], model: string): Promise<AnnotatedMessage[]>;
11
+ export {};
@@ -0,0 +1,217 @@
1
+ "use strict";
2
+ // src/classifier/index.ts
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.isPinned = isPinned;
5
+ exports.classifyMessage = classifyMessage;
6
+ exports.classifyAll = classifyAll;
7
+ const tokenizer_1 = require("../tokenizer");
8
+ const utils_1 = require("../utils");
9
+ const patterns_1 = require("./patterns");
10
+ // ─── Pinned Types ─────────────────────────────────────────────────────────────
11
+ function isPinned(classification) {
12
+ return (classification === 'SYSTEM_CONSTRAINT' ||
13
+ classification === 'USER_CORRECTION' ||
14
+ classification === 'TASK_DEFINITION' ||
15
+ classification === 'ERROR_ACTIVE');
16
+ }
17
+ // ─── Tool Output Sub-Classifier ───────────────────────────────────────────────
18
+ function detectToolType(content) {
19
+ const text = JSON.stringify(content).toLowerCase();
20
+ if (text.includes('read_file') || text.includes('cat ') || text.includes('file_content'))
21
+ return 'file_read';
22
+ if (text.includes('write_file') || text.includes('create_file') || text.includes('edit_file') || text.includes('str_replace'))
23
+ return 'write';
24
+ if (text.includes('run_command') || text.includes('bash') || text.includes('execute') || text.includes('shell'))
25
+ return 'command_exec';
26
+ if (text.includes('search') || text.includes('grep') || text.includes('find'))
27
+ return 'search';
28
+ return 'unknown';
29
+ }
30
+ function isToolOutputReferenced(message, laterMessages) {
31
+ const content = Array.isArray(message.content) ? message.content : [];
32
+ const toolUseId = content.find((b) => b.id || b.tool_use_id);
33
+ if (toolUseId) {
34
+ const id = toolUseId.id ?? toolUseId.tool_use_id;
35
+ if (id) {
36
+ return laterMessages.some((m) => JSON.stringify(m.content).includes(id));
37
+ }
38
+ }
39
+ return false;
40
+ }
41
+ function hasNewerToolCall(message, allMessages, currentIndex) {
42
+ const content = Array.isArray(message.content) ? message.content : [];
43
+ const toolName = content.find((b) => b.name)?.name;
44
+ if (!toolName)
45
+ return false;
46
+ const laterMessages = allMessages.slice(currentIndex + 1);
47
+ return laterMessages.some((m) => {
48
+ const laterContent = Array.isArray(m.content) ? m.content : [];
49
+ return laterContent.some((b) => b.name === toolName);
50
+ });
51
+ }
52
+ async function classifyToolOutput(message, allMessages, currentIndex, model) {
53
+ const content = Array.isArray(message.content) ? message.content : [];
54
+ const toolType = detectToolType(content);
55
+ const text = typeof message.content === 'string' ? message.content : JSON.stringify(message.content);
56
+ const tokenCount = await (0, tokenizer_1.countTokens)(text, model);
57
+ const laterMessages = allMessages.slice(currentIndex + 1);
58
+ const isReferenced = isToolOutputReferenced(message, laterMessages);
59
+ // Pure tool_result messages have no 'name' field, so hasNewerToolCall returns
60
+ // false — but their staleness should mirror the preceding tool_use message.
61
+ const hasToolResults = content.some((b) => b.type === 'tool_result');
62
+ const hasToolUse = content.some((b) => b.type === 'tool_use');
63
+ let hasNewer;
64
+ if (hasToolResults && !hasToolUse && currentIndex > 0) {
65
+ const preceding = allMessages[currentIndex - 1];
66
+ hasNewer = preceding ? hasNewerToolCall(preceding, allMessages, currentIndex - 1) : false;
67
+ }
68
+ else {
69
+ hasNewer = hasNewerToolCall(message, allMessages, currentIndex);
70
+ }
71
+ const turnsFromEnd = allMessages.length - currentIndex;
72
+ const isOld = turnsFromEnd > 6;
73
+ const isStale = !isReferenced && (hasNewer || isOld);
74
+ return {
75
+ classification: isStale ? 'TOOL_OUTPUT_STALE' : 'TOOL_OUTPUT_ACTIVE',
76
+ confidence: isStale ? 0.85 : 0.75,
77
+ toolType,
78
+ isLarge: tokenCount > 500,
79
+ };
80
+ }
81
+ // ─── Error Resolution Checker ─────────────────────────────────────────────────
82
+ function isErrorResolved(errorMessageIndex, allMessages) {
83
+ const searchWindow = allMessages.slice(errorMessageIndex + 1, errorMessageIndex + 9);
84
+ return searchWindow.some((m) => {
85
+ const text = typeof m.content === 'string' ? m.content : JSON.stringify(m.content);
86
+ return patterns_1.RESOLUTION_PATTERNS.some((pattern) => pattern.test(text));
87
+ });
88
+ }
89
+ // ─── Main Classifier ──────────────────────────────────────────────────────────
90
+ async function classifyMessage(message, index, allMessages, model) {
91
+ // Rule 1: System messages — deterministic
92
+ if (message.role === 'system') {
93
+ return { classification: 'SYSTEM_CONSTRAINT', confidence: 1.0, method: 'rule', metadata: {} };
94
+ }
95
+ const text = (0, utils_1.extractText)(message);
96
+ // Rule 2: Tool outputs — structural detection
97
+ const hasToolContent = Array.isArray(message.content) &&
98
+ message.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result');
99
+ if (hasToolContent) {
100
+ const toolResult = await classifyToolOutput(message, allMessages, index, model);
101
+ return {
102
+ classification: toolResult.classification,
103
+ confidence: toolResult.confidence,
104
+ method: 'rule',
105
+ metadata: { toolType: toolResult.toolType, isLarge: toolResult.isLarge },
106
+ };
107
+ }
108
+ // Rule 3: User corrections — check before task definition
109
+ if (message.role === 'user') {
110
+ const correctionMatches = patterns_1.CORRECTION_PATTERNS.filter((p) => p.test(text));
111
+ if (correctionMatches.length >= 1) {
112
+ return {
113
+ classification: 'USER_CORRECTION',
114
+ confidence: correctionMatches.length >= 2 ? 0.95 : 0.75,
115
+ method: 'heuristic',
116
+ metadata: { matchCount: correctionMatches.length },
117
+ };
118
+ }
119
+ }
120
+ // Rule 4: Task definition — first user messages
121
+ if (message.role === 'user' && index <= 2) {
122
+ const taskMatches = patterns_1.TASK_DEFINITION_PATTERNS.filter((p) => p.test(text));
123
+ if (taskMatches.length >= 1) {
124
+ return {
125
+ classification: 'TASK_DEFINITION',
126
+ confidence: index === 0 ? 0.95 : 0.80,
127
+ method: 'heuristic',
128
+ metadata: {},
129
+ };
130
+ }
131
+ }
132
+ // Rule 5: Errors — multiple pattern matches required to reduce false positives
133
+ const errorMatches = patterns_1.ERROR_PATTERNS.filter((p) => p.test(text));
134
+ if (errorMatches.length >= 2) {
135
+ const resolved = isErrorResolved(index, allMessages);
136
+ return {
137
+ classification: resolved ? 'ERROR_RESOLVED' : 'ERROR_ACTIVE',
138
+ confidence: errorMatches.length >= 3 ? 0.92 : 0.78,
139
+ method: 'heuristic',
140
+ metadata: { matchCount: errorMatches.length, resolved },
141
+ };
142
+ }
143
+ // Rule 6: Progress markers — short assistant messages with progress language
144
+ if (message.role === 'assistant') {
145
+ const progressMatches = patterns_1.PROGRESS_PATTERNS.filter((p) => p.test(text));
146
+ if (progressMatches.length >= 1 && text.length < 500) {
147
+ return {
148
+ classification: 'PROGRESS_MARKER',
149
+ confidence: 0.72,
150
+ method: 'heuristic',
151
+ metadata: {},
152
+ };
153
+ }
154
+ }
155
+ // Rule 7: Reasoning vs Decision — assistant only
156
+ if (message.role === 'assistant') {
157
+ const conclusionMatches = patterns_1.CONCLUSION_PATTERNS.filter((p) => p.test(text));
158
+ const tokenCount = await (0, tokenizer_1.countTokens)(text, model);
159
+ if (conclusionMatches.length >= 1) {
160
+ return {
161
+ classification: 'DECISION_FINAL',
162
+ confidence: 0.70,
163
+ method: 'heuristic',
164
+ metadata: { tokenCount },
165
+ };
166
+ }
167
+ if (tokenCount > 800) {
168
+ return {
169
+ classification: 'REASONING_INTERMEDIATE',
170
+ confidence: 0.65,
171
+ method: 'heuristic',
172
+ metadata: { tokenCount },
173
+ };
174
+ }
175
+ }
176
+ // Default — CONVERSATIONAL is the safe fallback; LOW confidence prevents aggressive compression
177
+ return {
178
+ classification: 'CONVERSATIONAL',
179
+ confidence: 0.60,
180
+ method: 'heuristic',
181
+ metadata: {},
182
+ };
183
+ }
184
+ // ─── Batch Classifier ─────────────────────────────────────────────────────────
185
+ async function classifyAll(messages, model) {
186
+ const annotated = [];
187
+ for (let i = 0; i < messages.length; i++) {
188
+ const message = messages[i];
189
+ if (!message)
190
+ continue;
191
+ const result = await classifyMessage(message, i, messages, model);
192
+ // Use full content for token count — tool messages have substantial JSON payload
193
+ const fullText = typeof message.content === 'string'
194
+ ? message.content
195
+ : JSON.stringify(message.content);
196
+ const tokenCount = await (0, tokenizer_1.countTokens)(fullText, model);
197
+ annotated.push({
198
+ id: (0, utils_1.generateId)(),
199
+ originalIndex: i,
200
+ original: message,
201
+ classification: result.classification,
202
+ classificationConfidence: result.confidence,
203
+ classificationMethod: result.method,
204
+ relevanceScore: 0, // filled by scorer
205
+ relevanceFactors: {
206
+ recencyScore: 0,
207
+ referenceCount: 0,
208
+ taskProximity: 0,
209
+ isPinned: isPinned(result.classification),
210
+ },
211
+ tokenCount,
212
+ compressionStrategy: 'PRESERVE', // filled by strategy selector
213
+ compressionApplied: false,
214
+ });
215
+ }
216
+ return annotated;
217
+ }
@@ -0,0 +1,7 @@
1
+ export declare const ERROR_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
2
+ export declare const RESOLUTION_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
3
+ export declare const CORRECTION_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
4
+ export declare const PROGRESS_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
5
+ export declare const TASK_DEFINITION_PATTERNS: readonly [RegExp, RegExp, RegExp];
6
+ export declare const CONCLUSION_PATTERNS: readonly [RegExp, RegExp, RegExp, RegExp, RegExp];
7
+ export declare const TECH_STACK_SIGNALS: Record<string, RegExp>;