opengauge 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -0
- package/bin/opengauge.js +70 -0
- package/dist/core/optimizer/checkpoint.d.ts +37 -0
- package/dist/core/optimizer/checkpoint.d.ts.map +1 -0
- package/dist/core/optimizer/checkpoint.js +81 -0
- package/dist/core/optimizer/checkpoint.js.map +1 -0
- package/dist/core/optimizer/compressor.d.ts +41 -0
- package/dist/core/optimizer/compressor.d.ts.map +1 -0
- package/dist/core/optimizer/compressor.js +134 -0
- package/dist/core/optimizer/compressor.js.map +1 -0
- package/dist/core/optimizer/dedup.d.ts +48 -0
- package/dist/core/optimizer/dedup.d.ts.map +1 -0
- package/dist/core/optimizer/dedup.js +147 -0
- package/dist/core/optimizer/dedup.js.map +1 -0
- package/dist/core/providers/adapter.d.ts +48 -0
- package/dist/core/providers/adapter.d.ts.map +1 -0
- package/dist/core/providers/adapter.js +22 -0
- package/dist/core/providers/adapter.js.map +1 -0
- package/dist/core/providers/anthropic.d.ts +12 -0
- package/dist/core/providers/anthropic.d.ts.map +1 -0
- package/dist/core/providers/anthropic.js +155 -0
- package/dist/core/providers/anthropic.js.map +1 -0
- package/dist/core/providers/gemini.d.ts +13 -0
- package/dist/core/providers/gemini.d.ts.map +1 -0
- package/dist/core/providers/gemini.js +154 -0
- package/dist/core/providers/gemini.js.map +1 -0
- package/dist/core/providers/ollama.d.ts +11 -0
- package/dist/core/providers/ollama.d.ts.map +1 -0
- package/dist/core/providers/ollama.js +119 -0
- package/dist/core/providers/ollama.js.map +1 -0
- package/dist/core/providers/openai.d.ts +12 -0
- package/dist/core/providers/openai.d.ts.map +1 -0
- package/dist/core/providers/openai.js +169 -0
- package/dist/core/providers/openai.js.map +1 -0
- package/dist/core/rag/assembler.d.ts +47 -0
- package/dist/core/rag/assembler.d.ts.map +1 -0
- package/dist/core/rag/assembler.js +178 -0
- package/dist/core/rag/assembler.js.map +1 -0
- package/dist/core/rag/embedder.d.ts +16 -0
- package/dist/core/rag/embedder.d.ts.map +1 -0
- package/dist/core/rag/embedder.js +223 -0
- package/dist/core/rag/embedder.js.map +1 -0
- package/dist/core/rag/retriever.d.ts +20 -0
- package/dist/core/rag/retriever.d.ts.map +1 -0
- package/dist/core/rag/retriever.js +71 -0
- package/dist/core/rag/retriever.js.map +1 -0
- package/dist/db/index.d.ts +5 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +48 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/queries.d.ts +72 -0
- package/dist/db/queries.d.ts.map +1 -0
- package/dist/db/queries.js +169 -0
- package/dist/db/queries.js.map +1 -0
- package/dist/db/schema.d.ts +3 -0
- package/dist/db/schema.d.ts.map +1 -0
- package/dist/db/schema.js +71 -0
- package/dist/db/schema.js.map +1 -0
- package/dist/server/config.d.ts +25 -0
- package/dist/server/config.d.ts.map +1 -0
- package/dist/server/config.js +69 -0
- package/dist/server/config.js.map +1 -0
- package/dist/server/index.d.ts +5 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +61 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/routes/index.d.ts +6 -0
- package/dist/server/routes/index.d.ts.map +1 -0
- package/dist/server/routes/index.js +272 -0
- package/dist/server/routes/index.js.map +1 -0
- package/dist/server/sse.d.ts +21 -0
- package/dist/server/sse.d.ts.map +1 -0
- package/dist/server/sse.js +40 -0
- package/dist/server/sse.js.map +1 -0
- package/dist/ui/static/app.js +515 -0
- package/dist/ui/static/index.html +13 -0
- package/dist/ui/static/styles.css +506 -0
- package/dist/ui/static/vendor.js +26 -0
- package/package.json +49 -0
package/README.md
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# OpenGauge
|
|
2
|
+
|
|
3
|
+
A local-first, token-efficient LLM chat interface that runs on your machine and helps reduce context bloat while preserving useful history.
|
|
4
|
+
|
|
5
|
+
## What this repo does
|
|
6
|
+
|
|
7
|
+
OpenGauge provides:
|
|
8
|
+
- A local web chat UI on `http://localhost:3000`
|
|
9
|
+
- Token optimization (compression + deduplication + checkpointing)
|
|
10
|
+
- Context retrieval from past messages (RAG-style memory)
|
|
11
|
+
- Provider support for Anthropic, OpenAI, Gemini, and Ollama
|
|
12
|
+
- Local storage via SQLite in `~/.opengauge/opengauge.db`
|
|
13
|
+
|
|
14
|
+
## Quick start
|
|
15
|
+
|
|
16
|
+
### Run directly
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
npx opengauge
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
### Run from source
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
git clone https://github.com/applytorque/opengauge.git
|
|
26
|
+
cd opengauge
|
|
27
|
+
npm install
|
|
28
|
+
npm run build
|
|
29
|
+
npm start
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Then open:
|
|
33
|
+
|
|
34
|
+
```txt
|
|
35
|
+
http://localhost:3000
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Provider setup
|
|
39
|
+
|
|
40
|
+
On first launch, use the in-app setup wizard or create this file:
|
|
41
|
+
|
|
42
|
+
`~/.opengauge/config.yml`
|
|
43
|
+
|
|
44
|
+
```yaml
|
|
45
|
+
providers:
|
|
46
|
+
anthropic:
|
|
47
|
+
api_key: YOUR_API_KEY
|
|
48
|
+
default_model: claude-opus-4-6
|
|
49
|
+
|
|
50
|
+
defaults:
|
|
51
|
+
provider: anthropic
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Useful scripts
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
npm run build # compile TypeScript + copy UI assets
|
|
58
|
+
npm start # start OpenGauge
|
|
59
|
+
npm pack --dry-run # preview npm package contents
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## License
|
|
63
|
+
|
|
64
|
+
MIT
|
package/bin/opengauge.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* OpenGauge CLI — starts the server and opens the browser
|
|
5
|
+
*
|
|
6
|
+
* Usage:
|
|
7
|
+
* npx opengauge
|
|
8
|
+
* npx opengauge --port 8080
|
|
9
|
+
* npx opengauge --no-open
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const { createServer } = require('../dist/server/index');
|
|
13
|
+
|
|
14
|
+
async function main() {
|
|
15
|
+
const args = process.argv.slice(2);
|
|
16
|
+
|
|
17
|
+
let port = 3000;
|
|
18
|
+
let shouldOpen = true;
|
|
19
|
+
|
|
20
|
+
for (let i = 0; i < args.length; i++) {
|
|
21
|
+
if (args[i] === '--port' && args[i + 1]) {
|
|
22
|
+
port = parseInt(args[i + 1], 10);
|
|
23
|
+
i++;
|
|
24
|
+
}
|
|
25
|
+
if (args[i] === '--no-open') {
|
|
26
|
+
shouldOpen = false;
|
|
27
|
+
}
|
|
28
|
+
if (args[i] === '--help' || args[i] === '-h') {
|
|
29
|
+
console.log(`
|
|
30
|
+
OpenGauge — Token-efficient LLM chat interface
|
|
31
|
+
|
|
32
|
+
Usage:
|
|
33
|
+
npx opengauge [options]
|
|
34
|
+
|
|
35
|
+
Options:
|
|
36
|
+
--port <number> Port to run on (default: 3000)
|
|
37
|
+
--no-open Don't open browser automatically
|
|
38
|
+
--help, -h Show this help message
|
|
39
|
+
|
|
40
|
+
Config:
|
|
41
|
+
~/.opengauge/config.yml Provider configuration
|
|
42
|
+
~/.opengauge/opengauge.db SQLite database
|
|
43
|
+
|
|
44
|
+
Homepage: https://github.com/opengauge/opengauge
|
|
45
|
+
`);
|
|
46
|
+
process.exit(0);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
console.log(`
|
|
51
|
+
⚡ OpenGauge v0.1.0
|
|
52
|
+
Token-efficient LLM chat interface
|
|
53
|
+
`);
|
|
54
|
+
|
|
55
|
+
await createServer(port);
|
|
56
|
+
|
|
57
|
+
if (shouldOpen) {
|
|
58
|
+
try {
|
|
59
|
+
const open = (await import('open')).default;
|
|
60
|
+
await open(`http://localhost:${port}`);
|
|
61
|
+
} catch {
|
|
62
|
+
console.log(` Open http://localhost:${port} in your browser\n`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
main().catch((err) => {
|
|
68
|
+
console.error('Failed to start OpenGauge:', err);
|
|
69
|
+
process.exit(1);
|
|
70
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conversation Checkpointing — Stage 3 of the Token Optimizer Pipeline
|
|
3
|
+
*
|
|
4
|
+
* At configurable intervals, older conversation history is summarized
|
|
5
|
+
* into a compressed checkpoint using the active LLM. Recent turns remain
|
|
6
|
+
* verbatim. The checkpoint replaces raw history in the context window.
|
|
7
|
+
* Original messages are preserved in the database.
|
|
8
|
+
*/
|
|
9
|
+
import { LLMProvider, ChatMessage } from '../providers/adapter';
|
|
10
|
+
export interface CheckpointConfig {
|
|
11
|
+
/** Number of turns between checkpoints. Default: 20 */
|
|
12
|
+
interval: number;
|
|
13
|
+
/** Number of recent turns to keep verbatim. Default: 10 */
|
|
14
|
+
keepRecent: number;
|
|
15
|
+
}
|
|
16
|
+
export declare const DEFAULT_CHECKPOINT_CONFIG: CheckpointConfig;
|
|
17
|
+
/**
|
|
18
|
+
* Determine if a checkpoint should be generated.
|
|
19
|
+
*/
|
|
20
|
+
export declare function shouldCheckpoint(messageCount: number, lastCheckpointMessageCount: number, config?: CheckpointConfig): boolean;
|
|
21
|
+
/**
|
|
22
|
+
* Generate a checkpoint summary using the LLM.
|
|
23
|
+
*
|
|
24
|
+
* @param messages - The messages to summarize.
|
|
25
|
+
* @param provider - The LLM provider to use for summarization.
|
|
26
|
+
* @param model - The model to use.
|
|
27
|
+
* @returns Summary text.
|
|
28
|
+
*/
|
|
29
|
+
export declare function generateCheckpointSummary(messages: ChatMessage[], provider: LLMProvider, model: string): Promise<string>;
|
|
30
|
+
/**
|
|
31
|
+
* Apply checkpointing to a message list for context assembly.
|
|
32
|
+
*
|
|
33
|
+
* Returns a modified message list where older messages are replaced
|
|
34
|
+
* with the checkpoint summary, and recent messages are kept verbatim.
|
|
35
|
+
*/
|
|
36
|
+
export declare function applyCheckpoint(allMessages: ChatMessage[], checkpointSummary: string | null, keepRecent?: number): ChatMessage[];
|
|
37
|
+
//# sourceMappingURL=checkpoint.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint.d.ts","sourceRoot":"","sources":["../../../src/core/optimizer/checkpoint.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEhE,MAAM,WAAW,gBAAgB;IAC/B,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,yBAAyB,EAAE,gBAGvC,CAAC;AAEF;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,YAAY,EAAE,MAAM,EACpB,0BAA0B,EAAE,MAAM,EAClC,MAAM,GAAE,gBAA4C,GACnD,OAAO,CAET;AAED;;;;;;;GAOG;AACH,wBAAsB,yBAAyB,CAC7C,QAAQ,EAAE,WAAW,EAAE,EACvB,QAAQ,EAAE,WAAW,EACrB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,CAAC,CA+BjB;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,WAAW,EAAE,EAC1B,iBAAiB,EAAE,MAAM,GAAG,IAAI,EAChC,UAAU,GAAE,MAA6C,GACxD,WAAW,EAAE,CAgBf"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Conversation Checkpointing — Stage 3 of the Token Optimizer Pipeline
|
|
4
|
+
*
|
|
5
|
+
* At configurable intervals, older conversation history is summarized
|
|
6
|
+
* into a compressed checkpoint using the active LLM. Recent turns remain
|
|
7
|
+
* verbatim. The checkpoint replaces raw history in the context window.
|
|
8
|
+
* Original messages are preserved in the database.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.DEFAULT_CHECKPOINT_CONFIG = void 0;
|
|
12
|
+
exports.shouldCheckpoint = shouldCheckpoint;
|
|
13
|
+
exports.generateCheckpointSummary = generateCheckpointSummary;
|
|
14
|
+
exports.applyCheckpoint = applyCheckpoint;
|
|
15
|
+
exports.DEFAULT_CHECKPOINT_CONFIG = {
|
|
16
|
+
interval: 20,
|
|
17
|
+
keepRecent: 10,
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Determine if a checkpoint should be generated.
|
|
21
|
+
*/
|
|
22
|
+
function shouldCheckpoint(messageCount, lastCheckpointMessageCount, config = exports.DEFAULT_CHECKPOINT_CONFIG) {
|
|
23
|
+
return messageCount - lastCheckpointMessageCount >= config.interval;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Generate a checkpoint summary using the LLM.
|
|
27
|
+
*
|
|
28
|
+
* @param messages - The messages to summarize.
|
|
29
|
+
* @param provider - The LLM provider to use for summarization.
|
|
30
|
+
* @param model - The model to use.
|
|
31
|
+
* @returns Summary text.
|
|
32
|
+
*/
|
|
33
|
+
async function generateCheckpointSummary(messages, provider, model) {
|
|
34
|
+
const conversationText = messages
|
|
35
|
+
.map((m) => `[${m.role}]: ${m.content}`)
|
|
36
|
+
.join('\n\n');
|
|
37
|
+
const summaryRequest = [
|
|
38
|
+
{
|
|
39
|
+
role: 'system',
|
|
40
|
+
content: `You are a conversation summarizer. Create a concise but comprehensive summary of the following conversation. Preserve key facts, decisions, code snippets, and context that would be important for continuing the conversation. Focus on:
|
|
41
|
+
1. Main topics discussed
|
|
42
|
+
2. Key decisions made
|
|
43
|
+
3. Important code or technical details
|
|
44
|
+
4. Any unresolved questions or pending tasks
|
|
45
|
+
5. The user's goals and preferences
|
|
46
|
+
|
|
47
|
+
Keep the summary under 500 words. Use bullet points for clarity.`,
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
role: 'user',
|
|
51
|
+
content: `Summarize this conversation:\n\n${conversationText}`,
|
|
52
|
+
},
|
|
53
|
+
];
|
|
54
|
+
const response = await provider.chat({
|
|
55
|
+
messages: summaryRequest,
|
|
56
|
+
model,
|
|
57
|
+
maxTokens: 1024,
|
|
58
|
+
temperature: 0.3,
|
|
59
|
+
});
|
|
60
|
+
return response.content;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Apply checkpointing to a message list for context assembly.
|
|
64
|
+
*
|
|
65
|
+
* Returns a modified message list where older messages are replaced
|
|
66
|
+
* with the checkpoint summary, and recent messages are kept verbatim.
|
|
67
|
+
*/
|
|
68
|
+
function applyCheckpoint(allMessages, checkpointSummary, keepRecent = exports.DEFAULT_CHECKPOINT_CONFIG.keepRecent) {
|
|
69
|
+
if (!checkpointSummary || allMessages.length <= keepRecent) {
|
|
70
|
+
return allMessages;
|
|
71
|
+
}
|
|
72
|
+
const systemMessages = allMessages.filter((m) => m.role === 'system');
|
|
73
|
+
const nonSystemMessages = allMessages.filter((m) => m.role !== 'system');
|
|
74
|
+
const recentMessages = nonSystemMessages.slice(-keepRecent);
|
|
75
|
+
const checkpointMessage = {
|
|
76
|
+
role: 'system',
|
|
77
|
+
content: `[Conversation Summary]\nThe following is a summary of the earlier part of this conversation:\n\n${checkpointSummary}\n\n[End of Summary — Recent messages follow]`,
|
|
78
|
+
};
|
|
79
|
+
return [...systemMessages, checkpointMessage, ...recentMessages];
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=checkpoint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint.js","sourceRoot":"","sources":["../../../src/core/optimizer/checkpoint.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;AAmBH,4CAMC;AAUD,8DAmCC;AAQD,0CAoBC;AAvFY,QAAA,yBAAyB,GAAqB;IACzD,QAAQ,EAAE,EAAE;IACZ,UAAU,EAAE,EAAE;CACf,CAAC;AAEF;;GAEG;AACH,SAAgB,gBAAgB,CAC9B,YAAoB,EACpB,0BAAkC,EAClC,SAA2B,iCAAyB;IAEpD,OAAO,YAAY,GAAG,0BAA0B,IAAI,MAAM,CAAC,QAAQ,CAAC;AACtE,CAAC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,yBAAyB,CAC7C,QAAuB,EACvB,QAAqB,EACrB,KAAa;IAEb,MAAM,gBAAgB,GAAG,QAAQ;SAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;SACvC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,cAAc,GAAkB;QACpC;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE;;;;;;;iEAOkD;SAC5D;QACD;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,mCAAmC,gBAAgB,EAAE;SAC/D;KACF,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC;QACnC,QAAQ,EAAE,cAAc;QACxB,KAAK;QACL,SAAS,EAAE,IAAI;QACf,WAAW,EAAE,GAAG;KACjB,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC,OAAO,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,SAAgB,eAAe,CAC7B,WAA0B,EAC1B,iBAAgC,EAChC,aAAqB,iCAAyB,CAAC,UAAU;IAEzD,IAAI,CAAC,iBAAiB,IAAI,WAAW,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;QAC3D,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;IACtE,MAAM,iBAAiB,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;IAEzE,MAAM,cAAc,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC;IAE5D,MAAM,iBAAiB,GAAgB;QACrC,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,mGAAmG,iBAAiB,+CAA+C;KAC7K,CAAC;IAEF,OAAO,CAAC,GAAG,cAAc,EAAE,iBAAiB,EAAE,GAAG,cAAc,CAAC,CAAC;AACnE,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Prompt Compression — Stage 1 of the Token Optimizer Pipeline
|
|
3
|
+
*
|
|
4
|
+
* Uses entropy-based filtering to remove low-information tokens.
|
|
5
|
+
* Tokens with high predictability (low perplexity) contribute little
|
|
6
|
+
* semantic value and can be dropped.
|
|
7
|
+
*
|
|
8
|
+
* This is a JS-native implementation inspired by LLMLingua, using
|
|
9
|
+
* precomputed token-importance heuristics instead of a local model.
|
|
10
|
+
*/
|
|
11
|
+
export interface CompressionResult {
|
|
12
|
+
original: string;
|
|
13
|
+
compressed: string;
|
|
14
|
+
tokensRaw: number;
|
|
15
|
+
tokensSent: number;
|
|
16
|
+
savingsPercent: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Compress a prompt by removing low-information content.
|
|
20
|
+
*
|
|
21
|
+
* @param text - The prompt text to compress.
|
|
22
|
+
* @param aggressiveness - 0 (no compression) to 1 (maximum compression). Default 0.3.
|
|
23
|
+
*/
|
|
24
|
+
export declare function compressPrompt(text: string, aggressiveness?: number): CompressionResult;
|
|
25
|
+
/**
|
|
26
|
+
* Compress an array of messages, preserving the most recent ones verbatim.
|
|
27
|
+
*
|
|
28
|
+
* @param messages - Array of message objects with role and content.
|
|
29
|
+
* @param preserveRecent - Number of recent messages to keep uncompressed. Default 4.
|
|
30
|
+
* @param aggressiveness - Compression level 0-1. Default 0.3.
|
|
31
|
+
*/
|
|
32
|
+
export declare function compressMessages(messages: Array<{
|
|
33
|
+
role: string;
|
|
34
|
+
content: string;
|
|
35
|
+
}>, preserveRecent?: number, aggressiveness?: number): Array<{
|
|
36
|
+
role: string;
|
|
37
|
+
content: string;
|
|
38
|
+
compressed: boolean;
|
|
39
|
+
savings: number;
|
|
40
|
+
}>;
|
|
41
|
+
//# sourceMappingURL=compressor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compressor.d.ts","sourceRoot":"","sources":["../../../src/core/optimizer/compressor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAyCH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;CACxB;AASD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,cAAc,GAAE,MAAY,GAC3B,iBAAiB,CA0DnB;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,EAClD,cAAc,GAAE,MAAU,EAC1B,cAAc,GAAE,MAAY,GAC3B,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAkBhF"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Prompt Compression — Stage 1 of the Token Optimizer Pipeline
|
|
4
|
+
*
|
|
5
|
+
* Uses entropy-based filtering to remove low-information tokens.
|
|
6
|
+
* Tokens with high predictability (low perplexity) contribute little
|
|
7
|
+
* semantic value and can be dropped.
|
|
8
|
+
*
|
|
9
|
+
* This is a JS-native implementation inspired by LLMLingua, using
|
|
10
|
+
* precomputed token-importance heuristics instead of a local model.
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.compressPrompt = compressPrompt;
|
|
14
|
+
exports.compressMessages = compressMessages;
|
|
15
|
+
// Common low-information words/patterns that can often be removed
|
|
16
|
+
// without significant semantic loss
|
|
17
|
+
const LOW_IMPORTANCE_PATTERNS = [
|
|
18
|
+
/\b(the|a|an)\b/gi,
|
|
19
|
+
/\b(is|are|was|were|be|been|being)\b/gi,
|
|
20
|
+
/\b(that|which|who|whom)\b/gi,
|
|
21
|
+
/\b(very|really|quite|rather|somewhat|fairly)\b/gi,
|
|
22
|
+
/\b(just|simply|basically|essentially|actually|literally)\b/gi,
|
|
23
|
+
/\b(in order to)\b/gi,
|
|
24
|
+
/\b(it is|there is|there are)\b/gi,
|
|
25
|
+
/\b(please|kindly)\b/gi,
|
|
26
|
+
];
|
|
27
|
+
// Phrases that can be shortened
|
|
28
|
+
const COMPRESSION_MAP = [
|
|
29
|
+
[/^hey there!?\s*/gi, 'hello '],
|
|
30
|
+
[/\bhow('?s| is) it going\??/gi, ''],
|
|
31
|
+
[/\bis there something i can help you with today\??/gi, 'How can I help?'],
|
|
32
|
+
[/\bis there anything specific you'd like help with\??/gi, 'What do you need help with?'],
|
|
33
|
+
[/\bi'?m an ai assistant!?\s*/gi, 'I can help with: '],
|
|
34
|
+
[/\bhere('?s| is) a quick (run ?down|overview) of what i can do:?/gi, 'Capabilities:'],
|
|
35
|
+
[/\bin order to\b/gi, 'to'],
|
|
36
|
+
[/\bas a result of\b/gi, 'because'],
|
|
37
|
+
[/\bdue to the fact that\b/gi, 'because'],
|
|
38
|
+
[/\bin the event that\b/gi, 'if'],
|
|
39
|
+
[/\bat this point in time\b/gi, 'now'],
|
|
40
|
+
[/\bin the near future\b/gi, 'soon'],
|
|
41
|
+
[/\bfor the purpose of\b/gi, 'to'],
|
|
42
|
+
[/\bin spite of the fact that\b/gi, 'although'],
|
|
43
|
+
[/\bwith regard to\b/gi, 'about'],
|
|
44
|
+
[/\bin reference to\b/gi, 'about'],
|
|
45
|
+
[/\bas a matter of fact\b/gi, 'actually'],
|
|
46
|
+
[/\bit should be noted that\b/gi, ''],
|
|
47
|
+
[/\bit is worth mentioning that\b/gi, ''],
|
|
48
|
+
[/\bit is important to note that\b/gi, ''],
|
|
49
|
+
[/\bas I mentioned (earlier|before|previously)\b/gi, ''],
|
|
50
|
+
[/\bas (we|I) discussed (earlier|before|previously)\b/gi, ''],
|
|
51
|
+
];
|
|
52
|
+
/**
|
|
53
|
+
* Rough token count estimation (~4 chars/token average).
|
|
54
|
+
*/
|
|
55
|
+
function estimateTokens(text) {
|
|
56
|
+
return Math.ceil(text.split(/\s+/).length * 1.3);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Compress a prompt by removing low-information content.
|
|
60
|
+
*
|
|
61
|
+
* @param text - The prompt text to compress.
|
|
62
|
+
* @param aggressiveness - 0 (no compression) to 1 (maximum compression). Default 0.3.
|
|
63
|
+
*/
|
|
64
|
+
function compressPrompt(text, aggressiveness = 0.3) {
|
|
65
|
+
const tokensRaw = estimateTokens(text);
|
|
66
|
+
let result = text;
|
|
67
|
+
// Always apply phrase compression (these are lossless transformations)
|
|
68
|
+
for (const [pattern, replacement] of COMPRESSION_MAP) {
|
|
69
|
+
result = result.replace(pattern, replacement);
|
|
70
|
+
}
|
|
71
|
+
if (aggressiveness > 0.4) {
|
|
72
|
+
// Strip markdown formatting from older context while preserving meaning
|
|
73
|
+
result = result
|
|
74
|
+
.replace(/^#{1,6}\s+/gm, '')
|
|
75
|
+
.replace(/^\s*[-*+]\s+/gm, '')
|
|
76
|
+
.replace(/\*\*(.*?)\*\*/g, '$1')
|
|
77
|
+
.replace(/\*(.*?)\*/g, '$1')
|
|
78
|
+
.replace(/`([^`]+)`/g, '$1');
|
|
79
|
+
}
|
|
80
|
+
// Apply word-level compression proportional to aggressiveness
|
|
81
|
+
if (aggressiveness > 0.2) {
|
|
82
|
+
// Remove filler words
|
|
83
|
+
const fillerPatterns = LOW_IMPORTANCE_PATTERNS.slice(3); // very, really, just, etc.
|
|
84
|
+
for (const pattern of fillerPatterns) {
|
|
85
|
+
result = result.replace(pattern, '');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (aggressiveness > 0.5) {
|
|
89
|
+
// Remove articles (more aggressive)
|
|
90
|
+
result = result.replace(/\b(the|a|an)\s+/gi, '');
|
|
91
|
+
// Remove additional low-information discourse words
|
|
92
|
+
result = result.replace(/\b(i|you|we|they|it|this|that|these|those|here|there|today|now|then|also|well|okay|great|sure)\b/gi, '');
|
|
93
|
+
}
|
|
94
|
+
if (aggressiveness > 0.7) {
|
|
95
|
+
// Remove copulas where surrounding context is clear
|
|
96
|
+
result = result.replace(/\b(is|are|was|were)\s+/gi, ' ');
|
|
97
|
+
}
|
|
98
|
+
// Clean up extra whitespace
|
|
99
|
+
result = result.replace(/\s{2,}/g, ' ').trim();
|
|
100
|
+
const tokensSent = estimateTokens(result);
|
|
101
|
+
const savingsPercent = tokensRaw > 0 ? Math.round(((tokensRaw - tokensSent) / tokensRaw) * 100) : 0;
|
|
102
|
+
return {
|
|
103
|
+
original: text,
|
|
104
|
+
compressed: result,
|
|
105
|
+
tokensRaw,
|
|
106
|
+
tokensSent,
|
|
107
|
+
savingsPercent: Math.max(0, savingsPercent),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Compress an array of messages, preserving the most recent ones verbatim.
|
|
112
|
+
*
|
|
113
|
+
* @param messages - Array of message objects with role and content.
|
|
114
|
+
* @param preserveRecent - Number of recent messages to keep uncompressed. Default 4.
|
|
115
|
+
* @param aggressiveness - Compression level 0-1. Default 0.3.
|
|
116
|
+
*/
|
|
117
|
+
function compressMessages(messages, preserveRecent = 4, aggressiveness = 0.3) {
|
|
118
|
+
return messages.map((msg, i) => {
|
|
119
|
+
const isRecent = i >= messages.length - preserveRecent;
|
|
120
|
+
const isSystem = msg.role === 'system';
|
|
121
|
+
// Never compress system messages or recent messages
|
|
122
|
+
if (isSystem || isRecent) {
|
|
123
|
+
return { ...msg, compressed: false, savings: 0 };
|
|
124
|
+
}
|
|
125
|
+
const result = compressPrompt(msg.content, aggressiveness);
|
|
126
|
+
return {
|
|
127
|
+
role: msg.role,
|
|
128
|
+
content: result.compressed,
|
|
129
|
+
compressed: true,
|
|
130
|
+
savings: result.savingsPercent,
|
|
131
|
+
};
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=compressor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compressor.js","sourceRoot":"","sources":["../../../src/core/optimizer/compressor.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;AA8DH,wCA6DC;AASD,4CAsBC;AAxJD,kEAAkE;AAClE,oCAAoC;AACpC,MAAM,uBAAuB,GAAa;IACxC,kBAAkB;IAClB,uCAAuC;IACvC,6BAA6B;IAC7B,kDAAkD;IAClD,8DAA8D;IAC9D,qBAAqB;IACrB,kCAAkC;IAClC,uBAAuB;CACxB,CAAC;AAEF,gCAAgC;AAChC,MAAM,eAAe,GAAuB;IAC1C,CAAC,mBAAmB,EAAE,QAAQ,CAAC;IAC/B,CAAC,8BAA8B,EAAE,EAAE,CAAC;IACpC,CAAC,qDAAqD,EAAE,iBAAiB,CAAC;IAC1E,CAAC,wDAAwD,EAAE,6BAA6B,CAAC;IACzF,CAAC,+BAA+B,EAAE,mBAAmB,CAAC;IACtD,CAAC,mEAAmE,EAAE,eAAe,CAAC;IACtF,CAAC,mBAAmB,EAAE,IAAI,CAAC;IAC3B,CAAC,sBAAsB,EAAE,SAAS,CAAC;IACnC,CAAC,4BAA4B,EAAE,SAAS,CAAC;IACzC,CAAC,yBAAyB,EAAE,IAAI,CAAC;IACjC,CAAC,6BAA6B,EAAE,KAAK,CAAC;IACtC,CAAC,0BAA0B,EAAE,MAAM,CAAC;IACpC,CAAC,0BAA0B,EAAE,IAAI,CAAC;IAClC,CAAC,iCAAiC,EAAE,UAAU,CAAC;IAC/C,CAAC,sBAAsB,EAAE,OAAO,CAAC;IACjC,CAAC,uBAAuB,EAAE,OAAO,CAAC;IAClC,CAAC,2BAA2B,EAAE,UAAU,CAAC;IACzC,CAAC,+BAA+B,EAAE,EAAE,CAAC;IACrC,CAAC,mCAAmC,EAAE,EAAE,CAAC;IACzC,CAAC,oCAAoC,EAAE,EAAE,CAAC;IAC1C,CAAC,kDAAkD,EAAE,EAAE,CAAC;IACxD,CAAC,uDAAuD,EAAE,EAAE,CAAC;CAC9D,CAAC;AAUF;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;AACnD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,cAAc,CAC5B,IAAY,EACZ,iBAAyB,GAAG;IAE5B,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,MAAM,GAAG,IAAI,CAAC;IAElB,uEAAuE;IACvE,KAAK,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,eAAe,EAAE,CAAC;QACrD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAChD,CAAC;IAED,IAAI,cAAc,GAAG,GAAG,EAAE,CAAC;QACzB,wEAAwE;QACxE,MAAM,GAAG,MAAM;aACZ,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;aAC3B,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC;aAC7B,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;aAC/B,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;aAC3B,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,8DAA8D;IAC9D,IAAI,cAAc,GAAG,GAAG,EAAE,CAAC;QACzB,sBAAsB;QACtB,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,2BAA2B;QACpF,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,IAAI,cAAc,GAAG,GAAG,EAAE,CAAC;QACzB,oCAAoC;QACpC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QAEjD,oDAAoD;QACpD,MAAM,GAAG,MAAM,CAAC,OAAO,CACrB,oGAAoG,EACpG,EAAE,CACH,CAAC;IACJ,CAAC;IAED,IAAI,cAAc,GAAG,GAAG,EAAE,CAAC;QACzB,oDAAoD;QACpD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,0BAA0B,EAAE,GAAG,CAAC,CAAC;IAC3D,CAAC;IAED,4BAA4B;IAC5B,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAE/C,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,cAAc,GAClB,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/E,OAAO;QACL,QAAQ,EAAE,IAAI;QACd,UAAU,EAAE,MAAM;QAClB,SAAS;QACT,UAAU;QACV,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,gBAAgB,CAC9B,QAAkD,EAClD,iBAAyB,CAAC,EAC1B,iBAAyB,GAAG;IAE5B,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QAC7B,MAAM,QAAQ,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC;QACvD,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC;QAEvC,oDAAoD;QACpD,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;YACzB,OAAO,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACnD,CAAC;QAED,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QAC3D,OAAO;YACL,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,OAAO,EAAE,MAAM,CAAC,UAAU;YAC1B,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,MAAM,CAAC,cAAc;SAC/B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Semantic Deduplication — Stage 2 of the Token Optimizer Pipeline
|
|
3
|
+
*
|
|
4
|
+
* Detects near-duplicate messages via cosine similarity on embeddings.
|
|
5
|
+
* Redundant turns are merged or dropped, keeping only the most
|
|
6
|
+
* recent/complete version.
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Compute cosine similarity between two vectors.
|
|
10
|
+
*/
|
|
11
|
+
export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;
|
|
12
|
+
export interface DedupMessage {
|
|
13
|
+
id: string;
|
|
14
|
+
role: string;
|
|
15
|
+
content: string;
|
|
16
|
+
embedding?: Float32Array;
|
|
17
|
+
created_at: number;
|
|
18
|
+
}
|
|
19
|
+
export interface DedupResult {
|
|
20
|
+
kept: DedupMessage[];
|
|
21
|
+
removed: DedupMessage[];
|
|
22
|
+
duplicateGroups: Array<{
|
|
23
|
+
kept: string;
|
|
24
|
+
removed: string[];
|
|
25
|
+
}>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Deduplicate messages by semantic similarity.
|
|
29
|
+
*
|
|
30
|
+
* @param messages - Messages with optional embeddings.
|
|
31
|
+
* @param threshold - Cosine similarity threshold (0-1). Above this = duplicate. Default 0.85.
|
|
32
|
+
* @returns Deduplicated message list, preferring more recent/longer messages.
|
|
33
|
+
*/
|
|
34
|
+
export declare function deduplicateMessages(messages: DedupMessage[], threshold?: number): DedupResult;
|
|
35
|
+
/**
|
|
36
|
+
* Text-only deduplication without embeddings.
|
|
37
|
+
* Uses normalized text comparison and Jaccard similarity.
|
|
38
|
+
*/
|
|
39
|
+
export declare function deduplicateByText(messages: Array<{
|
|
40
|
+
id: string;
|
|
41
|
+
role: string;
|
|
42
|
+
content: string;
|
|
43
|
+
created_at: number;
|
|
44
|
+
}>, jaccardThreshold?: number): {
|
|
45
|
+
kept: typeof messages;
|
|
46
|
+
removed: typeof messages;
|
|
47
|
+
};
|
|
48
|
+
//# sourceMappingURL=dedup.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../../src/core/optimizer/dedup.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAgBzE;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,YAAY,EAAE,CAAC;IACrB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,eAAe,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;CAC7D;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,YAAY,EAAE,EACxB,SAAS,GAAE,MAAa,GACvB,WAAW,CAgEb;AAaD;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC,EAClF,gBAAgB,GAAE,MAAY,GAC7B;IAAE,IAAI,EAAE,OAAO,QAAQ,CAAC;IAAC,OAAO,EAAE,OAAO,QAAQ,CAAA;CAAE,CA4BrD"}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Semantic Deduplication — Stage 2 of the Token Optimizer Pipeline
|
|
4
|
+
*
|
|
5
|
+
* Detects near-duplicate messages via cosine similarity on embeddings.
|
|
6
|
+
* Redundant turns are merged or dropped, keeping only the most
|
|
7
|
+
* recent/complete version.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.cosineSimilarity = cosineSimilarity;
|
|
11
|
+
exports.deduplicateMessages = deduplicateMessages;
|
|
12
|
+
exports.deduplicateByText = deduplicateByText;
|
|
13
|
+
/**
|
|
14
|
+
* Compute cosine similarity between two vectors.
|
|
15
|
+
*/
|
|
16
|
+
function cosineSimilarity(a, b) {
|
|
17
|
+
if (a.length !== b.length)
|
|
18
|
+
return 0;
|
|
19
|
+
let dotProduct = 0;
|
|
20
|
+
let normA = 0;
|
|
21
|
+
let normB = 0;
|
|
22
|
+
for (let i = 0; i < a.length; i++) {
|
|
23
|
+
dotProduct += a[i] * b[i];
|
|
24
|
+
normA += a[i] * a[i];
|
|
25
|
+
normB += b[i] * b[i];
|
|
26
|
+
}
|
|
27
|
+
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
|
28
|
+
if (denominator === 0)
|
|
29
|
+
return 0;
|
|
30
|
+
return dotProduct / denominator;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Deduplicate messages by semantic similarity.
|
|
34
|
+
*
|
|
35
|
+
* @param messages - Messages with optional embeddings.
|
|
36
|
+
* @param threshold - Cosine similarity threshold (0-1). Above this = duplicate. Default 0.85.
|
|
37
|
+
* @returns Deduplicated message list, preferring more recent/longer messages.
|
|
38
|
+
*/
|
|
39
|
+
function deduplicateMessages(messages, threshold = 0.85) {
|
|
40
|
+
const kept = [];
|
|
41
|
+
const removed = [];
|
|
42
|
+
const duplicateGroups = [];
|
|
43
|
+
// Process messages in reverse chronological order (newest first)
|
|
44
|
+
const sorted = [...messages].sort((a, b) => b.created_at - a.created_at);
|
|
45
|
+
for (const msg of sorted) {
|
|
46
|
+
// System messages are never deduplicated
|
|
47
|
+
if (msg.role === 'system') {
|
|
48
|
+
kept.push(msg);
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
// Without embeddings, check exact/near-exact text match
|
|
52
|
+
if (!msg.embedding) {
|
|
53
|
+
const isDuplicate = kept.some((k) => k.role === msg.role &&
|
|
54
|
+
(k.content === msg.content ||
|
|
55
|
+
normalizeText(k.content) === normalizeText(msg.content)));
|
|
56
|
+
if (isDuplicate) {
|
|
57
|
+
removed.push(msg);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
kept.push(msg);
|
|
61
|
+
}
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
// With embeddings, use cosine similarity
|
|
65
|
+
let bestMatch = null;
|
|
66
|
+
let bestSimilarity = 0;
|
|
67
|
+
for (const k of kept) {
|
|
68
|
+
if (k.role !== msg.role || !k.embedding)
|
|
69
|
+
continue;
|
|
70
|
+
const similarity = cosineSimilarity(msg.embedding, k.embedding);
|
|
71
|
+
if (similarity > bestSimilarity) {
|
|
72
|
+
bestSimilarity = similarity;
|
|
73
|
+
bestMatch = k;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (bestMatch && bestSimilarity >= threshold) {
|
|
77
|
+
removed.push(msg);
|
|
78
|
+
// Track the group
|
|
79
|
+
const existingGroup = duplicateGroups.find((g) => g.kept === bestMatch.id);
|
|
80
|
+
if (existingGroup) {
|
|
81
|
+
existingGroup.removed.push(msg.id);
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
duplicateGroups.push({ kept: bestMatch.id, removed: [msg.id] });
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
kept.push(msg);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
// Restore chronological order for kept messages
|
|
92
|
+
kept.sort((a, b) => a.created_at - b.created_at);
|
|
93
|
+
return { kept, removed, duplicateGroups };
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Simple text normalization for near-exact matching.
|
|
97
|
+
*/
|
|
98
|
+
function normalizeText(text) {
|
|
99
|
+
return text
|
|
100
|
+
.toLowerCase()
|
|
101
|
+
.replace(/\s+/g, ' ')
|
|
102
|
+
.replace(/[^\w\s]/g, '')
|
|
103
|
+
.trim();
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Text-only deduplication without embeddings.
|
|
107
|
+
* Uses normalized text comparison and Jaccard similarity.
|
|
108
|
+
*/
|
|
109
|
+
function deduplicateByText(messages, jaccardThreshold = 0.7) {
|
|
110
|
+
const kept = [];
|
|
111
|
+
const removed = [];
|
|
112
|
+
// Process newest first
|
|
113
|
+
const sorted = [...messages].sort((a, b) => b.created_at - a.created_at);
|
|
114
|
+
for (const msg of sorted) {
|
|
115
|
+
if (msg.role === 'system') {
|
|
116
|
+
kept.push(msg);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
const isDuplicate = kept.some((k) => {
|
|
120
|
+
if (k.role !== msg.role)
|
|
121
|
+
return false;
|
|
122
|
+
const sim = jaccardSimilarity(msg.content, k.content);
|
|
123
|
+
return sim >= jaccardThreshold;
|
|
124
|
+
});
|
|
125
|
+
if (isDuplicate) {
|
|
126
|
+
removed.push(msg);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
kept.push(msg);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
kept.sort((a, b) => a.created_at - b.created_at);
|
|
133
|
+
return { kept, removed };
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Compute Jaccard similarity between two texts based on word sets.
|
|
137
|
+
*/
|
|
138
|
+
function jaccardSimilarity(a, b) {
|
|
139
|
+
const setA = new Set(normalizeText(a).split(' '));
|
|
140
|
+
const setB = new Set(normalizeText(b).split(' '));
|
|
141
|
+
const intersection = new Set([...setA].filter((x) => setB.has(x)));
|
|
142
|
+
const union = new Set([...setA, ...setB]);
|
|
143
|
+
if (union.size === 0)
|
|
144
|
+
return 0;
|
|
145
|
+
return intersection.size / union.size;
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=dedup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../../src/core/optimizer/dedup.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;AAKH,4CAgBC;AAuBD,kDAmEC;AAiBD,8CA+BC;AA7JD;;GAEG;AACH,SAAgB,gBAAgB,CAAC,CAAe,EAAE,CAAe;IAC/D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IAEpC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxD,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,UAAU,GAAG,WAAW,CAAC;AAClC,CAAC;AAgBD;;;;;;GAMG;AACH,SAAgB,mBAAmB,CACjC,QAAwB,EACxB,YAAoB,IAAI;IAExB,MAAM,IAAI,GAAmB,EAAE,CAAC;IAChC,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,eAAe,GAA+C,EAAE,CAAC;IAEvE,iEAAiE;IACjE,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEzE,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,yCAAyC;QACzC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QAED,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAC3B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI;gBACnB,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAC,OAAO;oBACxB,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAC7D,CAAC;YAEF,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,CAAC;YACD,SAAS;QACX,CAAC;QAED,yCAAyC;QACzC,IAAI,SAAS,GAAwB,IAAI,CAAC;QAC1C,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS;gBAAE,SAAS;YAElD,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;YAChE,IAAI,UAAU,GAAG,cAAc,EAAE,CAAC;gBAChC,cAAc,GAAG,UAAU,CAAC;gBAC5B,SAAS,GAAG,CAAC,CAAC;YAChB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,IAAI,cAAc,IAAI,SAAS,EAAE,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,kBAAkB;YAClB,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAU,CAAC,EAAE,CAAC,CAAC;YAC5E,IAAI,aAAa,EAAE,CAAC;gBAClB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,eAAe,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,gDAAgD;IAChD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEjD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAgB,iBAAiB,CAC/B,QAAkF,EAClF,mBAA2B,GAAG;IAE9B,MAAM,IAAI,GAAoB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAoB,EAAE,CAAC;IAEpC,uBAAuB;IACvB,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEzE,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YAClC,IAAI,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAC;YACtC,MAAM,GAAG,GAAG,iBAAiB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;YACtD,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,IAAI,WAAW,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IACjD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,CAAS,EAAE,CAAS;IAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAElD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;IAE1C,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,OAAO,YAAY,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;AACxC,CAAC"}
|