@blockrun/clawrouter 0.12.102 → 0.12.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +0 -15
- package/dist/cli.js +80037 -407
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +1406 -34
- package/dist/index.js +81085 -1075
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/auth.d.ts +0 -82
- package/dist/auth.d.ts.map +0 -1
- package/dist/auth.js +0 -388
- package/dist/auth.js.map +0 -1
- package/dist/balance.d.ts +0 -97
- package/dist/balance.d.ts.map +0 -1
- package/dist/balance.js +0 -157
- package/dist/balance.js.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/compression/codebook.d.ts +0 -24
- package/dist/compression/codebook.d.ts.map +0 -1
- package/dist/compression/codebook.js +0 -119
- package/dist/compression/codebook.js.map +0 -1
- package/dist/compression/index.d.ts +0 -33
- package/dist/compression/index.d.ts.map +0 -1
- package/dist/compression/index.js +0 -246
- package/dist/compression/index.js.map +0 -1
- package/dist/compression/layers/deduplication.d.ts +0 -28
- package/dist/compression/layers/deduplication.d.ts.map +0 -1
- package/dist/compression/layers/deduplication.js +0 -101
- package/dist/compression/layers/deduplication.js.map +0 -1
- package/dist/compression/layers/dictionary.d.ts +0 -21
- package/dist/compression/layers/dictionary.d.ts.map +0 -1
- package/dist/compression/layers/dictionary.js +0 -73
- package/dist/compression/layers/dictionary.js.map +0 -1
- package/dist/compression/layers/dynamic-codebook.d.ts +0 -26
- package/dist/compression/layers/dynamic-codebook.d.ts.map +0 -1
- package/dist/compression/layers/dynamic-codebook.js +0 -149
- package/dist/compression/layers/dynamic-codebook.js.map +0 -1
- package/dist/compression/layers/json-compact.d.ts +0 -23
- package/dist/compression/layers/json-compact.d.ts.map +0 -1
- package/dist/compression/layers/json-compact.js +0 -79
- package/dist/compression/layers/json-compact.js.map +0 -1
- package/dist/compression/layers/observation.d.ts +0 -21
- package/dist/compression/layers/observation.d.ts.map +0 -1
- package/dist/compression/layers/observation.js +0 -130
- package/dist/compression/layers/observation.js.map +0 -1
- package/dist/compression/layers/paths.d.ts +0 -24
- package/dist/compression/layers/paths.d.ts.map +0 -1
- package/dist/compression/layers/paths.js +0 -110
- package/dist/compression/layers/paths.js.map +0 -1
- package/dist/compression/layers/whitespace.d.ts +0 -27
- package/dist/compression/layers/whitespace.d.ts.map +0 -1
- package/dist/compression/layers/whitespace.js +0 -60
- package/dist/compression/layers/whitespace.js.map +0 -1
- package/dist/compression/types.d.ts +0 -84
- package/dist/compression/types.d.ts.map +0 -1
- package/dist/compression/types.js +0 -27
- package/dist/compression/types.js.map +0 -1
- package/dist/config.d.ts +0 -12
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -22
- package/dist/config.js.map +0 -1
- package/dist/dedup.d.ts +0 -34
- package/dist/dedup.d.ts.map +0 -1
- package/dist/dedup.js +0 -150
- package/dist/dedup.js.map +0 -1
- package/dist/doctor.d.ts +0 -8
- package/dist/doctor.d.ts.map +0 -1
- package/dist/doctor.js +0 -375
- package/dist/doctor.js.map +0 -1
- package/dist/error-classification.test.d.ts +0 -2
- package/dist/error-classification.test.d.ts.map +0 -1
- package/dist/error-classification.test.js +0 -59
- package/dist/error-classification.test.js.map +0 -1
- package/dist/errors.d.ts +0 -54
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js +0 -85
- package/dist/errors.js.map +0 -1
- package/dist/exclude-models.d.ts +0 -28
- package/dist/exclude-models.d.ts.map +0 -1
- package/dist/exclude-models.integration.test.d.ts +0 -2
- package/dist/exclude-models.integration.test.d.ts.map +0 -1
- package/dist/exclude-models.integration.test.js +0 -46
- package/dist/exclude-models.integration.test.js.map +0 -1
- package/dist/exclude-models.js +0 -70
- package/dist/exclude-models.js.map +0 -1
- package/dist/exclude-models.test.d.ts +0 -5
- package/dist/exclude-models.test.d.ts.map +0 -1
- package/dist/exclude-models.test.js +0 -95
- package/dist/exclude-models.test.js.map +0 -1
- package/dist/fs-read.d.ts +0 -11
- package/dist/fs-read.d.ts.map +0 -1
- package/dist/fs-read.js +0 -47
- package/dist/fs-read.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/journal.d.ts +0 -67
- package/dist/journal.d.ts.map +0 -1
- package/dist/journal.js +0 -184
- package/dist/journal.js.map +0 -1
- package/dist/journal.test.d.ts +0 -2
- package/dist/journal.test.d.ts.map +0 -1
- package/dist/journal.test.js +0 -201
- package/dist/journal.test.js.map +0 -1
- package/dist/logger.d.ts +0 -33
- package/dist/logger.d.ts.map +0 -1
- package/dist/logger.js +0 -35
- package/dist/logger.js.map +0 -1
- package/dist/models.d.ts +0 -107
- package/dist/models.d.ts.map +0 -1
- package/dist/models.js +0 -941
- package/dist/models.js.map +0 -1
- package/dist/models.test.d.ts +0 -2
- package/dist/models.test.d.ts.map +0 -1
- package/dist/models.test.js +0 -38
- package/dist/models.test.js.map +0 -1
- package/dist/partners/index.d.ts +0 -8
- package/dist/partners/index.d.ts.map +0 -1
- package/dist/partners/index.js +0 -6
- package/dist/partners/index.js.map +0 -1
- package/dist/partners/registry.d.ts +0 -51
- package/dist/partners/registry.d.ts.map +0 -1
- package/dist/partners/registry.js +0 -256
- package/dist/partners/registry.js.map +0 -1
- package/dist/partners/tools.d.ts +0 -24
- package/dist/partners/tools.d.ts.map +0 -1
- package/dist/partners/tools.js +0 -97
- package/dist/partners/tools.js.map +0 -1
- package/dist/payment-preauth.d.ts +0 -17
- package/dist/payment-preauth.d.ts.map +0 -1
- package/dist/payment-preauth.js +0 -102
- package/dist/payment-preauth.js.map +0 -1
- package/dist/provider.d.ts +0 -19
- package/dist/provider.d.ts.map +0 -1
- package/dist/provider.js +0 -45
- package/dist/provider.js.map +0 -1
- package/dist/proxy.d.ts +0 -210
- package/dist/proxy.d.ts.map +0 -1
- package/dist/proxy.debrand.test.d.ts +0 -2
- package/dist/proxy.debrand.test.d.ts.map +0 -1
- package/dist/proxy.debrand.test.js +0 -101
- package/dist/proxy.debrand.test.js.map +0 -1
- package/dist/proxy.degraded-response.test.d.ts +0 -2
- package/dist/proxy.degraded-response.test.d.ts.map +0 -1
- package/dist/proxy.degraded-response.test.js +0 -105
- package/dist/proxy.degraded-response.test.js.map +0 -1
- package/dist/proxy.image-download.test.d.ts +0 -2
- package/dist/proxy.image-download.test.d.ts.map +0 -1
- package/dist/proxy.image-download.test.js +0 -92
- package/dist/proxy.image-download.test.js.map +0 -1
- package/dist/proxy.js +0 -3810
- package/dist/proxy.js.map +0 -1
- package/dist/proxy.models-endpoint.test.d.ts +0 -2
- package/dist/proxy.models-endpoint.test.d.ts.map +0 -1
- package/dist/proxy.models-endpoint.test.js +0 -19
- package/dist/proxy.models-endpoint.test.js.map +0 -1
- package/dist/proxy.multiturn-thinking.test.d.ts +0 -2
- package/dist/proxy.multiturn-thinking.test.d.ts.map +0 -1
- package/dist/proxy.multiturn-thinking.test.js +0 -82
- package/dist/proxy.multiturn-thinking.test.js.map +0 -1
- package/dist/proxy.payment-chain-reuse.test.d.ts +0 -2
- package/dist/proxy.payment-chain-reuse.test.d.ts.map +0 -1
- package/dist/proxy.payment-chain-reuse.test.js +0 -30
- package/dist/proxy.payment-chain-reuse.test.js.map +0 -1
- package/dist/proxy.solana-resilience.test.d.ts +0 -12
- package/dist/proxy.solana-resilience.test.d.ts.map +0 -1
- package/dist/proxy.solana-resilience.test.js +0 -39
- package/dist/proxy.solana-resilience.test.js.map +0 -1
- package/dist/report.d.ts +0 -6
- package/dist/report.d.ts.map +0 -1
- package/dist/report.js +0 -44
- package/dist/report.js.map +0 -1
- package/dist/response-cache.advanced.test.d.ts +0 -2
- package/dist/response-cache.advanced.test.d.ts.map +0 -1
- package/dist/response-cache.advanced.test.js +0 -566
- package/dist/response-cache.advanced.test.js.map +0 -1
- package/dist/response-cache.d.ts +0 -85
- package/dist/response-cache.d.ts.map +0 -1
- package/dist/response-cache.edge-cases.test.d.ts +0 -2
- package/dist/response-cache.edge-cases.test.d.ts.map +0 -1
- package/dist/response-cache.edge-cases.test.js +0 -488
- package/dist/response-cache.edge-cases.test.js.map +0 -1
- package/dist/response-cache.extreme.test.d.ts +0 -2
- package/dist/response-cache.extreme.test.d.ts.map +0 -1
- package/dist/response-cache.extreme.test.js +0 -561
- package/dist/response-cache.extreme.test.js.map +0 -1
- package/dist/response-cache.js +0 -240
- package/dist/response-cache.js.map +0 -1
- package/dist/response-cache.test.d.ts +0 -2
- package/dist/response-cache.test.d.ts.map +0 -1
- package/dist/response-cache.test.js +0 -242
- package/dist/response-cache.test.js.map +0 -1
- package/dist/retry.d.ts +0 -42
- package/dist/retry.d.ts.map +0 -1
- package/dist/retry.js +0 -99
- package/dist/retry.js.map +0 -1
- package/dist/router/config.d.ts +0 -11
- package/dist/router/config.d.ts.map +0 -1
- package/dist/router/config.js +0 -1215
- package/dist/router/config.js.map +0 -1
- package/dist/router/index.d.ts +0 -18
- package/dist/router/index.d.ts.map +0 -1
- package/dist/router/index.js +0 -19
- package/dist/router/index.js.map +0 -1
- package/dist/router/llm-classifier.d.ts +0 -29
- package/dist/router/llm-classifier.d.ts.map +0 -1
- package/dist/router/llm-classifier.js +0 -99
- package/dist/router/llm-classifier.js.map +0 -1
- package/dist/router/rules.d.ts +0 -12
- package/dist/router/rules.d.ts.map +0 -1
- package/dist/router/rules.js +0 -202
- package/dist/router/rules.js.map +0 -1
- package/dist/router/selector.d.ts +0 -58
- package/dist/router/selector.d.ts.map +0 -1
- package/dist/router/selector.js +0 -165
- package/dist/router/selector.js.map +0 -1
- package/dist/router/selector.test.d.ts +0 -2
- package/dist/router/selector.test.d.ts.map +0 -1
- package/dist/router/selector.test.js +0 -67
- package/dist/router/selector.test.js.map +0 -1
- package/dist/router/strategy.d.ts +0 -19
- package/dist/router/strategy.d.ts.map +0 -1
- package/dist/router/strategy.js +0 -138
- package/dist/router/strategy.js.map +0 -1
- package/dist/router/strategy.test.d.ts +0 -2
- package/dist/router/strategy.test.d.ts.map +0 -1
- package/dist/router/strategy.test.js +0 -131
- package/dist/router/strategy.test.js.map +0 -1
- package/dist/router/types.d.ts +0 -127
- package/dist/router/types.d.ts.map +0 -1
- package/dist/router/types.js +0 -11
- package/dist/router/types.js.map +0 -1
- package/dist/session.d.ts +0 -124
- package/dist/session.d.ts.map +0 -1
- package/dist/session.js +0 -256
- package/dist/session.js.map +0 -1
- package/dist/session.test.d.ts +0 -2
- package/dist/session.test.d.ts.map +0 -1
- package/dist/session.test.js +0 -163
- package/dist/session.test.js.map +0 -1
- package/dist/solana-balance.d.ts +0 -48
- package/dist/solana-balance.d.ts.map +0 -1
- package/dist/solana-balance.js +0 -142
- package/dist/solana-balance.js.map +0 -1
- package/dist/spend-control.d.ts +0 -112
- package/dist/spend-control.d.ts.map +0 -1
- package/dist/spend-control.js +0 -278
- package/dist/spend-control.js.map +0 -1
- package/dist/spend-control.test.d.ts +0 -5
- package/dist/spend-control.test.d.ts.map +0 -1
- package/dist/spend-control.test.js +0 -210
- package/dist/spend-control.test.js.map +0 -1
- package/dist/stats.d.ts +0 -64
- package/dist/stats.d.ts.map +0 -1
- package/dist/stats.js +0 -295
- package/dist/stats.js.map +0 -1
- package/dist/types.d.ts +0 -150
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -9
- package/dist/types.js.map +0 -1
- package/dist/update-hint.test.d.ts +0 -9
- package/dist/update-hint.test.d.ts.map +0 -1
- package/dist/update-hint.test.js +0 -156
- package/dist/update-hint.test.js.map +0 -1
- package/dist/updater.d.ts +0 -10
- package/dist/updater.d.ts.map +0 -1
- package/dist/updater.js +0 -55
- package/dist/updater.js.map +0 -1
- package/dist/version.d.ts +0 -3
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js +0 -16
- package/dist/version.js.map +0 -1
- package/dist/wallet.d.ts +0 -52
- package/dist/wallet.d.ts.map +0 -1
- package/dist/wallet.js +0 -94
- package/dist/wallet.js.map +0 -1
- package/dist/wallet.test.d.ts +0 -6
- package/dist/wallet.test.d.ts.map +0 -1
- package/dist/wallet.test.js +0 -117
- package/dist/wallet.test.js.map +0 -1
- package/dist/x402-sdk.test.d.ts +0 -6
- package/dist/x402-sdk.test.d.ts.map +0 -1
- package/dist/x402-sdk.test.js +0 -161
- package/dist/x402-sdk.test.js.map +0 -1
package/dist/balance.js
DELETED
|
@@ -1,157 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Balance Monitor for ClawRouter
|
|
3
|
-
*
|
|
4
|
-
* Monitors USDC balance on Base network with intelligent caching.
|
|
5
|
-
* Provides pre-request balance checks to prevent failed payments.
|
|
6
|
-
*
|
|
7
|
-
* Caching Strategy:
|
|
8
|
-
* - TTL: 30 seconds (balance is cached to avoid excessive RPC calls)
|
|
9
|
-
* - Optimistic deduction: after successful payment, subtract estimated cost from cache
|
|
10
|
-
* - Invalidation: on payment failure, immediately refresh from RPC
|
|
11
|
-
*/
|
|
12
|
-
import { createPublicClient, http, erc20Abi } from "viem";
|
|
13
|
-
import { base } from "viem/chains";
|
|
14
|
-
import { RpcError } from "./errors.js";
|
|
15
|
-
/** USDC contract address on Base mainnet */
|
|
16
|
-
const USDC_BASE = "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913";
|
|
17
|
-
/** Cache TTL in milliseconds (30 seconds) */
|
|
18
|
-
const CACHE_TTL_MS = 30_000;
|
|
19
|
-
/** Balance thresholds in USDC smallest unit (6 decimals) */
|
|
20
|
-
export const BALANCE_THRESHOLDS = {
|
|
21
|
-
/** Low balance warning threshold: $1.00 */
|
|
22
|
-
LOW_BALANCE_MICROS: 1000000n,
|
|
23
|
-
/** Effectively zero threshold: $0.0001 (covers dust/rounding) */
|
|
24
|
-
ZERO_THRESHOLD: 100n,
|
|
25
|
-
};
|
|
26
|
-
/**
|
|
27
|
-
* Monitors USDC balance on Base network.
|
|
28
|
-
*
|
|
29
|
-
* Usage:
|
|
30
|
-
* const monitor = new BalanceMonitor("0x...");
|
|
31
|
-
* const info = await monitor.checkBalance();
|
|
32
|
-
* if (info.isLow) console.warn("Low balance!");
|
|
33
|
-
*/
|
|
34
|
-
export class BalanceMonitor {
|
|
35
|
-
client;
|
|
36
|
-
walletAddress;
|
|
37
|
-
/** Cached balance (null = not yet fetched) */
|
|
38
|
-
cachedBalance = null;
|
|
39
|
-
/** Timestamp when cache was last updated */
|
|
40
|
-
cachedAt = 0;
|
|
41
|
-
constructor(walletAddress) {
|
|
42
|
-
this.walletAddress = walletAddress;
|
|
43
|
-
this.client = createPublicClient({
|
|
44
|
-
chain: base,
|
|
45
|
-
transport: http(undefined, {
|
|
46
|
-
timeout: 10_000, // 10 second timeout to prevent hanging on slow RPC
|
|
47
|
-
}),
|
|
48
|
-
});
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Check current USDC balance.
|
|
52
|
-
* Uses cache if valid, otherwise fetches from RPC.
|
|
53
|
-
*/
|
|
54
|
-
async checkBalance() {
|
|
55
|
-
const now = Date.now();
|
|
56
|
-
// Use cache only when balance is positive and still fresh.
|
|
57
|
-
// Zero balance is never cached — always re-fetch so a funded wallet is
|
|
58
|
-
// detected on the next request without waiting for cache expiry.
|
|
59
|
-
if (this.cachedBalance !== null &&
|
|
60
|
-
this.cachedBalance > 0n &&
|
|
61
|
-
now - this.cachedAt < CACHE_TTL_MS) {
|
|
62
|
-
return this.buildInfo(this.cachedBalance);
|
|
63
|
-
}
|
|
64
|
-
// Fetch from RPC
|
|
65
|
-
const balance = await this.fetchBalance();
|
|
66
|
-
if (balance > 0n) {
|
|
67
|
-
this.cachedBalance = balance;
|
|
68
|
-
this.cachedAt = now;
|
|
69
|
-
}
|
|
70
|
-
return this.buildInfo(balance);
|
|
71
|
-
}
|
|
72
|
-
/**
|
|
73
|
-
* Check if balance is sufficient for an estimated cost.
|
|
74
|
-
*
|
|
75
|
-
* @param estimatedCostMicros - Estimated cost in USDC smallest unit (6 decimals)
|
|
76
|
-
*/
|
|
77
|
-
async checkSufficient(estimatedCostMicros) {
|
|
78
|
-
const info = await this.checkBalance();
|
|
79
|
-
if (info.balance >= estimatedCostMicros) {
|
|
80
|
-
return { sufficient: true, info };
|
|
81
|
-
}
|
|
82
|
-
const shortfall = estimatedCostMicros - info.balance;
|
|
83
|
-
return {
|
|
84
|
-
sufficient: false,
|
|
85
|
-
info,
|
|
86
|
-
shortfall: this.formatUSDC(shortfall),
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
/**
|
|
90
|
-
* Optimistically deduct estimated cost from cached balance.
|
|
91
|
-
* Call this after a successful payment to keep cache accurate.
|
|
92
|
-
*
|
|
93
|
-
* @param amountMicros - Amount to deduct in USDC smallest unit
|
|
94
|
-
*/
|
|
95
|
-
deductEstimated(amountMicros) {
|
|
96
|
-
if (this.cachedBalance !== null && this.cachedBalance >= amountMicros) {
|
|
97
|
-
this.cachedBalance -= amountMicros;
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* Invalidate cache, forcing next checkBalance() to fetch from RPC.
|
|
102
|
-
* Call this after a payment failure to get accurate balance.
|
|
103
|
-
*/
|
|
104
|
-
invalidate() {
|
|
105
|
-
this.cachedBalance = null;
|
|
106
|
-
this.cachedAt = 0;
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* Force refresh balance from RPC (ignores cache).
|
|
110
|
-
*/
|
|
111
|
-
async refresh() {
|
|
112
|
-
this.invalidate();
|
|
113
|
-
return this.checkBalance();
|
|
114
|
-
}
|
|
115
|
-
/**
|
|
116
|
-
* Format USDC amount (in micros) as "$X.XX".
|
|
117
|
-
*/
|
|
118
|
-
formatUSDC(amountMicros) {
|
|
119
|
-
// USDC has 6 decimals
|
|
120
|
-
const dollars = Number(amountMicros) / 1_000_000;
|
|
121
|
-
return `$${dollars.toFixed(2)}`;
|
|
122
|
-
}
|
|
123
|
-
/**
|
|
124
|
-
* Get the wallet address being monitored.
|
|
125
|
-
*/
|
|
126
|
-
getWalletAddress() {
|
|
127
|
-
return this.walletAddress;
|
|
128
|
-
}
|
|
129
|
-
/** Fetch balance from RPC */
|
|
130
|
-
async fetchBalance() {
|
|
131
|
-
try {
|
|
132
|
-
const balance = await this.client.readContract({
|
|
133
|
-
address: USDC_BASE,
|
|
134
|
-
abi: erc20Abi,
|
|
135
|
-
functionName: "balanceOf",
|
|
136
|
-
args: [this.walletAddress],
|
|
137
|
-
});
|
|
138
|
-
return balance;
|
|
139
|
-
}
|
|
140
|
-
catch (error) {
|
|
141
|
-
// Throw typed error instead of silently returning 0
|
|
142
|
-
// This allows callers to distinguish "node down" from "wallet empty"
|
|
143
|
-
throw new RpcError(error instanceof Error ? error.message : "Unknown error", error);
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
/** Build BalanceInfo from raw balance */
|
|
147
|
-
buildInfo(balance) {
|
|
148
|
-
return {
|
|
149
|
-
balance,
|
|
150
|
-
balanceUSD: this.formatUSDC(balance),
|
|
151
|
-
isLow: balance < BALANCE_THRESHOLDS.LOW_BALANCE_MICROS,
|
|
152
|
-
isEmpty: balance < BALANCE_THRESHOLDS.ZERO_THRESHOLD,
|
|
153
|
-
walletAddress: this.walletAddress,
|
|
154
|
-
};
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
//# sourceMappingURL=balance.js.map
|
package/dist/balance.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"balance.js","sourceRoot":"","sources":["../src/balance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,4CAA4C;AAC5C,MAAM,SAAS,GAAG,4CAAqD,CAAC;AAExE,6CAA6C;AAC7C,MAAM,YAAY,GAAG,MAAM,CAAC;AAE5B,4DAA4D;AAC5D,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,2CAA2C;IAC3C,kBAAkB,EAAE,QAAU;IAC9B,iEAAiE;IACjE,cAAc,EAAE,IAAI;CACZ,CAAC;AA0BX;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IACR,MAAM,CAAC;IACP,aAAa,CAAgB;IAE9C,8CAA8C;IACtC,aAAa,GAAkB,IAAI,CAAC;IAC5C,4CAA4C;IACpC,QAAQ,GAAG,CAAC,CAAC;IAErB,YAAY,aAAqB;QAC/B,IAAI,CAAC,aAAa,GAAG,aAA8B,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,kBAAkB,CAAC;YAC/B,KAAK,EAAE,IAAI;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE;gBACzB,OAAO,EAAE,MAAM,EAAE,mDAAmD;aACrE,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,2DAA2D;QAC3D,uEAAuE;QACvE,iEAAiE;QACjE,IACE,IAAI,CAAC,aAAa,KAAK,IAAI;YAC3B,IAAI,CAAC,aAAa,GAAG,EAAE;YACvB,GAAG,GAAG,IAAI,CAAC,QAAQ,GAAG,YAAY,EAClC,CAAC;YACD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC5C,CAAC;QAED,iBAAiB;QACjB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;YACjB,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;YAC7B,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;QACtB,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,eAAe,CAAC,mBAA2B;QAC/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAEvC,IAAI,IAAI,CAAC,OAAO,IAAI,mBAAmB,EAAE,CAAC;YACxC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;QAED,MAAM,SAAS,GAAG,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC;QACrD,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,IAAI;YACJ,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;SACtC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,YAAoB;QAClC,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,aAAa,IAAI,YAAY,EAAE,CAAC;YACtE,IAAI,CAAC,aAAa,IAAI,YAAY,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,UAAU;QACR,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,YAAoB;QAC7B,sBAAsB;QACtB,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC;QACjD,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,6BAA6B;IACrB,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;gBAC7C,OAAO,EAAE,SAAS;gBAClB,GAAG,EAAE,QAAQ;gBACb,YAAY,EAAE,WAAW;gBACzB,IAAI,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC;aAC3B,CAAC,CAAC;YACH,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,oDAAoD;YACpD,qEAAqE;YACrE,MAAM,IAAI,QAAQ,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAED,yCAAyC;IACjC,SAAS,CAAC,OAAe;QAC/B,OAAO;YACL,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YACpC,KAAK,EAAE,OAAO,GAAG,kBAAkB,CAAC,kBAAkB;YACtD,OAAO,EAAE,OAAO,GAAG,kBAAkB,CAAC,cAAc;YACpD,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC;IACJ,CAAC;CACF"}
|
package/dist/cli.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;GAYG"}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Dictionary Codebook
|
|
3
|
-
*
|
|
4
|
-
* Static dictionary of frequently repeated phrases observed in LLM prompts.
|
|
5
|
-
* Built from analysis of BlockRun production logs.
|
|
6
|
-
*
|
|
7
|
-
* Format: Short code ($XX) -> Long phrase
|
|
8
|
-
* The LLM receives a codebook header and decodes in-context.
|
|
9
|
-
*/
|
|
10
|
-
export declare const STATIC_CODEBOOK: Record<string, string>;
|
|
11
|
-
/**
|
|
12
|
-
* Get the inverse codebook for decompression.
|
|
13
|
-
*/
|
|
14
|
-
export declare function getInverseCodebook(): Record<string, string>;
|
|
15
|
-
/**
|
|
16
|
-
* Generate the codebook header for inclusion in system message.
|
|
17
|
-
* LLMs can decode in-context using this header.
|
|
18
|
-
*/
|
|
19
|
-
export declare function generateCodebookHeader(usedCodes: Set<string>, pathMap?: Record<string, string>): string;
|
|
20
|
-
/**
|
|
21
|
-
* Decompress a string using the codebook (for logging).
|
|
22
|
-
*/
|
|
23
|
-
export declare function decompressContent(content: string, codebook?: Record<string, string>): string;
|
|
24
|
-
//# sourceMappingURL=codebook.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"codebook.d.ts","sourceRoot":"","sources":["../../src/compression/codebook.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAuElD,CAAC;AAEF;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAM3D;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,GACnC,MAAM,CAwBR;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,MAAM,EACf,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAmB,GACjD,MAAM,CAMR"}
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Dictionary Codebook
|
|
3
|
-
*
|
|
4
|
-
* Static dictionary of frequently repeated phrases observed in LLM prompts.
|
|
5
|
-
* Built from analysis of BlockRun production logs.
|
|
6
|
-
*
|
|
7
|
-
* Format: Short code ($XX) -> Long phrase
|
|
8
|
-
* The LLM receives a codebook header and decodes in-context.
|
|
9
|
-
*/
|
|
10
|
-
// Static codebook - common patterns from system prompts
|
|
11
|
-
// Ordered by expected frequency and impact
|
|
12
|
-
export const STATIC_CODEBOOK = {
|
|
13
|
-
// High-impact: OpenClaw/Agent system prompt patterns (very common)
|
|
14
|
-
$OC01: "unbrowse_", // Common prefix in tool names
|
|
15
|
-
$OC02: "<location>",
|
|
16
|
-
$OC03: "</location>",
|
|
17
|
-
$OC04: "<name>",
|
|
18
|
-
$OC05: "</name>",
|
|
19
|
-
$OC06: "<description>",
|
|
20
|
-
$OC07: "</description>",
|
|
21
|
-
$OC08: "(may need login)",
|
|
22
|
-
$OC09: "API skill for OpenClaw",
|
|
23
|
-
$OC10: "endpoints",
|
|
24
|
-
// Skill/tool markers
|
|
25
|
-
$SK01: "<available_skills>",
|
|
26
|
-
$SK02: "</available_skills>",
|
|
27
|
-
$SK03: "<skill>",
|
|
28
|
-
$SK04: "</skill>",
|
|
29
|
-
// Schema patterns (very common in tool definitions)
|
|
30
|
-
$T01: 'type: "function"',
|
|
31
|
-
$T02: '"type": "function"',
|
|
32
|
-
$T03: '"type": "string"',
|
|
33
|
-
$T04: '"type": "object"',
|
|
34
|
-
$T05: '"type": "array"',
|
|
35
|
-
$T06: '"type": "boolean"',
|
|
36
|
-
$T07: '"type": "number"',
|
|
37
|
-
// Common descriptions
|
|
38
|
-
$D01: "description:",
|
|
39
|
-
$D02: '"description":',
|
|
40
|
-
// Common instructions
|
|
41
|
-
$I01: "You are a personal assistant",
|
|
42
|
-
$I02: "Tool names are case-sensitive",
|
|
43
|
-
$I03: "Call tools exactly as listed",
|
|
44
|
-
$I04: "Use when",
|
|
45
|
-
$I05: "without asking",
|
|
46
|
-
// Safety phrases
|
|
47
|
-
$S01: "Do not manipulate or persuade",
|
|
48
|
-
$S02: "Prioritize safety and human oversight",
|
|
49
|
-
$S03: "unless explicitly requested",
|
|
50
|
-
// JSON patterns
|
|
51
|
-
$J01: '"required": ["',
|
|
52
|
-
$J02: '"properties": {',
|
|
53
|
-
$J03: '"additionalProperties": false',
|
|
54
|
-
// Heartbeat patterns
|
|
55
|
-
$H01: "HEARTBEAT_OK",
|
|
56
|
-
$H02: "Read HEARTBEAT.md if it exists",
|
|
57
|
-
// Role markers
|
|
58
|
-
$R01: '"role": "system"',
|
|
59
|
-
$R02: '"role": "user"',
|
|
60
|
-
$R03: '"role": "assistant"',
|
|
61
|
-
$R04: '"role": "tool"',
|
|
62
|
-
// Common endings/phrases
|
|
63
|
-
$E01: "would you like to",
|
|
64
|
-
$E02: "Let me know if you",
|
|
65
|
-
$E03: "internal APIs",
|
|
66
|
-
$E04: "session cookies",
|
|
67
|
-
// BlockRun model aliases (common in prompts)
|
|
68
|
-
$M01: "blockrun/",
|
|
69
|
-
$M02: "openai/",
|
|
70
|
-
$M03: "anthropic/",
|
|
71
|
-
$M04: "google/",
|
|
72
|
-
$M05: "xai/",
|
|
73
|
-
};
|
|
74
|
-
/**
|
|
75
|
-
* Get the inverse codebook for decompression.
|
|
76
|
-
*/
|
|
77
|
-
export function getInverseCodebook() {
|
|
78
|
-
const inverse = {};
|
|
79
|
-
for (const [code, phrase] of Object.entries(STATIC_CODEBOOK)) {
|
|
80
|
-
inverse[phrase] = code;
|
|
81
|
-
}
|
|
82
|
-
return inverse;
|
|
83
|
-
}
|
|
84
|
-
/**
|
|
85
|
-
* Generate the codebook header for inclusion in system message.
|
|
86
|
-
* LLMs can decode in-context using this header.
|
|
87
|
-
*/
|
|
88
|
-
export function generateCodebookHeader(usedCodes, pathMap = {}) {
|
|
89
|
-
if (usedCodes.size === 0 && Object.keys(pathMap).length === 0) {
|
|
90
|
-
return "";
|
|
91
|
-
}
|
|
92
|
-
const parts = [];
|
|
93
|
-
// Add used dictionary codes
|
|
94
|
-
if (usedCodes.size > 0) {
|
|
95
|
-
const codeEntries = Array.from(usedCodes)
|
|
96
|
-
.map((code) => `${code}=${STATIC_CODEBOOK[code]}`)
|
|
97
|
-
.join(", ");
|
|
98
|
-
parts.push(`[Dict: ${codeEntries}]`);
|
|
99
|
-
}
|
|
100
|
-
// Add path map
|
|
101
|
-
if (Object.keys(pathMap).length > 0) {
|
|
102
|
-
const pathEntries = Object.entries(pathMap)
|
|
103
|
-
.map(([code, path]) => `${code}=${path}`)
|
|
104
|
-
.join(", ");
|
|
105
|
-
parts.push(`[Paths: ${pathEntries}]`);
|
|
106
|
-
}
|
|
107
|
-
return parts.join("\n");
|
|
108
|
-
}
|
|
109
|
-
/**
|
|
110
|
-
* Decompress a string using the codebook (for logging).
|
|
111
|
-
*/
|
|
112
|
-
export function decompressContent(content, codebook = STATIC_CODEBOOK) {
|
|
113
|
-
let result = content;
|
|
114
|
-
for (const [code, phrase] of Object.entries(codebook)) {
|
|
115
|
-
result = result.split(code).join(phrase);
|
|
116
|
-
}
|
|
117
|
-
return result;
|
|
118
|
-
}
|
|
119
|
-
//# sourceMappingURL=codebook.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"codebook.js","sourceRoot":"","sources":["../../src/compression/codebook.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,wDAAwD;AACxD,2CAA2C;AAC3C,MAAM,CAAC,MAAM,eAAe,GAA2B;IACrD,mEAAmE;IACnE,KAAK,EAAE,WAAW,EAAE,8BAA8B;IAClD,KAAK,EAAE,YAAY;IACnB,KAAK,EAAE,aAAa;IACpB,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,eAAe;IACtB,KAAK,EAAE,gBAAgB;IACvB,KAAK,EAAE,kBAAkB;IACzB,KAAK,EAAE,wBAAwB;IAC/B,KAAK,EAAE,WAAW;IAElB,qBAAqB;IACrB,KAAK,EAAE,oBAAoB;IAC3B,KAAK,EAAE,qBAAqB;IAC5B,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,UAAU;IAEjB,oDAAoD;IACpD,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,oBAAoB;IAC1B,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,iBAAiB;IACvB,IAAI,EAAE,mBAAmB;IACzB,IAAI,EAAE,kBAAkB;IAExB,sBAAsB;IACtB,IAAI,EAAE,cAAc;IACpB,IAAI,EAAE,gBAAgB;IAEtB,sBAAsB;IACtB,IAAI,EAAE,8BAA8B;IACpC,IAAI,EAAE,+BAA+B;IACrC,IAAI,EAAE,8BAA8B;IACpC,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,gBAAgB;IAEtB,iBAAiB;IACjB,IAAI,EAAE,+BAA+B;IACrC,IAAI,EAAE,uCAAuC;IAC7C,IAAI,EAAE,6BAA6B;IAEnC,gBAAgB;IAChB,IAAI,EAAE,gBAAgB;IACtB,IAAI,EAAE,iBAAiB;IACvB,IAAI,EAAE,+BAA+B;IAErC,qBAAqB;IACrB,IAAI,EAAE,cAAc;IACpB,IAAI,EAAE,gCAAgC;IAEtC,eAAe;IACf,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,gBAAgB;IACtB,IAAI,EAAE,qBAAqB;IAC3B,IAAI,EAAE,gBAAgB;IAEtB,yBAAyB;IACzB,IAAI,EAAE,mBAAmB;IACzB,IAAI,EAAE,oBAAoB;IAC1B,IAAI,EAAE,eAAe;IACrB,IAAI,EAAE,iBAAiB;IAEvB,6CAA6C;IAC7C,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QAC7D,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IACzB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,SAAsB,EACtB,UAAkC,EAAE;IAEpC,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,4BAA4B;IAC5B,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;aACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;aACjD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,UAAU,WAAW,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,eAAe;IACf,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;aACxC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;aACxC,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,WAAW,WAAW,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAAe,EACf,WAAmC,eAAe;IAElD,IAAI,MAAM,GAAG,OAAO,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACtD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* LLM-Safe Context Compression
|
|
3
|
-
*
|
|
4
|
-
* Reduces token usage by 15-40% while preserving semantic meaning.
|
|
5
|
-
* Implements 7 compression layers inspired by claw-compactor.
|
|
6
|
-
*
|
|
7
|
-
* Usage:
|
|
8
|
-
* const result = await compressContext(messages);
|
|
9
|
-
* // result.messages -> compressed version to send to provider
|
|
10
|
-
* // result.originalMessages -> original for logging
|
|
11
|
-
*/
|
|
12
|
-
import { NormalizedMessage, CompressionConfig, CompressionResult } from "./types.js";
|
|
13
|
-
export * from "./types.js";
|
|
14
|
-
export { STATIC_CODEBOOK } from "./codebook.js";
|
|
15
|
-
/**
|
|
16
|
-
* Main compression function.
|
|
17
|
-
*
|
|
18
|
-
* Applies 5 layers in sequence:
|
|
19
|
-
* 1. Deduplication - Remove exact duplicate messages
|
|
20
|
-
* 2. Whitespace - Normalize excessive whitespace
|
|
21
|
-
* 3. Dictionary - Replace common phrases with codes
|
|
22
|
-
* 4. Paths - Shorten repeated file paths
|
|
23
|
-
* 5. JSON - Compact JSON in tool calls
|
|
24
|
-
*
|
|
25
|
-
* Then prepends a codebook header for the LLM to decode in-context.
|
|
26
|
-
*/
|
|
27
|
-
export declare function compressContext(messages: NormalizedMessage[], config?: Partial<CompressionConfig>): Promise<CompressionResult>;
|
|
28
|
-
/**
|
|
29
|
-
* Quick check if compression would benefit these messages.
|
|
30
|
-
* Returns true if messages are large enough to warrant compression.
|
|
31
|
-
*/
|
|
32
|
-
export declare function shouldCompress(messages: NormalizedMessage[]): boolean;
|
|
33
|
-
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/compression/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EAGlB,MAAM,YAAY,CAAC;AAUpB,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAqEhD;;;;;;;;;;;GAWG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,iBAAiB,EAAE,EAC7B,MAAM,GAAE,OAAO,CAAC,iBAAiB,CAAM,GACtC,OAAO,CAAC,iBAAiB,CAAC,CA+J5B;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAIrE"}
|
|
@@ -1,246 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* LLM-Safe Context Compression
|
|
3
|
-
*
|
|
4
|
-
* Reduces token usage by 15-40% while preserving semantic meaning.
|
|
5
|
-
* Implements 7 compression layers inspired by claw-compactor.
|
|
6
|
-
*
|
|
7
|
-
* Usage:
|
|
8
|
-
* const result = await compressContext(messages);
|
|
9
|
-
* // result.messages -> compressed version to send to provider
|
|
10
|
-
* // result.originalMessages -> original for logging
|
|
11
|
-
*/
|
|
12
|
-
import { DEFAULT_COMPRESSION_CONFIG, } from "./types.js";
|
|
13
|
-
import { deduplicateMessages } from "./layers/deduplication.js";
|
|
14
|
-
import { normalizeMessagesWhitespace } from "./layers/whitespace.js";
|
|
15
|
-
import { encodeMessages } from "./layers/dictionary.js";
|
|
16
|
-
import { shortenPaths } from "./layers/paths.js";
|
|
17
|
-
import { compactMessagesJson } from "./layers/json-compact.js";
|
|
18
|
-
import { compressObservations } from "./layers/observation.js";
|
|
19
|
-
import { applyDynamicCodebook, generateDynamicCodebookHeader } from "./layers/dynamic-codebook.js";
|
|
20
|
-
import { generateCodebookHeader, STATIC_CODEBOOK } from "./codebook.js";
|
|
21
|
-
export * from "./types.js";
|
|
22
|
-
export { STATIC_CODEBOOK } from "./codebook.js";
|
|
23
|
-
/**
|
|
24
|
-
* Calculate total character count for messages.
|
|
25
|
-
*/
|
|
26
|
-
function calculateTotalChars(messages) {
|
|
27
|
-
return messages.reduce((total, msg) => {
|
|
28
|
-
let chars = 0;
|
|
29
|
-
if (typeof msg.content === "string") {
|
|
30
|
-
chars = msg.content.length;
|
|
31
|
-
}
|
|
32
|
-
else if (Array.isArray(msg.content)) {
|
|
33
|
-
// For multimodal content, stringify to get approximate size
|
|
34
|
-
chars = JSON.stringify(msg.content).length;
|
|
35
|
-
}
|
|
36
|
-
if (msg.tool_calls) {
|
|
37
|
-
chars += JSON.stringify(msg.tool_calls).length;
|
|
38
|
-
}
|
|
39
|
-
return total + chars;
|
|
40
|
-
}, 0);
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* Deep clone messages to preserve originals.
|
|
44
|
-
*/
|
|
45
|
-
function cloneMessages(messages) {
|
|
46
|
-
return JSON.parse(JSON.stringify(messages));
|
|
47
|
-
}
|
|
48
|
-
/**
|
|
49
|
-
* Prepend codebook header to the first USER message (not system).
|
|
50
|
-
*
|
|
51
|
-
* Why not system message?
|
|
52
|
-
* - Google Gemini uses systemInstruction which doesn't support codebook format
|
|
53
|
-
* - The codebook header in user message is still visible to all LLMs
|
|
54
|
-
* - This ensures compatibility across all providers
|
|
55
|
-
*/
|
|
56
|
-
function prependCodebookHeader(messages, usedCodes, pathMap) {
|
|
57
|
-
const header = generateCodebookHeader(usedCodes, pathMap);
|
|
58
|
-
if (!header)
|
|
59
|
-
return messages;
|
|
60
|
-
// Find first user message (not system - Google's systemInstruction doesn't support codebook)
|
|
61
|
-
const userIndex = messages.findIndex((m) => m.role === "user");
|
|
62
|
-
if (userIndex === -1) {
|
|
63
|
-
// No user message, add codebook as system (fallback)
|
|
64
|
-
return [{ role: "system", content: header }, ...messages];
|
|
65
|
-
}
|
|
66
|
-
// Prepend to first user message (only if content is a string)
|
|
67
|
-
return messages.map((msg, i) => {
|
|
68
|
-
if (i === userIndex) {
|
|
69
|
-
// Only prepend to string content - skip arrays (multimodal messages)
|
|
70
|
-
if (typeof msg.content === "string") {
|
|
71
|
-
return {
|
|
72
|
-
...msg,
|
|
73
|
-
content: `${header}\n\n${msg.content}`,
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
// For non-string content, don't modify the message
|
|
77
|
-
// The codebook header would corrupt array content
|
|
78
|
-
}
|
|
79
|
-
return msg;
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* Main compression function.
|
|
84
|
-
*
|
|
85
|
-
* Applies 5 layers in sequence:
|
|
86
|
-
* 1. Deduplication - Remove exact duplicate messages
|
|
87
|
-
* 2. Whitespace - Normalize excessive whitespace
|
|
88
|
-
* 3. Dictionary - Replace common phrases with codes
|
|
89
|
-
* 4. Paths - Shorten repeated file paths
|
|
90
|
-
* 5. JSON - Compact JSON in tool calls
|
|
91
|
-
*
|
|
92
|
-
* Then prepends a codebook header for the LLM to decode in-context.
|
|
93
|
-
*/
|
|
94
|
-
export async function compressContext(messages, config = {}) {
|
|
95
|
-
const fullConfig = {
|
|
96
|
-
...DEFAULT_COMPRESSION_CONFIG,
|
|
97
|
-
...config,
|
|
98
|
-
layers: {
|
|
99
|
-
...DEFAULT_COMPRESSION_CONFIG.layers,
|
|
100
|
-
...config.layers,
|
|
101
|
-
},
|
|
102
|
-
dictionary: {
|
|
103
|
-
...DEFAULT_COMPRESSION_CONFIG.dictionary,
|
|
104
|
-
...config.dictionary,
|
|
105
|
-
},
|
|
106
|
-
};
|
|
107
|
-
// If compression disabled, return as-is
|
|
108
|
-
if (!fullConfig.enabled) {
|
|
109
|
-
const originalChars = calculateTotalChars(messages);
|
|
110
|
-
return {
|
|
111
|
-
messages,
|
|
112
|
-
originalMessages: messages,
|
|
113
|
-
originalChars,
|
|
114
|
-
compressedChars: originalChars,
|
|
115
|
-
compressionRatio: 1,
|
|
116
|
-
stats: {
|
|
117
|
-
duplicatesRemoved: 0,
|
|
118
|
-
whitespaceSavedChars: 0,
|
|
119
|
-
dictionarySubstitutions: 0,
|
|
120
|
-
pathsShortened: 0,
|
|
121
|
-
jsonCompactedChars: 0,
|
|
122
|
-
observationsCompressed: 0,
|
|
123
|
-
observationCharsSaved: 0,
|
|
124
|
-
dynamicSubstitutions: 0,
|
|
125
|
-
dynamicCharsSaved: 0,
|
|
126
|
-
},
|
|
127
|
-
codebook: {},
|
|
128
|
-
pathMap: {},
|
|
129
|
-
dynamicCodes: {},
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
|
-
// Preserve originals for logging
|
|
133
|
-
const originalMessages = fullConfig.preserveRaw ? cloneMessages(messages) : messages;
|
|
134
|
-
const originalChars = calculateTotalChars(messages);
|
|
135
|
-
// Initialize stats
|
|
136
|
-
const stats = {
|
|
137
|
-
duplicatesRemoved: 0,
|
|
138
|
-
whitespaceSavedChars: 0,
|
|
139
|
-
dictionarySubstitutions: 0,
|
|
140
|
-
pathsShortened: 0,
|
|
141
|
-
jsonCompactedChars: 0,
|
|
142
|
-
observationsCompressed: 0,
|
|
143
|
-
observationCharsSaved: 0,
|
|
144
|
-
dynamicSubstitutions: 0,
|
|
145
|
-
dynamicCharsSaved: 0,
|
|
146
|
-
};
|
|
147
|
-
let result = cloneMessages(messages);
|
|
148
|
-
let usedCodes = new Set();
|
|
149
|
-
let pathMap = {};
|
|
150
|
-
let dynamicCodes = {};
|
|
151
|
-
// Layer 1: Deduplication
|
|
152
|
-
if (fullConfig.layers.deduplication) {
|
|
153
|
-
const dedupResult = deduplicateMessages(result);
|
|
154
|
-
result = dedupResult.messages;
|
|
155
|
-
stats.duplicatesRemoved = dedupResult.duplicatesRemoved;
|
|
156
|
-
}
|
|
157
|
-
// Layer 2: Whitespace normalization
|
|
158
|
-
if (fullConfig.layers.whitespace) {
|
|
159
|
-
const wsResult = normalizeMessagesWhitespace(result);
|
|
160
|
-
result = wsResult.messages;
|
|
161
|
-
stats.whitespaceSavedChars = wsResult.charsSaved;
|
|
162
|
-
}
|
|
163
|
-
// Layer 3: Dictionary encoding
|
|
164
|
-
if (fullConfig.layers.dictionary) {
|
|
165
|
-
const dictResult = encodeMessages(result);
|
|
166
|
-
result = dictResult.messages;
|
|
167
|
-
stats.dictionarySubstitutions = dictResult.substitutionCount;
|
|
168
|
-
usedCodes = dictResult.usedCodes;
|
|
169
|
-
}
|
|
170
|
-
// Layer 4: Path shortening
|
|
171
|
-
if (fullConfig.layers.paths) {
|
|
172
|
-
const pathResult = shortenPaths(result);
|
|
173
|
-
result = pathResult.messages;
|
|
174
|
-
pathMap = pathResult.pathMap;
|
|
175
|
-
stats.pathsShortened = Object.keys(pathMap).length;
|
|
176
|
-
}
|
|
177
|
-
// Layer 5: JSON compaction
|
|
178
|
-
if (fullConfig.layers.jsonCompact) {
|
|
179
|
-
const jsonResult = compactMessagesJson(result);
|
|
180
|
-
result = jsonResult.messages;
|
|
181
|
-
stats.jsonCompactedChars = jsonResult.charsSaved;
|
|
182
|
-
}
|
|
183
|
-
// Layer 6: Observation compression (BIG WIN - 97% on tool results)
|
|
184
|
-
if (fullConfig.layers.observation) {
|
|
185
|
-
const obsResult = compressObservations(result);
|
|
186
|
-
result = obsResult.messages;
|
|
187
|
-
stats.observationsCompressed = obsResult.observationsCompressed;
|
|
188
|
-
stats.observationCharsSaved = obsResult.charsSaved;
|
|
189
|
-
}
|
|
190
|
-
// Layer 7: Dynamic codebook (learns from actual content)
|
|
191
|
-
if (fullConfig.layers.dynamicCodebook) {
|
|
192
|
-
const dynResult = applyDynamicCodebook(result);
|
|
193
|
-
result = dynResult.messages;
|
|
194
|
-
stats.dynamicSubstitutions = dynResult.substitutions;
|
|
195
|
-
stats.dynamicCharsSaved = dynResult.charsSaved;
|
|
196
|
-
dynamicCodes = dynResult.dynamicCodes;
|
|
197
|
-
}
|
|
198
|
-
// Add codebook header if enabled and we have codes to include
|
|
199
|
-
if (fullConfig.dictionary.includeCodebookHeader &&
|
|
200
|
-
(usedCodes.size > 0 || Object.keys(pathMap).length > 0 || Object.keys(dynamicCodes).length > 0)) {
|
|
201
|
-
result = prependCodebookHeader(result, usedCodes, pathMap);
|
|
202
|
-
// Also add dynamic codebook header if we have dynamic codes
|
|
203
|
-
if (Object.keys(dynamicCodes).length > 0) {
|
|
204
|
-
const dynHeader = generateDynamicCodebookHeader(dynamicCodes);
|
|
205
|
-
if (dynHeader) {
|
|
206
|
-
const systemIndex = result.findIndex((m) => m.role === "system");
|
|
207
|
-
// Only prepend to string content - skip arrays (multimodal messages)
|
|
208
|
-
if (systemIndex >= 0 && typeof result[systemIndex].content === "string") {
|
|
209
|
-
result[systemIndex] = {
|
|
210
|
-
...result[systemIndex],
|
|
211
|
-
content: `${dynHeader}\n${result[systemIndex].content}`,
|
|
212
|
-
};
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
// Calculate final stats
|
|
218
|
-
const compressedChars = calculateTotalChars(result);
|
|
219
|
-
const compressionRatio = compressedChars / originalChars;
|
|
220
|
-
// Build used codebook for logging
|
|
221
|
-
const usedCodebook = {};
|
|
222
|
-
usedCodes.forEach((code) => {
|
|
223
|
-
usedCodebook[code] = STATIC_CODEBOOK[code];
|
|
224
|
-
});
|
|
225
|
-
return {
|
|
226
|
-
messages: result,
|
|
227
|
-
originalMessages,
|
|
228
|
-
originalChars,
|
|
229
|
-
compressedChars,
|
|
230
|
-
compressionRatio,
|
|
231
|
-
stats,
|
|
232
|
-
codebook: usedCodebook,
|
|
233
|
-
pathMap,
|
|
234
|
-
dynamicCodes,
|
|
235
|
-
};
|
|
236
|
-
}
|
|
237
|
-
/**
|
|
238
|
-
* Quick check if compression would benefit these messages.
|
|
239
|
-
* Returns true if messages are large enough to warrant compression.
|
|
240
|
-
*/
|
|
241
|
-
export function shouldCompress(messages) {
|
|
242
|
-
const chars = calculateTotalChars(messages);
|
|
243
|
-
// Only compress if > 5000 chars (roughly 1000 tokens)
|
|
244
|
-
return chars > 5000;
|
|
245
|
-
}
|
|
246
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/compression/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAKL,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AACnG,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAExE,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAA6B;IACxD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACpC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACpC,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;QAC7B,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,4DAA4D;YAC5D,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAC7C,CAAC;QACD,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACnB,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;QACjD,CAAC;QACD,OAAO,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC,EAAE,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,QAA6B;IAClD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,qBAAqB,CAC5B,QAA6B,EAC7B,SAAsB,EACtB,OAA+B;IAE/B,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM;QAAE,OAAO,QAAQ,CAAC;IAE7B,6FAA6F;IAC7F,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAE/D,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,qDAAqD;QACrD,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,QAAQ,CAAC,CAAC;IAC5D,CAAC;IAED,8DAA8D;IAC9D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QAC7B,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACpB,qEAAqE;YACrE,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACpC,OAAO;oBACL,GAAG,GAAG;oBACN,OAAO,EAAE,GAAG,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE;iBACvC,CAAC;YACJ,CAAC;YACD,mDAAmD;YACnD,kDAAkD;QACpD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAA6B,EAC7B,SAAqC,EAAE;IAEvC,MAAM,UAAU,GAAsB;QACpC,GAAG,0BAA0B;QAC7B,GAAG,MAAM;QACT,MAAM,EAAE;YACN,GAAG,0BAA0B,CAAC,MAAM;YACpC,GAAG,MAAM,CAAC,MAAM;SACjB;QACD,UAAU,EAAE;YACV,GAAG,0BAA0B,CAAC,UAAU;YACxC,GAAG,MAAM,CAAC,UAAU;SACrB;KACF,CAAC;IAEF,wCAAwC;IACxC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,aAAa,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACpD,OAAO;YACL,QAAQ;YACR,gBAAgB,EAAE,QAAQ;YAC1B,aAAa;YACb,eAAe,EAAE,aAAa;YAC9B,gBAAgB,EAAE,CAAC;YACnB,KAAK,EAAE;gBACL,iBAAiB,EAAE,CAAC;gBACpB,oBAAoB,EAAE,CAAC;gBACvB,uBAAuB,EAAE,CAAC;gBAC1B,cAAc,EAAE,CAAC;gBACjB,kBAAkB,EAAE,CAAC;gBACrB,sBAAsB,EAAE,CAAC;gBACzB,qBAAqB,EAAE,CAAC;gBACxB,oBAAoB,EAAE,CAAC;gBACvB,iBAAiB,EAAE,CAAC;aACrB;YACD,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,EAAE;SACjB,CAAC;IACJ,CAAC;IAED,iCAAiC;IACjC,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACrF,MAAM,aAAa,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAEpD,mBAAmB;IACnB,MAAM,KAAK,GAAqB;QAC9B,iBAAiB,EAAE,CAAC;QACpB,oBAAoB,EAAE,CAAC;QACvB,uBAAuB,EAAE,CAAC;QAC1B,cAAc,EAAE,CAAC;QACjB,kBAAkB,EAAE,CAAC;QACrB,sBAAsB,EAAE,CAAC;QACzB,qBAAqB,EAAE,CAAC;QACxB,oBAAoB,EAAE,CAAC;QACvB,iBAAiB,EAAE,CAAC;KACrB,CAAC;IAEF,IAAI,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,OAAO,GAA2B,EAAE,CAAC;IACzC,IAAI,YAAY,GAA2B,EAAE,CAAC;IAE9C,yBAAyB;IACzB,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC;QAC9B,KAAK,CAAC,iBAAiB,GAAG,WAAW,CAAC,iBAAiB,CAAC;IAC1D,CAAC;IAED,oCAAoC;IACpC,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAC3B,KAAK,CAAC,oBAAoB,GAAG,QAAQ,CAAC,UAAU,CAAC;IACnD,CAAC;IAED,+BAA+B;IAC/B,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;QAC7B,KAAK,CAAC,uBAAuB,GAAG,UAAU,CAAC,iBAAiB,CAAC;QAC7D,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;IACnC,CAAC;IAED,2BAA2B;IAC3B,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;QAC7B,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAC7B,KAAK,CAAC,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACrD,CAAC;IAED,2BAA2B;IAC3B,IAAI,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;QAC7B,KAAK,CAAC,kBAAkB,GAAG,UAAU,CAAC,UAAU,CAAC;IACnD,CAAC;IAED,mEAAmE;IACnE,IAAI,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC;QAC5B,KAAK,CAAC,sBAAsB,GAAG,SAAS,CAAC,sBAAsB,CAAC;QAChE,KAAK,CAAC,qBAAqB,GAAG,SAAS,CAAC,UAAU,CAAC;IACrD,CAAC;IAED,yDAAyD;IACzD,IAAI,UAAU,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC;QAC5B,KAAK,CAAC,oBAAoB,GAAG,SAAS,CAAC,aAAa,CAAC;QACrD,KAAK,CAAC,iBAAiB,GAAG,SAAS,CAAC,UAAU,CAAC;QAC/C,YAAY,GAAG,SAAS,CAAC,YAAY,CAAC;IACxC,CAAC;IAED,8DAA8D;IAC9D,IACE,UAAU,CAAC,UAAU,CAAC,qBAAqB;QAC3C,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC/F,CAAC;QACD,MAAM,GAAG,qBAAqB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC3D,4DAA4D;QAC5D,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,6BAA6B,CAAC,YAAY,CAAC,CAAC;YAC9D,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;gBACjE,qEAAqE;gBACrE,IAAI,WAAW,IAAI,CAAC,IAAI,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACxE,MAAM,CAAC,WAAW,CAAC,GAAG;wBACpB,GAAG,MAAM,CAAC,WAAW,CAAC;wBACtB,OAAO,EAAE,GAAG,SAAS,KAAK,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;qBACxD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,eAAe,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,gBAAgB,GAAG,eAAe,GAAG,aAAa,CAAC;IAEzD,kCAAkC;IAClC,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,YAAY,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,QAAQ,EAAE,MAAM;QAChB,gBAAgB;QAChB,aAAa;QACb,eAAe;QACf,gBAAgB;QAChB,KAAK;QACL,QAAQ,EAAE,YAAY;QACtB,OAAO;QACP,YAAY;KACb,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,QAA6B;IAC1D,MAAM,KAAK,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAC5C,sDAAsD;IACtD,OAAO,KAAK,GAAG,IAAI,CAAC;AACtB,CAAC"}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Layer 1: Message Deduplication
|
|
3
|
-
*
|
|
4
|
-
* Removes exact duplicate messages from conversation history.
|
|
5
|
-
* Common in heartbeat patterns and repeated tool calls.
|
|
6
|
-
*
|
|
7
|
-
* Safe for LLM: Identical messages add no new information.
|
|
8
|
-
* Expected savings: 2-5%
|
|
9
|
-
*/
|
|
10
|
-
import { NormalizedMessage } from "../types.js";
|
|
11
|
-
export interface DeduplicationResult {
|
|
12
|
-
messages: NormalizedMessage[];
|
|
13
|
-
duplicatesRemoved: number;
|
|
14
|
-
originalCount: number;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Remove exact duplicate messages from the conversation.
|
|
18
|
-
*
|
|
19
|
-
* Strategy:
|
|
20
|
-
* - Keep first occurrence of each unique message
|
|
21
|
-
* - Preserve order for semantic coherence
|
|
22
|
-
* - Never dedupe system messages (they set context)
|
|
23
|
-
* - Allow duplicate user messages (user might repeat intentionally)
|
|
24
|
-
* - CRITICAL: Never dedupe assistant messages with tool_calls that are
|
|
25
|
-
* referenced by subsequent tool messages (breaks Anthropic tool_use/tool_result pairing)
|
|
26
|
-
*/
|
|
27
|
-
export declare function deduplicateMessages(messages: NormalizedMessage[]): DeduplicationResult;
|
|
28
|
-
//# sourceMappingURL=deduplication.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"deduplication.d.ts","sourceRoot":"","sources":["../../../src/compression/layers/deduplication.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGhD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;CACvB;AAiCD;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,mBAAmB,CA+DtF"}
|