@drsamonline/promptcache 0.0.0-stage → 0.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +23 -4
- package/src/index.js +168 -0
- package/README.md +0 -3
package/package.json
CHANGED
|
@@ -1,6 +1,25 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drsamonline/promptcache",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.12.3",
|
|
4
|
+
"description": "Native prompt caching: stable-prefix detection, per-provider hints, warm/cold tracking.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"main": "./src/index.js",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": "./src/index.js",
|
|
10
|
+
"./src/*": "./src/*"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"src"
|
|
14
|
+
],
|
|
15
|
+
"repository": {
|
|
16
|
+
"type": "git",
|
|
17
|
+
"url": "git://github.com/drsamonline/sui.git",
|
|
18
|
+
"directory": "packages/promptcache"
|
|
19
|
+
},
|
|
20
|
+
"homepage": "https://github.com/drsamonline/sui#readme",
|
|
21
|
+
"publishConfig": {
|
|
22
|
+
"access": "public",
|
|
23
|
+
"registry": "https://registry.npmjs.org/"
|
|
24
|
+
}
|
|
25
|
+
}
|
package/src/index.js
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @drsamonline/promptcache — native prompt caching (B4.6).
|
|
3
|
+
*
|
|
4
|
+
* Providers cache a *prefix*. Anthropic wants `cache_control` breakpoints,
|
|
5
|
+
* OpenAI caches automatically above a token threshold and takes a
|
|
6
|
+
* `prompt_cache_key` for affinity, and everything else does nothing. So the work
|
|
7
|
+
* here is: find the stable prefix, notice when it goes volatile, and send only
|
|
8
|
+
* the hint the provider actually understands.
|
|
9
|
+
*
|
|
10
|
+
* There is no state on the hot path (I1): `PrefixMemory` is an in-memory LRU of
|
|
11
|
+
* prefix fingerprints, which is the cheapest thing that can answer "have I sent
|
|
12
|
+
* this prefix before?".
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { createHash } from 'node:crypto';
|
|
16
|
+
|
|
17
|
+
/** Things that make a prefix volatile — each one breaks a provider cache. */
|
|
18
|
+
export const DYNAMIC_PATTERNS = [
|
|
19
|
+
{ name: 'iso timestamp', re: /\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/ },
|
|
20
|
+
{ name: 'date', re: /\b\d{4}-\d{2}-\d{2}\b/ },
|
|
21
|
+
{ name: 'clock time', re: /\b\d{1,2}:\d{2}(:\d{2})?\s?(am|pm)?\b/i },
|
|
22
|
+
{ name: 'uuid', re: /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i },
|
|
23
|
+
{ name: 'request id', re: /\b(req|request|trace|span)[_-]?id\b/i },
|
|
24
|
+
{ name: 'random hex', re: /\b[0-9a-f]{16,}\b/i },
|
|
25
|
+
{ name: 'counter', re: /\b(nonce|salt|seed)\b/i },
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Split messages into a stable prefix and a volatile tail.
|
|
30
|
+
*
|
|
31
|
+
* The prefix is everything up to (not including) the last user message — that is
|
|
32
|
+
* what a provider can cache. Everything before it that carries a timestamp or an
|
|
33
|
+
* id is reported, because that one string is why the cache never hits.
|
|
34
|
+
*
|
|
35
|
+
* @param {Array<{role: string, content: string}>} messages
|
|
36
|
+
* @returns {{prefix: Array<{role: string, content: string}>, tail: Array<{role: string, content: string}>, prefixChars: number, fingerprint: string, volatile: Array<{index: number, name: string}>}}
|
|
37
|
+
*/
|
|
38
|
+
export function detectPrefix(messages) {
|
|
39
|
+
const lastUser = messages.map((m) => m.role).lastIndexOf('user');
|
|
40
|
+
const cut = lastUser === -1 ? messages.length : lastUser;
|
|
41
|
+
const prefix = messages.slice(0, cut);
|
|
42
|
+
const tail = messages.slice(cut);
|
|
43
|
+
|
|
44
|
+
const prefixText = prefix.map((m) => `${m.role}:${m.content}`).join('\n');
|
|
45
|
+
const volatile = [];
|
|
46
|
+
for (let i = 0; i < prefix.length; i++) {
|
|
47
|
+
for (const pattern of DYNAMIC_PATTERNS) {
|
|
48
|
+
if (pattern.re.test(prefix[i].content ?? '')) volatile.push({ index: i, name: pattern.name });
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
prefix,
|
|
54
|
+
tail,
|
|
55
|
+
prefixChars: prefixText.length,
|
|
56
|
+
fingerprint: createHash('sha256').update(prefixText).digest('hex').slice(0, 16),
|
|
57
|
+
volatile,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Remember which prefixes have been sent, so "warm" is a fact rather than a hope.
|
|
63
|
+
*/
|
|
64
|
+
export class PrefixMemory {
|
|
65
|
+
/** @param {{maxEntries?: number, now?: () => number}} [opts] */
|
|
66
|
+
constructor(opts = {}) {
|
|
67
|
+
this.maxEntries = opts.maxEntries ?? 500;
|
|
68
|
+
this.now = opts.now ?? Date.now;
|
|
69
|
+
/** @type {Map<string, {firstSeen: number, lastSeen: number, sends: number}>} */
|
|
70
|
+
this.map = new Map();
|
|
71
|
+
this.warm = 0;
|
|
72
|
+
this.cold = 0;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** @param {string} fingerprint */
|
|
76
|
+
touch(fingerprint) {
|
|
77
|
+
const existing = this.map.get(fingerprint);
|
|
78
|
+
if (existing) {
|
|
79
|
+
existing.lastSeen = this.now();
|
|
80
|
+
existing.sends += 1;
|
|
81
|
+
this.map.delete(fingerprint);
|
|
82
|
+
this.map.set(fingerprint, existing);
|
|
83
|
+
this.warm += 1;
|
|
84
|
+
return { warm: true, sends: existing.sends, firstSeen: existing.firstSeen };
|
|
85
|
+
}
|
|
86
|
+
this.map.set(fingerprint, { firstSeen: this.now(), lastSeen: this.now(), sends: 1 });
|
|
87
|
+
if (this.map.size > this.maxEntries) {
|
|
88
|
+
const oldest = this.map.keys().next();
|
|
89
|
+
if (!oldest.done) this.map.delete(oldest.value);
|
|
90
|
+
}
|
|
91
|
+
this.cold += 1;
|
|
92
|
+
return { warm: false, sends: 1, firstSeen: this.now() };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
stats() {
|
|
96
|
+
const total = this.warm + this.cold;
|
|
97
|
+
return {
|
|
98
|
+
prefixes: this.map.size,
|
|
99
|
+
warm: this.warm,
|
|
100
|
+
cold: this.cold,
|
|
101
|
+
hitRate: total ? Math.round((this.warm / total) * 1000) / 1000 : 0,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** The minimum prefix a provider will cache. Below this, a hint is noise. */
|
|
107
|
+
const MIN_CACHEABLE_CHARS = { anthropic: 1200, openai: 2400, gemini: 4000 };
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Build the hint a specific provider understands, or say that it has none.
|
|
111
|
+
*
|
|
112
|
+
* @param {string} providerType 'openai' | 'anthropic' | 'gemini' | 'mock'
|
|
113
|
+
* @param {{prefixChars: number, fingerprint: string, warm: boolean}} detected
|
|
114
|
+
* @returns {{supported: boolean, reason: string, anthropic?: {breakpoint: boolean}, openai?: {promptCacheKey: string}}}
|
|
115
|
+
*/
|
|
116
|
+
export function cacheHints(providerType, detected) {
|
|
117
|
+
const minimum = MIN_CACHEABLE_CHARS[providerType];
|
|
118
|
+
|
|
119
|
+
if (providerType === 'anthropic') {
|
|
120
|
+
if (detected.prefixChars < minimum) {
|
|
121
|
+
return { supported: false, reason: `prefix is ${detected.prefixChars} chars; Anthropic caches above ~${minimum}` };
|
|
122
|
+
}
|
|
123
|
+
return { supported: true, reason: 'cache_control breakpoint on the prefix', anthropic: { breakpoint: true } };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (providerType === 'openai') {
|
|
127
|
+
if (detected.prefixChars < minimum) {
|
|
128
|
+
return { supported: false, reason: `prefix is ${detected.prefixChars} chars; OpenAI caches above ~${minimum}` };
|
|
129
|
+
}
|
|
130
|
+
return { supported: true, reason: 'prompt_cache_key for routing affinity', openai: { promptCacheKey: detected.fingerprint } };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (providerType === 'gemini') {
|
|
134
|
+
return {
|
|
135
|
+
supported: false,
|
|
136
|
+
reason: `Gemini needs an explicit cached-content resource; a per-request hint does nothing (prefix ${detected.prefixChars} chars)`,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return { supported: false, reason: `provider type "${providerType}" has no prompt-cache hint` };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* One call that does the whole job for a request.
|
|
145
|
+
*
|
|
146
|
+
* @param {{messages: Array<{role: string, content: string}>, providerType: string, memory: PrefixMemory, enabled?: boolean}} input
|
|
147
|
+
*/
|
|
148
|
+
export function preparePromptCache(input) {
|
|
149
|
+
const enabled = input.enabled !== false;
|
|
150
|
+
if (!enabled) {
|
|
151
|
+
return { hints: { supported: false, reason: 'prompt caching is off' }, detected: null, warm: false };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const detected = detectPrefix(input.messages);
|
|
155
|
+
const seen = input.memory.touch(detected.fingerprint);
|
|
156
|
+
const hints = cacheHints(input.providerType, detected);
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
detected: {
|
|
160
|
+
prefixChars: detected.prefixChars,
|
|
161
|
+
fingerprint: detected.fingerprint,
|
|
162
|
+
volatile: detected.volatile,
|
|
163
|
+
},
|
|
164
|
+
warm: seen.warm,
|
|
165
|
+
sends: seen.sends,
|
|
166
|
+
hints,
|
|
167
|
+
};
|
|
168
|
+
}
|
package/README.md
DELETED