cachegate 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +127 -112
- package/README.md +31 -13
- package/cache.js +72 -51
- package/embeddings.js +42 -32
- package/metrics.js +609 -556
- package/package.json +15 -1
- package/redisClient.js +55 -45
- package/router.js +254 -218
- package/semanticCache.js +159 -154
- package/server.js +282 -113
- package/.dockerignore +0 -11
- package/.gitattributes +0 -12
- package/.github/ISSUE_TEMPLATE/bug_report.md +0 -33
- package/.github/ISSUE_TEMPLATE/config.yml +0 -5
- package/.github/ISSUE_TEMPLATE/feature_request.md +0 -29
- package/.github/PULL_REQUEST_TEMPLATE.md +0 -25
- package/.github/workflows/test.yml +0 -63
- package/CODE_OF_CONDUCT.md +0 -66
- package/CONTRIBUTING.md +0 -94
- package/Dockerfile +0 -24
- package/SECURITY.md +0 -39
- package/sync-oss-release.sh +0 -160
- package/test/auth-config.test.js +0 -27
- package/test/cache.test.js +0 -33
- package/test/embeddings.test.js +0 -24
- package/test/env-path.test.js +0 -41
- package/test/failover.test.js +0 -99
- package/test/metrics-postgres.test.js +0 -183
- package/test/metrics.test.js +0 -282
- package/test/router.test.js +0 -195
- package/test/semanticCache.test.js +0 -167
- package/test/server.test.js +0 -357
- package/test/streaming.test.js +0 -248
package/.env.example
CHANGED
|
@@ -1,112 +1,127 @@
|
|
|
1
|
-
# cachegate environment template
|
|
2
|
-
# Copy the values you need into your real .env file.
|
|
3
|
-
# NEVER commit .env to Git.
|
|
4
|
-
|
|
5
|
-
PORT=4000
|
|
6
|
-
|
|
7
|
-
# Required: protects the router from unauthorized use. The server
|
|
8
|
-
# refuses to start without this UNLESS ALLOW_INSECURE_LOCAL_DEV=true
|
|
9
|
-
# (below) is explicitly set - a missing key never silently means "no
|
|
10
|
-
# auth enforced."
|
|
11
|
-
# Generate with: openssl rand -hex 32
|
|
12
|
-
MODEL_ROUTER_INTERNAL_KEY=your-random-internal-key
|
|
13
|
-
|
|
14
|
-
# Optional: skip the MODEL_ROUTER_INTERNAL_KEY requirement above, for a
|
|
15
|
-
# throwaway LOCAL instance only. Never set this on anything reachable
|
|
16
|
-
# from outside your own machine.
|
|
17
|
-
# ALLOW_INSECURE_LOCAL_DEV=true
|
|
18
|
-
|
|
19
|
-
# Required (at least one provider key - Anthropic, OpenAI, or both).
|
|
20
|
-
# The model itself is named PER REQUEST in the API call's own "model"
|
|
21
|
-
# field (see README's "Usage" section), not configured here - there is
|
|
22
|
-
# no ANTHROPIC_MODEL/OPENAI_MODEL env var to set.
|
|
23
|
-
ANTHROPIC_API_KEY=sk-ant-api03-...
|
|
24
|
-
|
|
25
|
-
# Also required for the semantic cache's embeddings, regardless of
|
|
26
|
-
# which provider actually serves chat requests - it's the only
|
|
27
|
-
# embedding backend implemented (see semanticCache.js / README).
|
|
28
|
-
# OPENAI_API_KEY=sk-proj-...
|
|
29
|
-
|
|
30
|
-
# Optional: which OpenAI embedding model the semantic cache uses.
|
|
31
|
-
# Only matters if OPENAI_API_KEY is set. Default is a good balance of
|
|
32
|
-
# cost and quality; change only if you know you want a different one.
|
|
33
|
-
# EMBEDDING_MODEL=text-embedding-3-small
|
|
34
|
-
|
|
35
|
-
# Optional: Redis for response caching (strongly recommended for cost savings)
|
|
36
|
-
# Local development:
|
|
37
|
-
# REDIS_URL=redis://localhost:6379
|
|
38
|
-
# Render Key Value:
|
|
39
|
-
# REDIS_URL=rediss://default:PASSWORD@HOST:PORT
|
|
40
|
-
|
|
41
|
-
# Optional: persistent metrics storage (see metrics.js's own comment).
|
|
42
|
-
# Unset (the default) keeps metrics as local JSONL files, which don't
|
|
43
|
-
# survive a restart/redeploy on most hosts - fine for a standalone
|
|
44
|
-
# deployment with no database of its own. Set this to a real Postgres
|
|
45
|
-
# connection string to persist metrics there instead.
|
|
46
|
-
# DATABASE_URL=postgres://user:pass@host:5432/dbname
|
|
47
|
-
#
|
|
48
|
-
# MEMOCODE_ROUTER_DATABASE_URL does the exact same thing and takes
|
|
49
|
-
# priority over DATABASE_URL if both are set - use this instead when
|
|
50
|
-
# embedding cachegate inside an app that already has its own
|
|
51
|
-
# DATABASE_URL pointed at a different database (so the two never
|
|
52
|
-
# collide), or just to keep this router's own connection string
|
|
53
|
-
# explicitly distinct from whatever else reads DATABASE_URL in your stack.
|
|
54
|
-
# MEMOCODE_ROUTER_DATABASE_URL=postgres://user:pass@host:5432/dbname
|
|
55
|
-
|
|
56
|
-
# Optional: where JSONL metrics files live, if Postgres isn't
|
|
57
|
-
# configured. Despite the name, this now names a DIRECTORY (kept for
|
|
58
|
-
# backward compatibility with older configs that pointed it at a
|
|
59
|
-
# single file - see metrics.js's own comment). Defaults to ./data
|
|
60
|
-
# next to this file.
|
|
61
|
-
# METRICS_LOG_PATH=./data
|
|
62
|
-
|
|
63
|
-
# Optional: how long metrics history is kept before pruneOlderThan()'s
|
|
64
|
-
# scheduled daily job deletes it. Default 90 days deliberately matches
|
|
65
|
-
# /dashboard/data's own longest supported range - pruning any sooner
|
|
66
|
-
# would make its "Last 90 days" option quietly lie.
|
|
67
|
-
# METRICS_RETENTION_DAYS=90
|
|
68
|
-
|
|
69
|
-
# Optional: semantic cache tuning (all have sane defaults - see README's
|
|
70
|
-
# "Two kinds of cache hit" section before changing these).
|
|
71
|
-
# SEMANTIC_CACHE_ENABLED=false # hard off-switch, even with a key configured
|
|
72
|
-
# SEMANTIC_CACHE_THRESHOLD=0.93 # cosine similarity floor for a match
|
|
73
|
-
# SEMANTIC_CACHE_MAX_CANDIDATES=200 # per-model list cap (brute-force scan size)
|
|
74
|
-
# SEMANTIC_CACHE_TTL_SECONDS=3600
|
|
75
|
-
|
|
76
|
-
# Optional:
|
|
77
|
-
#
|
|
78
|
-
#
|
|
79
|
-
#
|
|
80
|
-
#
|
|
81
|
-
#
|
|
82
|
-
|
|
83
|
-
#
|
|
84
|
-
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
|
|
91
|
-
#
|
|
92
|
-
#
|
|
93
|
-
|
|
94
|
-
#
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
#
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
#
|
|
102
|
-
#
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
#
|
|
106
|
-
|
|
107
|
-
#
|
|
108
|
-
|
|
109
|
-
#
|
|
110
|
-
#
|
|
111
|
-
#
|
|
112
|
-
#
|
|
1
|
+
# cachegate environment template
|
|
2
|
+
# Copy the values you need into your real .env file.
|
|
3
|
+
# NEVER commit .env to Git.
|
|
4
|
+
|
|
5
|
+
PORT=4000
|
|
6
|
+
|
|
7
|
+
# Required: protects the router from unauthorized use. The server
|
|
8
|
+
# refuses to start without this UNLESS ALLOW_INSECURE_LOCAL_DEV=true
|
|
9
|
+
# (below) is explicitly set - a missing key never silently means "no
|
|
10
|
+
# auth enforced."
|
|
11
|
+
# Generate with: openssl rand -hex 32
|
|
12
|
+
MODEL_ROUTER_INTERNAL_KEY=your-random-internal-key
|
|
13
|
+
|
|
14
|
+
# Optional: skip the MODEL_ROUTER_INTERNAL_KEY requirement above, for a
|
|
15
|
+
# throwaway LOCAL instance only. Never set this on anything reachable
|
|
16
|
+
# from outside your own machine.
|
|
17
|
+
# ALLOW_INSECURE_LOCAL_DEV=true
|
|
18
|
+
|
|
19
|
+
# Required (at least one provider key - Anthropic, OpenAI, or both).
|
|
20
|
+
# The model itself is named PER REQUEST in the API call's own "model"
|
|
21
|
+
# field (see README's "Usage" section), not configured here - there is
|
|
22
|
+
# no ANTHROPIC_MODEL/OPENAI_MODEL env var to set.
|
|
23
|
+
ANTHROPIC_API_KEY=sk-ant-api03-...
|
|
24
|
+
|
|
25
|
+
# Also required for the semantic cache's embeddings, regardless of
|
|
26
|
+
# which provider actually serves chat requests - it's the only
|
|
27
|
+
# embedding backend implemented (see semanticCache.js / README).
|
|
28
|
+
# OPENAI_API_KEY=sk-proj-...
|
|
29
|
+
|
|
30
|
+
# Optional: which OpenAI embedding model the semantic cache uses.
|
|
31
|
+
# Only matters if OPENAI_API_KEY is set. Default is a good balance of
|
|
32
|
+
# cost and quality; change only if you know you want a different one.
|
|
33
|
+
# EMBEDDING_MODEL=text-embedding-3-small
|
|
34
|
+
|
|
35
|
+
# Optional: Redis for response caching (strongly recommended for cost savings)
|
|
36
|
+
# Local development:
|
|
37
|
+
# REDIS_URL=redis://localhost:6379
|
|
38
|
+
# Render Key Value:
|
|
39
|
+
# REDIS_URL=rediss://default:PASSWORD@HOST:PORT
|
|
40
|
+
|
|
41
|
+
# Optional: persistent metrics storage (see metrics.js's own comment).
|
|
42
|
+
# Unset (the default) keeps metrics as local JSONL files, which don't
|
|
43
|
+
# survive a restart/redeploy on most hosts - fine for a standalone
|
|
44
|
+
# deployment with no database of its own. Set this to a real Postgres
|
|
45
|
+
# connection string to persist metrics there instead.
|
|
46
|
+
# DATABASE_URL=postgres://user:pass@host:5432/dbname
|
|
47
|
+
#
|
|
48
|
+
# MEMOCODE_ROUTER_DATABASE_URL does the exact same thing and takes
|
|
49
|
+
# priority over DATABASE_URL if both are set - use this instead when
|
|
50
|
+
# embedding cachegate inside an app that already has its own
|
|
51
|
+
# DATABASE_URL pointed at a different database (so the two never
|
|
52
|
+
# collide), or just to keep this router's own connection string
|
|
53
|
+
# explicitly distinct from whatever else reads DATABASE_URL in your stack.
|
|
54
|
+
# MEMOCODE_ROUTER_DATABASE_URL=postgres://user:pass@host:5432/dbname
|
|
55
|
+
|
|
56
|
+
# Optional: where JSONL metrics files live, if Postgres isn't
|
|
57
|
+
# configured. Despite the name, this now names a DIRECTORY (kept for
|
|
58
|
+
# backward compatibility with older configs that pointed it at a
|
|
59
|
+
# single file - see metrics.js's own comment). Defaults to ./data
|
|
60
|
+
# next to this file.
|
|
61
|
+
# METRICS_LOG_PATH=./data
|
|
62
|
+
|
|
63
|
+
# Optional: how long metrics history is kept before pruneOlderThan()'s
|
|
64
|
+
# scheduled daily job deletes it. Default 90 days deliberately matches
|
|
65
|
+
# /dashboard/data's own longest supported range - pruning any sooner
|
|
66
|
+
# would make its "Last 90 days" option quietly lie.
|
|
67
|
+
# METRICS_RETENTION_DAYS=90
|
|
68
|
+
|
|
69
|
+
# Optional: semantic cache tuning (all have sane defaults - see README's
|
|
70
|
+
# "Two kinds of cache hit" section before changing these).
|
|
71
|
+
# SEMANTIC_CACHE_ENABLED=false # hard off-switch, even with a key configured
|
|
72
|
+
# SEMANTIC_CACHE_THRESHOLD=0.93 # cosine similarity floor for a match
|
|
73
|
+
# SEMANTIC_CACHE_MAX_CANDIDATES=200 # per-model list cap (brute-force scan size)
|
|
74
|
+
# SEMANTIC_CACHE_TTL_SECONDS=3600
|
|
75
|
+
|
|
76
|
+
# Optional: how many recent requests a provider needs before its error
|
|
77
|
+
# rate is treated as meaningful health data (router.js). Below this
|
|
78
|
+
# sample size, a provider is always considered healthy - a single
|
|
79
|
+
# unlucky request (1/1 or 1/2 errors) shouldn't bounce it out of
|
|
80
|
+
# rotation; it's noise, not a signal, until enough requests have run.
|
|
81
|
+
# ROUTER_HEALTH_MIN_SAMPLES=5
|
|
82
|
+
|
|
83
|
+
# Optional: hard timeout (ms) on each semantic-cache embedding call
|
|
84
|
+
# (embeddings.js). These sit on the hot request path (1-2 per cache
|
|
85
|
+
# miss, when SEMANTIC_CACHE_ENABLED is on) - a hung embedding provider
|
|
86
|
+
# would otherwise stall every chat request, including Anthropic-only
|
|
87
|
+
# ones with nothing to do with OpenAI. On timeout, the request degrades
|
|
88
|
+
# to "skip semantic caching" rather than hanging.
|
|
89
|
+
# EMBEDDING_TIMEOUT_MS=5000
|
|
90
|
+
|
|
91
|
+
# Optional: override the default "router:" virtual-model tiers
|
|
92
|
+
# (router.js's own DEFAULT_TIERS) with your own JSON, e.g. to add a
|
|
93
|
+
# model, swap a provider, or define a new tier name entirely. Must be
|
|
94
|
+
# valid JSON matching the shape:
|
|
95
|
+
# {"router:fast-cheap":[{"provider":"openai","model":"gpt-4o-mini"}]}
|
|
96
|
+
# Invalid JSON logs a warning and falls back to the built-in defaults
|
|
97
|
+
# rather than crashing.
|
|
98
|
+
# ROUTER_TIERS_JSON={"router:fast-cheap":[...]}
|
|
99
|
+
|
|
100
|
+
# Optional: routing strategy for "router:" virtual models (router.js).
|
|
101
|
+
# See README's "Where this leaves things" section for what each one
|
|
102
|
+
# actually does before changing this - there's no blended cost/latency
|
|
103
|
+
# score, only these three explicit options.
|
|
104
|
+
# ROUTER_STRATEGY=cost # default: cheapest healthy candidate
|
|
105
|
+
# ROUTER_STRATEGY=latency # fastest healthy candidate, cost only as a tiebreaker
|
|
106
|
+
# ROUTER_STRATEGY=latency-guarded-cost # cheapest healthy candidate, excluding any that's too much slower than the fastest known one
|
|
107
|
+
# ROUTER_LATENCY_GUARD_MULTIPLIER=3 # only used by latency-guarded-cost
|
|
108
|
+
|
|
109
|
+
# Optional: rate limiting on /v1/* (server.js). Default 300 requests
|
|
110
|
+
# per 60 seconds - shared across EVERY caller of this router combined,
|
|
111
|
+
# not per end user if you're fronting it with your own per-user auth.
|
|
112
|
+
# See server.js's own comment above the rate limiter for the reasoning.
|
|
113
|
+
# RATE_LIMIT_MAX=300
|
|
114
|
+
# RATE_LIMIT_WINDOW_MS=60000
|
|
115
|
+
|
|
116
|
+
# Optional: separate, more generous rate limit for the read-only
|
|
117
|
+
# aggregate endpoints (/stats, /dashboard/data) - lower stakes than
|
|
118
|
+
# /v1 (no provider spend on the line) but still real server work.
|
|
119
|
+
# Shares RATE_LIMIT_WINDOW_MS above.
|
|
120
|
+
# READ_RATE_LIMIT_MAX=120
|
|
121
|
+
|
|
122
|
+
# Optional: max JSON body size accepted on /v1/* (server.js). Default
|
|
123
|
+
# 2mb comfortably covers even a very long text conversation - this
|
|
124
|
+
# router doesn't support image/multimodal content, so there's no
|
|
125
|
+
# legitimate reason for a much larger payload. Raise only if you have
|
|
126
|
+
# a specific reason to expect longer request bodies.
|
|
127
|
+
# JSON_BODY_LIMIT=2mb
|
package/README.md
CHANGED
|
@@ -21,8 +21,9 @@ fills instead:
|
|
|
21
21
|
now ship a more sophisticated vector-indexed semantic cache than this
|
|
22
22
|
project's brute-force cosine scan — stated plainly, not glossed over;
|
|
23
23
|
see "Two kinds of cache hit" below for what this one actually does.)
|
|
24
|
-
- **Node.js
|
|
25
|
-
this fits directly into a JS/TS stack with no
|
|
24
|
+
- **Node.js-native (plain JavaScript, no build step)** — most comparable
|
|
25
|
+
gateways are Python; this fits directly into a JS/TS stack with no
|
|
26
|
+
cross-language bridge.
|
|
26
27
|
- **Small and embeddable** — a handful of files, no framework beyond
|
|
27
28
|
Express, easy to read end to end and drop into an existing app's own
|
|
28
29
|
backend rather than standing up a separate service.
|
|
@@ -224,12 +225,15 @@ curl http://localhost:4000/v1/chat/completions \
|
|
|
224
225
|
}'
|
|
225
226
|
```
|
|
226
227
|
|
|
227
|
-
`GET /
|
|
228
|
-
strategy
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
228
|
+
`GET /stats` (auth required) lists the configured tiers and the active
|
|
229
|
+
routing strategy, alongside which providers have a key configured -
|
|
230
|
+
security-review finding (2026-09-02): this used to live on the PUBLIC
|
|
231
|
+
`GET /health` instead, world-readable internal routing configuration
|
|
232
|
+
with no reason to be. Tiers are defined in `router.js`
|
|
233
|
+
(`DEFAULT_TIERS`) and can be overridden per deployment via the
|
|
234
|
+
`ROUTER_TIERS_JSON` env var; the strategy is `ROUTER_STRATEGY` (`cost` /
|
|
235
|
+
`latency` / `latency-guarded-cost`, default `cost`) - see "Where this
|
|
236
|
+
leaves things" below for what each one actually does.
|
|
233
237
|
|
|
234
238
|
Streamed dispatch - add `"stream": true` to either form above and get
|
|
235
239
|
back SSE chunks instead of one JSON body (see "Streaming" below for
|
|
@@ -263,7 +267,12 @@ auth, for local development only.
|
|
|
263
267
|
- OpenAI-compatible `/v1/chat/completions` endpoint - direct dispatch to
|
|
264
268
|
a named provider model, or routed dispatch via a `router:` capability
|
|
265
269
|
tier (cheapest currently-healthy candidate, by estimated cost; see
|
|
266
|
-
`router.js`).
|
|
270
|
+
`router.js`). "Unhealthy" means a recent error rate of 50% or higher
|
|
271
|
+
over that provider's own rolling request window - but only once it has
|
|
272
|
+
at least `ROUTER_HEALTH_MIN_SAMPLES` (default 5) recent requests to
|
|
273
|
+
judge from; below that, a provider is always treated as healthy, so a
|
|
274
|
+
single unlucky request (1/1 or 1/2 errors) can't bounce it out of
|
|
275
|
+
rotation on noise alone.
|
|
267
276
|
- **`stream: true` works** for plain text content, on both providers,
|
|
268
277
|
including replaying a cache hit (exact or semantic) as a stream so a
|
|
269
278
|
streaming caller still gets the caching benefit. See "Streaming"
|
|
@@ -287,10 +296,12 @@ auth, for local development only.
|
|
|
287
296
|
history to work from, not just a number thrown away after each
|
|
288
297
|
response.
|
|
289
298
|
- Rate limiting on `/v1/*` (`RATE_LIMIT_MAX` requests per
|
|
290
|
-
`RATE_LIMIT_WINDOW_MS`, defaults
|
|
299
|
+
`RATE_LIMIT_WINDOW_MS`, defaults 300/60s) - this proxy sits in front of
|
|
291
300
|
paid, metered keys, so an unbounded client has no ceiling otherwise.
|
|
292
|
-
- `GET /health` for monitoring (public, no auth
|
|
293
|
-
|
|
301
|
+
- `GET /health` for monitoring (public, no auth - deliberately minimal:
|
|
302
|
+
process/dependency status only, no provider or routing configuration)
|
|
303
|
+
and `GET /stats` for a quick record-count-windowed aggregate snapshot,
|
|
304
|
+
plus the configured providers/tiers/strategy (auth required).
|
|
294
305
|
- **A cost dashboard** at `GET /dashboard` - a static page (no auth
|
|
295
306
|
itself; its own JS asks for the internal key and stores it in
|
|
296
307
|
localStorage, then calls the authenticated data endpoint below) with
|
|
@@ -333,6 +344,13 @@ near-duplicate traffic is common; it's pure overhead when it isn't. Set
|
|
|
333
344
|
`SEMANTIC_CACHE_ENABLED=false` to disable it outright while keeping the
|
|
334
345
|
exact-match cache and `OPENAI_API_KEY` for other things.
|
|
335
346
|
|
|
347
|
+
Both embedding calls carry a hard timeout (`EMBEDDING_TIMEOUT_MS`,
|
|
348
|
+
default 5000ms) - they sit on the hot request path of every cache miss,
|
|
349
|
+
so a hung embedding provider degrades to "skip semantic caching for this
|
|
350
|
+
request" instead of stalling chat traffic that has nothing to do with
|
|
351
|
+
OpenAI (an Anthropic-only request still needs an embedding call to check
|
|
352
|
+
the semantic cache).
|
|
353
|
+
|
|
336
354
|
Storage is a plain Redis list per model, capped at
|
|
337
355
|
`SEMANTIC_CACHE_MAX_CANDIDATES` (default 200) - a lookup does a
|
|
338
356
|
brute-force cosine-similarity scan over that list in Node, not an
|
|
@@ -447,7 +465,7 @@ positioning. A few things worth knowing before relying on it:
|
|
|
447
465
|
fraction of a cent, not a full re-ranking. With no latency data at
|
|
448
466
|
all yet, it degrades to plain `cost`.
|
|
449
467
|
|
|
450
|
-
`GET /
|
|
468
|
+
`GET /stats` reports the active strategy (`routing_strategy`); a
|
|
451
469
|
decision's `reason.strategy` and `reason.latencyGuardExcludedACandidate`
|
|
452
470
|
say which one ran and whether the guard actually did anything, same
|
|
453
471
|
transparency style as the rest of the routing decision. Tier
|
package/cache.js
CHANGED
|
@@ -1,51 +1,72 @@
|
|
|
1
|
-
// model-router/cache.js
|
|
2
|
-
//
|
|
3
|
-
// The exact-match cache: same model + same messages + same params ->
|
|
4
|
-
// same cached response, by content hash. First and free - checked
|
|
5
|
-
// before the semantic cache (semanticCache.js), which is slower (an
|
|
6
|
-
// embedding call) and probabilistic (a similarity threshold, not an
|
|
7
|
-
// exact match). Connection is shared via redisClient.js.
|
|
8
|
-
|
|
9
|
-
const crypto = require('crypto');
|
|
10
|
-
const redis = require('./redisClient');
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
}
|
|
1
|
+
// model-router/cache.js
|
|
2
|
+
//
|
|
3
|
+
// The exact-match cache: same model + same messages + same params ->
|
|
4
|
+
// same cached response, by content hash. First and free - checked
|
|
5
|
+
// before the semantic cache (semanticCache.js), which is slower (an
|
|
6
|
+
// embedding call) and probabilistic (a similarity threshold, not an
|
|
7
|
+
// exact match). Connection is shared via redisClient.js.
|
|
8
|
+
|
|
9
|
+
const crypto = require('crypto');
|
|
10
|
+
const redis = require('./redisClient');
|
|
11
|
+
|
|
12
|
+
// `scope` (seams work, roadmap: engine/cloud "wrap it, don't fork it"):
|
|
13
|
+
// an opaque, caller-supplied isolation key - a tenant id, a namespace,
|
|
14
|
+
// whatever a wrapping deployment needs two callers to never share a
|
|
15
|
+
// cache entry over. null/undefined (every call in this codebase today)
|
|
16
|
+
// means exactly what it always has: one global cache, no isolation -
|
|
17
|
+
// scope is omitted from both the key prefix AND the hashed payload in
|
|
18
|
+
// that case, so an unconfigured deployment's cache keys are BYTE-
|
|
19
|
+
// IDENTICAL to before this parameter existed (no cache invalidation on
|
|
20
|
+
// upgrade). When a caller does pass a scope, it's folded into both the
|
|
21
|
+
// prefix and the hash (not the prefix alone) - so two scopes are
|
|
22
|
+
// isolated even if the caller's own scope-naming convention were ever
|
|
23
|
+
// guessed or leaked; a compromised/guessed prefix alone can't be walked
|
|
24
|
+
// into another scope's cached content.
|
|
25
|
+
function buildCacheKey(scope, payload) {
|
|
26
|
+
const normalized = JSON.stringify({
|
|
27
|
+
...(scope != null ? { scope } : {}),
|
|
28
|
+
model: payload.model,
|
|
29
|
+
messages: payload.messages,
|
|
30
|
+
temperature: payload.temperature ?? 0.0,
|
|
31
|
+
max_tokens: payload.max_tokens,
|
|
32
|
+
tools: payload.tools,
|
|
33
|
+
tool_choice: payload.tool_choice,
|
|
34
|
+
// response_format changes the SHAPE of the answer (json_object vs
|
|
35
|
+
// plain text), so it must participate in the key too - otherwise a
|
|
36
|
+
// cached plain-text response could be served to a json_object caller
|
|
37
|
+
// (or vice versa). openai.js forwards it (see its own chat()); this
|
|
38
|
+
// file used to omit it, making an "exact" hit not always exact.
|
|
39
|
+
response_format: payload.response_format
|
|
40
|
+
});
|
|
41
|
+
const hash = crypto.createHash('sha256').update(normalized).digest('hex');
|
|
42
|
+
const prefix = scope != null ? `ROUTER:${scope}:` : 'ROUTER:';
|
|
43
|
+
return `${prefix}${payload.model}:${hash}`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
module.exports = {
|
|
47
|
+
buildCacheKey,
|
|
48
|
+
|
|
49
|
+
isConnected() {
|
|
50
|
+
return redis.isConnected();
|
|
51
|
+
},
|
|
52
|
+
|
|
53
|
+
async get(scope, payload) {
|
|
54
|
+
if (!redis.isConnected()) return null;
|
|
55
|
+
try {
|
|
56
|
+
const cached = await redis.client.get(buildCacheKey(scope, payload));
|
|
57
|
+
return cached ? JSON.parse(cached) : null;
|
|
58
|
+
} catch (err) {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
|
|
63
|
+
async set(scope, payload, response, ttlSeconds = 3600) {
|
|
64
|
+
if (!redis.isConnected()) return false;
|
|
65
|
+
try {
|
|
66
|
+
await redis.client.set(buildCacheKey(scope, payload), JSON.stringify(response), { EX: ttlSeconds });
|
|
67
|
+
return true;
|
|
68
|
+
} catch (err) {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
};
|
package/embeddings.js
CHANGED
|
@@ -1,32 +1,42 @@
|
|
|
1
|
-
// model-router/embeddings.js
|
|
2
|
-
//
|
|
3
|
-
// The only embedding backend right now is OpenAI's - which means
|
|
4
|
-
// semantic caching needs an OPENAI_API_KEY configured even for a
|
|
5
|
-
// deployment that only ever talks to Anthropic for chat. That's a real
|
|
6
|
-
// constraint, not hidden: isEnabled() is what semanticCache.js checks
|
|
7
|
-
// before doing anything, and it degrades to "disabled" (not an error)
|
|
8
|
-
// when the key isn't set, same as the Redis cache does when REDIS_URL
|
|
9
|
-
// isn't set.
|
|
10
|
-
|
|
11
|
-
const { OpenAI } = require('openai');
|
|
12
|
-
|
|
13
|
-
let client;
|
|
14
|
-
function getClient() {
|
|
15
|
-
if (!client) client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
|
16
|
-
return client;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
function isEnabled() {
|
|
20
|
-
return Boolean(process.env.OPENAI_API_KEY);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
async function embed(text) {
|
|
24
|
-
if (!isEnabled()) {
|
|
25
|
-
throw new Error('OPENAI_API_KEY not configured - embeddings unavailable');
|
|
26
|
-
}
|
|
27
|
-
const model = process.env.EMBEDDING_MODEL || 'text-embedding-3-small';
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
1
|
+
// model-router/embeddings.js
|
|
2
|
+
//
|
|
3
|
+
// The only embedding backend right now is OpenAI's - which means
|
|
4
|
+
// semantic caching needs an OPENAI_API_KEY configured even for a
|
|
5
|
+
// deployment that only ever talks to Anthropic for chat. That's a real
|
|
6
|
+
// constraint, not hidden: isEnabled() is what semanticCache.js checks
|
|
7
|
+
// before doing anything, and it degrades to "disabled" (not an error)
|
|
8
|
+
// when the key isn't set, same as the Redis cache does when REDIS_URL
|
|
9
|
+
// isn't set.
|
|
10
|
+
|
|
11
|
+
const { OpenAI } = require('openai');
|
|
12
|
+
|
|
13
|
+
let client;
|
|
14
|
+
function getClient() {
|
|
15
|
+
if (!client) client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
|
16
|
+
return client;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function isEnabled() {
|
|
20
|
+
return Boolean(process.env.OPENAI_API_KEY);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function embed(text) {
|
|
24
|
+
if (!isEnabled()) {
|
|
25
|
+
throw new Error('OPENAI_API_KEY not configured - embeddings unavailable');
|
|
26
|
+
}
|
|
27
|
+
const model = process.env.EMBEDDING_MODEL || 'text-embedding-3-small';
|
|
28
|
+
// A hard timeout per embedding call: these run on the hot request path
|
|
29
|
+
// (semanticCache.js's findMatch/store, one or two per miss), so a hung
|
|
30
|
+
// embedding provider must not be able to stall every chat request -
|
|
31
|
+
// including ones that never touch embeddings at all if the semantic
|
|
32
|
+
// cache is enabled. AbortSignal.timeout aborts the underlying fetch, and
|
|
33
|
+
// the thrown error is caught by semanticCache.js's own try/catch, which
|
|
34
|
+
// degrades to "skip semantic caching" rather than failing the request.
|
|
35
|
+
const response = await getClient().embeddings.create(
|
|
36
|
+
{ model, input: text },
|
|
37
|
+
{ signal: AbortSignal.timeout(Number(process.env.EMBEDDING_TIMEOUT_MS) || 5000) }
|
|
38
|
+
);
|
|
39
|
+
return response.data[0].embedding;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
module.exports = { isEnabled, embed };
|