cachegate 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/.env.example CHANGED
@@ -1,112 +1,127 @@
1
- # cachegate environment template
2
- # Copy the values you need into your real .env file.
3
- # NEVER commit .env to Git.
4
-
5
- PORT=4000
6
-
7
- # Required: protects the router from unauthorized use. The server
8
- # refuses to start without this UNLESS ALLOW_INSECURE_LOCAL_DEV=true
9
- # (below) is explicitly set - a missing key never silently means "no
10
- # auth enforced."
11
- # Generate with: openssl rand -hex 32
12
- MODEL_ROUTER_INTERNAL_KEY=your-random-internal-key
13
-
14
- # Optional: skip the MODEL_ROUTER_INTERNAL_KEY requirement above, for a
15
- # throwaway LOCAL instance only. Never set this on anything reachable
16
- # from outside your own machine.
17
- # ALLOW_INSECURE_LOCAL_DEV=true
18
-
19
- # Required (at least one provider key - Anthropic, OpenAI, or both).
20
- # The model itself is named PER REQUEST in the API call's own "model"
21
- # field (see README's "Usage" section), not configured here - there is
22
- # no ANTHROPIC_MODEL/OPENAI_MODEL env var to set.
23
- ANTHROPIC_API_KEY=sk-ant-api03-...
24
-
25
- # Also required for the semantic cache's embeddings, regardless of
26
- # which provider actually serves chat requests - it's the only
27
- # embedding backend implemented (see semanticCache.js / README).
28
- # OPENAI_API_KEY=sk-proj-...
29
-
30
- # Optional: which OpenAI embedding model the semantic cache uses.
31
- # Only matters if OPENAI_API_KEY is set. Default is a good balance of
32
- # cost and quality; change only if you know you want a different one.
33
- # EMBEDDING_MODEL=text-embedding-3-small
34
-
35
- # Optional: Redis for response caching (strongly recommended for cost savings)
36
- # Local development:
37
- # REDIS_URL=redis://localhost:6379
38
- # Render Key Value:
39
- # REDIS_URL=rediss://default:PASSWORD@HOST:PORT
40
-
41
- # Optional: persistent metrics storage (see metrics.js's own comment).
42
- # Unset (the default) keeps metrics as local JSONL files, which don't
43
- # survive a restart/redeploy on most hosts - fine for a standalone
44
- # deployment with no database of its own. Set this to a real Postgres
45
- # connection string to persist metrics there instead.
46
- # DATABASE_URL=postgres://user:pass@host:5432/dbname
47
- #
48
- # MEMOCODE_ROUTER_DATABASE_URL does the exact same thing and takes
49
- # priority over DATABASE_URL if both are set - use this instead when
50
- # embedding cachegate inside an app that already has its own
51
- # DATABASE_URL pointed at a different database (so the two never
52
- # collide), or just to keep this router's own connection string
53
- # explicitly distinct from whatever else reads DATABASE_URL in your stack.
54
- # MEMOCODE_ROUTER_DATABASE_URL=postgres://user:pass@host:5432/dbname
55
-
56
- # Optional: where JSONL metrics files live, if Postgres isn't
57
- # configured. Despite the name, this now names a DIRECTORY (kept for
58
- # backward compatibility with older configs that pointed it at a
59
- # single file - see metrics.js's own comment). Defaults to ./data
60
- # next to this file.
61
- # METRICS_LOG_PATH=./data
62
-
63
- # Optional: how long metrics history is kept before pruneOlderThan()'s
64
- # scheduled daily job deletes it. Default 90 days deliberately matches
65
- # /dashboard/data's own longest supported range - pruning any sooner
66
- # would make its "Last 90 days" option quietly lie.
67
- # METRICS_RETENTION_DAYS=90
68
-
69
- # Optional: semantic cache tuning (all have sane defaults - see README's
70
- # "Two kinds of cache hit" section before changing these).
71
- # SEMANTIC_CACHE_ENABLED=false # hard off-switch, even with a key configured
72
- # SEMANTIC_CACHE_THRESHOLD=0.93 # cosine similarity floor for a match
73
- # SEMANTIC_CACHE_MAX_CANDIDATES=200 # per-model list cap (brute-force scan size)
74
- # SEMANTIC_CACHE_TTL_SECONDS=3600
75
-
76
- # Optional: override the default "router:" virtual-model tiers
77
- # (router.js's own DEFAULT_TIERS) with your own JSON, e.g. to add a
78
- # model, swap a provider, or define a new tier name entirely. Must be
79
- # valid JSON matching the shape:
80
- # {"router:fast-cheap":[{"provider":"openai","model":"gpt-4o-mini"}]}
81
- # Invalid JSON logs a warning and falls back to the built-in defaults
82
- # rather than crashing.
83
- # ROUTER_TIERS_JSON={"router:fast-cheap":[...]}
84
-
85
- # Optional: routing strategy for "router:" virtual models (router.js).
86
- # See README's "Where this leaves things" section for what each one
87
- # actually does before changing this - there's no blended cost/latency
88
- # score, only these three explicit options.
89
- # ROUTER_STRATEGY=cost # default: cheapest healthy candidate
90
- # ROUTER_STRATEGY=latency # fastest healthy candidate, cost only as a tiebreaker
91
- # ROUTER_STRATEGY=latency-guarded-cost # cheapest healthy candidate, excluding any that's too much slower than the fastest known one
92
- # ROUTER_LATENCY_GUARD_MULTIPLIER=3 # only used by latency-guarded-cost
93
-
94
- # Optional: rate limiting on /v1/* (server.js). Default 300 requests
95
- # per 60 seconds - shared across EVERY caller of this router combined,
96
- # not per end user if you're fronting it with your own per-user auth.
97
- # See server.js's own comment above the rate limiter for the reasoning.
98
- # RATE_LIMIT_MAX=300
99
- # RATE_LIMIT_WINDOW_MS=60000
100
-
101
- # Optional: separate, more generous rate limit for the read-only
102
- # aggregate endpoints (/stats, /dashboard/data) - lower stakes than
103
- # /v1 (no provider spend on the line) but still real server work.
104
- # Shares RATE_LIMIT_WINDOW_MS above.
105
- # READ_RATE_LIMIT_MAX=120
106
-
107
- # Optional: max JSON body size accepted on /v1/* (server.js). Default
108
- # 2mb comfortably covers even a very long text conversation - this
109
- # router doesn't support image/multimodal content, so there's no
110
- # legitimate reason for a much larger payload. Raise only if you have
111
- # a specific reason to expect longer request bodies.
112
- # JSON_BODY_LIMIT=2mb
1
+ # cachegate environment template
2
+ # Copy the values you need into your real .env file.
3
+ # NEVER commit .env to Git.
4
+
5
+ PORT=4000
6
+
7
+ # Required: protects the router from unauthorized use. The server
8
+ # refuses to start without this UNLESS ALLOW_INSECURE_LOCAL_DEV=true
9
+ # (below) is explicitly set - a missing key never silently means "no
10
+ # auth enforced."
11
+ # Generate with: openssl rand -hex 32
12
+ MODEL_ROUTER_INTERNAL_KEY=your-random-internal-key
13
+
14
+ # Optional: skip the MODEL_ROUTER_INTERNAL_KEY requirement above, for a
15
+ # throwaway LOCAL instance only. Never set this on anything reachable
16
+ # from outside your own machine.
17
+ # ALLOW_INSECURE_LOCAL_DEV=true
18
+
19
+ # Required (at least one provider key - Anthropic, OpenAI, or both).
20
+ # The model itself is named PER REQUEST in the API call's own "model"
21
+ # field (see README's "Usage" section), not configured here - there is
22
+ # no ANTHROPIC_MODEL/OPENAI_MODEL env var to set.
23
+ ANTHROPIC_API_KEY=sk-ant-api03-...
24
+
25
+ # Also required for the semantic cache's embeddings, regardless of
26
+ # which provider actually serves chat requests - it's the only
27
+ # embedding backend implemented (see semanticCache.js / README).
28
+ # OPENAI_API_KEY=sk-proj-...
29
+
30
+ # Optional: which OpenAI embedding model the semantic cache uses.
31
+ # Only matters if OPENAI_API_KEY is set. Default is a good balance of
32
+ # cost and quality; change only if you know you want a different one.
33
+ # EMBEDDING_MODEL=text-embedding-3-small
34
+
35
+ # Optional: Redis for response caching (strongly recommended for cost savings)
36
+ # Local development:
37
+ # REDIS_URL=redis://localhost:6379
38
+ # Render Key Value:
39
+ # REDIS_URL=rediss://default:PASSWORD@HOST:PORT
40
+
41
+ # Optional: persistent metrics storage (see metrics.js's own comment).
42
+ # Unset (the default) keeps metrics as local JSONL files, which don't
43
+ # survive a restart/redeploy on most hosts - fine for a standalone
44
+ # deployment with no database of its own. Set this to a real Postgres
45
+ # connection string to persist metrics there instead.
46
+ # DATABASE_URL=postgres://user:pass@host:5432/dbname
47
+ #
48
+ # MEMOCODE_ROUTER_DATABASE_URL does the exact same thing and takes
49
+ # priority over DATABASE_URL if both are set - use this instead when
50
+ # embedding cachegate inside an app that already has its own
51
+ # DATABASE_URL pointed at a different database (so the two never
52
+ # collide), or just to keep this router's own connection string
53
+ # explicitly distinct from whatever else reads DATABASE_URL in your stack.
54
+ # MEMOCODE_ROUTER_DATABASE_URL=postgres://user:pass@host:5432/dbname
55
+
56
+ # Optional: where JSONL metrics files live, if Postgres isn't
57
+ # configured. Despite the name, this now names a DIRECTORY (kept for
58
+ # backward compatibility with older configs that pointed it at a
59
+ # single file - see metrics.js's own comment). Defaults to ./data
60
+ # next to this file.
61
+ # METRICS_LOG_PATH=./data
62
+
63
+ # Optional: how long metrics history is kept before pruneOlderThan()'s
64
+ # scheduled daily job deletes it. Default 90 days deliberately matches
65
+ # /dashboard/data's own longest supported range - pruning any sooner
66
+ # would make its "Last 90 days" option quietly lie.
67
+ # METRICS_RETENTION_DAYS=90
68
+
69
+ # Optional: semantic cache tuning (all have sane defaults - see README's
70
+ # "Two kinds of cache hit" section before changing these).
71
+ # SEMANTIC_CACHE_ENABLED=false # hard off-switch, even with a key configured
72
+ # SEMANTIC_CACHE_THRESHOLD=0.93 # cosine similarity floor for a match
73
+ # SEMANTIC_CACHE_MAX_CANDIDATES=200 # per-model list cap (brute-force scan size)
74
+ # SEMANTIC_CACHE_TTL_SECONDS=3600
75
+
76
+ # Optional: how many recent requests a provider needs before its error
77
+ # rate is treated as meaningful health data (router.js). Below this
78
+ # sample size, a provider is always considered healthy - a single
79
+ # unlucky request (1/1 or 1/2 errors) shouldn't bounce it out of
80
+ # rotation; it's noise, not a signal, until enough requests have run.
81
+ # ROUTER_HEALTH_MIN_SAMPLES=5
82
+
83
+ # Optional: hard timeout (ms) on each semantic-cache embedding call
84
+ # (embeddings.js). These sit on the hot request path (1-2 per cache
85
+ # miss, when SEMANTIC_CACHE_ENABLED is on) - a hung embedding provider
86
+ # would otherwise stall every chat request, including Anthropic-only
87
+ # ones with nothing to do with OpenAI. On timeout, the request degrades
88
+ # to "skip semantic caching" rather than hanging.
89
+ # EMBEDDING_TIMEOUT_MS=5000
90
+
91
+ # Optional: override the default "router:" virtual-model tiers
92
+ # (router.js's own DEFAULT_TIERS) with your own JSON, e.g. to add a
93
+ # model, swap a provider, or define a new tier name entirely. Must be
94
+ # valid JSON matching the shape:
95
+ # {"router:fast-cheap":[{"provider":"openai","model":"gpt-4o-mini"}]}
96
+ # Invalid JSON logs a warning and falls back to the built-in defaults
97
+ # rather than crashing.
98
+ # ROUTER_TIERS_JSON={"router:fast-cheap":[...]}
99
+
100
+ # Optional: routing strategy for "router:" virtual models (router.js).
101
+ # See README's "Where this leaves things" section for what each one
102
+ # actually does before changing this - there's no blended cost/latency
103
+ # score, only these three explicit options.
104
+ # ROUTER_STRATEGY=cost # default: cheapest healthy candidate
105
+ # ROUTER_STRATEGY=latency # fastest healthy candidate, cost only as a tiebreaker
106
+ # ROUTER_STRATEGY=latency-guarded-cost # cheapest healthy candidate, excluding any that's too much slower than the fastest known one
107
+ # ROUTER_LATENCY_GUARD_MULTIPLIER=3 # only used by latency-guarded-cost
108
+
109
+ # Optional: rate limiting on /v1/* (server.js). Default 300 requests
110
+ # per 60 seconds - shared across EVERY caller of this router combined,
111
+ # not per end user if you're fronting it with your own per-user auth.
112
+ # See server.js's own comment above the rate limiter for the reasoning.
113
+ # RATE_LIMIT_MAX=300
114
+ # RATE_LIMIT_WINDOW_MS=60000
115
+
116
+ # Optional: separate, more generous rate limit for the read-only
117
+ # aggregate endpoints (/stats, /dashboard/data) - lower stakes than
118
+ # /v1 (no provider spend on the line) but still real server work.
119
+ # Shares RATE_LIMIT_WINDOW_MS above.
120
+ # READ_RATE_LIMIT_MAX=120
121
+
122
+ # Optional: max JSON body size accepted on /v1/* (server.js). Default
123
+ # 2mb comfortably covers even a very long text conversation - this
124
+ # router doesn't support image/multimodal content, so there's no
125
+ # legitimate reason for a much larger payload. Raise only if you have
126
+ # a specific reason to expect longer request bodies.
127
+ # JSON_BODY_LIMIT=2mb
package/README.md CHANGED
@@ -21,8 +21,9 @@ fills instead:
21
21
  now ship a more sophisticated vector-indexed semantic cache than this
22
22
  project's brute-force cosine scan — stated plainly, not glossed over;
23
23
  see "Two kinds of cache hit" below for what this one actually does.)
24
- - **Node.js/TypeScript-native** — most comparable gateways are Python;
25
- this fits directly into a JS/TS stack with no cross-language bridge.
24
+ - **Node.js-native (plain JavaScript, no build step)** — most comparable
25
+ gateways are Python; this fits directly into a JS/TS stack with no
26
+ cross-language bridge.
26
27
  - **Small and embeddable** — a handful of files, no framework beyond
27
28
  Express, easy to read end to end and drop into an existing app's own
28
29
  backend rather than standing up a separate service.
@@ -224,12 +225,15 @@ curl http://localhost:4000/v1/chat/completions \
224
225
  }'
225
226
  ```
226
227
 
227
- `GET /health` lists the configured tiers and the active routing
228
- strategy. Tiers are defined in `router.js` (`DEFAULT_TIERS`) and can be
229
- overridden per deployment via the `ROUTER_TIERS_JSON` env var; the
230
- strategy is `ROUTER_STRATEGY` (`cost` / `latency` / `latency-guarded-cost`,
231
- default `cost`) - see "Where this leaves things" below for what each
232
- one actually does.
228
+ `GET /stats` (auth required) lists the configured tiers and the active
229
+ routing strategy, alongside which providers have a key configured -
230
+ security-review finding (2026-09-02): this used to live on the PUBLIC
231
+ `GET /health` instead, world-readable internal routing configuration
232
+ with no reason to be. Tiers are defined in `router.js`
233
+ (`DEFAULT_TIERS`) and can be overridden per deployment via the
234
+ `ROUTER_TIERS_JSON` env var; the strategy is `ROUTER_STRATEGY` (`cost` /
235
+ `latency` / `latency-guarded-cost`, default `cost`) - see "Where this
236
+ leaves things" below for what each one actually does.
233
237
 
234
238
  Streamed dispatch - add `"stream": true` to either form above and get
235
239
  back SSE chunks instead of one JSON body (see "Streaming" below for
@@ -263,7 +267,12 @@ auth, for local development only.
263
267
  - OpenAI-compatible `/v1/chat/completions` endpoint - direct dispatch to
264
268
  a named provider model, or routed dispatch via a `router:` capability
265
269
  tier (cheapest currently-healthy candidate, by estimated cost; see
266
- `router.js`).
270
+ `router.js`). "Unhealthy" means a recent error rate of 50% or higher
271
+ over that provider's own rolling request window - but only once it has
272
+ at least `ROUTER_HEALTH_MIN_SAMPLES` (default 5) recent requests to
273
+ judge from; below that, a provider is always treated as healthy, so a
274
+ single unlucky request (1/1 or 1/2 errors) can't bounce it out of
275
+ rotation on noise alone.
267
276
  - **`stream: true` works** for plain text content, on both providers,
268
277
  including replaying a cache hit (exact or semantic) as a stream so a
269
278
  streaming caller still gets the caching benefit. See "Streaming"
@@ -287,10 +296,12 @@ auth, for local development only.
287
296
  history to work from, not just a number thrown away after each
288
297
  response.
289
298
  - Rate limiting on `/v1/*` (`RATE_LIMIT_MAX` requests per
290
- `RATE_LIMIT_WINDOW_MS`, defaults 60/60s) - this proxy sits in front of
299
+ `RATE_LIMIT_WINDOW_MS`, defaults 300/60s) - this proxy sits in front of
291
300
  paid, metered keys, so an unbounded client has no ceiling otherwise.
292
- - `GET /health` for monitoring (public, no auth) and `GET /stats` for a
293
- quick record-count-windowed aggregate snapshot (auth required).
301
+ - `GET /health` for monitoring (public, no auth - deliberately minimal:
302
+ process/dependency status only, no provider or routing configuration)
303
+ and `GET /stats` for a quick record-count-windowed aggregate snapshot,
304
+ plus the configured providers/tiers/strategy (auth required).
294
305
  - **A cost dashboard** at `GET /dashboard` - a static page (no auth
295
306
  itself; its own JS asks for the internal key and stores it in
296
307
  localStorage, then calls the authenticated data endpoint below) with
@@ -333,6 +344,13 @@ near-duplicate traffic is common; it's pure overhead when it isn't. Set
333
344
  `SEMANTIC_CACHE_ENABLED=false` to disable it outright while keeping the
334
345
  exact-match cache and `OPENAI_API_KEY` for other things.
335
346
 
347
+ Both embedding calls carry a hard timeout (`EMBEDDING_TIMEOUT_MS`,
348
+ default 5000ms) - they sit on the hot request path of every cache miss,
349
+ so a hung embedding provider degrades to "skip semantic caching for this
350
+ request" instead of stalling chat traffic that has nothing to do with
351
+ OpenAI (an Anthropic-only request still needs an embedding call to check
352
+ the semantic cache).
353
+
336
354
  Storage is a plain Redis list per model, capped at
337
355
  `SEMANTIC_CACHE_MAX_CANDIDATES` (default 200) - a lookup does a
338
356
  brute-force cosine-similarity scan over that list in Node, not an
@@ -447,7 +465,7 @@ positioning. A few things worth knowing before relying on it:
447
465
  fraction of a cent, not a full re-ranking. With no latency data at
448
466
  all yet, it degrades to plain `cost`.
449
467
 
450
- `GET /health` reports the active strategy (`routing_strategy`); a
468
+ `GET /stats` reports the active strategy (`routing_strategy`); a
451
469
  decision's `reason.strategy` and `reason.latencyGuardExcludedACandidate`
452
470
  say which one ran and whether the guard actually did anything, same
453
471
  transparency style as the rest of the routing decision. Tier
package/cache.js CHANGED
@@ -1,51 +1,72 @@
1
- // model-router/cache.js
2
- //
3
- // The exact-match cache: same model + same messages + same params ->
4
- // same cached response, by content hash. First and free - checked
5
- // before the semantic cache (semanticCache.js), which is slower (an
6
- // embedding call) and probabilistic (a similarity threshold, not an
7
- // exact match). Connection is shared via redisClient.js.
8
-
9
- const crypto = require('crypto');
10
- const redis = require('./redisClient');
11
-
12
- function buildCacheKey(payload) {
13
- const normalized = JSON.stringify({
14
- model: payload.model,
15
- messages: payload.messages,
16
- temperature: payload.temperature ?? 0.0,
17
- max_tokens: payload.max_tokens,
18
- tools: payload.tools,
19
- tool_choice: payload.tool_choice
20
- });
21
- const hash = crypto.createHash('sha256').update(normalized).digest('hex');
22
- return `ROUTER:${payload.model}:${hash}`;
23
- }
24
-
25
- module.exports = {
26
- buildCacheKey,
27
-
28
- isConnected() {
29
- return redis.isConnected();
30
- },
31
-
32
- async get(payload) {
33
- if (!redis.isConnected()) return null;
34
- try {
35
- const cached = await redis.client.get(buildCacheKey(payload));
36
- return cached ? JSON.parse(cached) : null;
37
- } catch (err) {
38
- return null;
39
- }
40
- },
41
-
42
- async set(payload, response, ttlSeconds = 3600) {
43
- if (!redis.isConnected()) return false;
44
- try {
45
- await redis.client.set(buildCacheKey(payload), JSON.stringify(response), { EX: ttlSeconds });
46
- return true;
47
- } catch (err) {
48
- return false;
49
- }
50
- }
51
- };
1
+ // model-router/cache.js
2
+ //
3
+ // The exact-match cache: same model + same messages + same params ->
4
+ // same cached response, by content hash. First and free - checked
5
+ // before the semantic cache (semanticCache.js), which is slower (an
6
+ // embedding call) and probabilistic (a similarity threshold, not an
7
+ // exact match). Connection is shared via redisClient.js.
8
+
9
+ const crypto = require('crypto');
10
+ const redis = require('./redisClient');
11
+
12
+ // `scope` (seams work, roadmap: engine/cloud "wrap it, don't fork it"):
13
+ // an opaque, caller-supplied isolation key - a tenant id, a namespace,
14
+ // whatever a wrapping deployment needs two callers to never share a
15
+ // cache entry over. null/undefined (every call in this codebase today)
16
+ // means exactly what it always has: one global cache, no isolation -
17
+ // scope is omitted from both the key prefix AND the hashed payload in
18
+ // that case, so an unconfigured deployment's cache keys are BYTE-
19
+ // IDENTICAL to before this parameter existed (no cache invalidation on
20
+ // upgrade). When a caller does pass a scope, it's folded into both the
21
+ // prefix and the hash (not the prefix alone) - so two scopes are
22
+ // isolated even if the caller's own scope-naming convention were ever
23
+ // guessed or leaked; a compromised/guessed prefix alone can't be walked
24
+ // into another scope's cached content.
25
+ function buildCacheKey(scope, payload) {
26
+ const normalized = JSON.stringify({
27
+ ...(scope != null ? { scope } : {}),
28
+ model: payload.model,
29
+ messages: payload.messages,
30
+ temperature: payload.temperature ?? 0.0,
31
+ max_tokens: payload.max_tokens,
32
+ tools: payload.tools,
33
+ tool_choice: payload.tool_choice,
34
+ // response_format changes the SHAPE of the answer (json_object vs
35
+ // plain text), so it must participate in the key too - otherwise a
36
+ // cached plain-text response could be served to a json_object caller
37
+ // (or vice versa). openai.js forwards it (see its own chat()); this
38
+ // file used to omit it, making an "exact" hit not always exact.
39
+ response_format: payload.response_format
40
+ });
41
+ const hash = crypto.createHash('sha256').update(normalized).digest('hex');
42
+ const prefix = scope != null ? `ROUTER:${scope}:` : 'ROUTER:';
43
+ return `${prefix}${payload.model}:${hash}`;
44
+ }
45
+
46
+ module.exports = {
47
+ buildCacheKey,
48
+
49
+ isConnected() {
50
+ return redis.isConnected();
51
+ },
52
+
53
+ async get(scope, payload) {
54
+ if (!redis.isConnected()) return null;
55
+ try {
56
+ const cached = await redis.client.get(buildCacheKey(scope, payload));
57
+ return cached ? JSON.parse(cached) : null;
58
+ } catch (err) {
59
+ return null;
60
+ }
61
+ },
62
+
63
+ async set(scope, payload, response, ttlSeconds = 3600) {
64
+ if (!redis.isConnected()) return false;
65
+ try {
66
+ await redis.client.set(buildCacheKey(scope, payload), JSON.stringify(response), { EX: ttlSeconds });
67
+ return true;
68
+ } catch (err) {
69
+ return false;
70
+ }
71
+ }
72
+ };
package/embeddings.js CHANGED
@@ -1,32 +1,42 @@
1
- // model-router/embeddings.js
2
- //
3
- // The only embedding backend right now is OpenAI's - which means
4
- // semantic caching needs an OPENAI_API_KEY configured even for a
5
- // deployment that only ever talks to Anthropic for chat. That's a real
6
- // constraint, not hidden: isEnabled() is what semanticCache.js checks
7
- // before doing anything, and it degrades to "disabled" (not an error)
8
- // when the key isn't set, same as the Redis cache does when REDIS_URL
9
- // isn't set.
10
-
11
- const { OpenAI } = require('openai');
12
-
13
- let client;
14
- function getClient() {
15
- if (!client) client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
16
- return client;
17
- }
18
-
19
- function isEnabled() {
20
- return Boolean(process.env.OPENAI_API_KEY);
21
- }
22
-
23
- async function embed(text) {
24
- if (!isEnabled()) {
25
- throw new Error('OPENAI_API_KEY not configured - embeddings unavailable');
26
- }
27
- const model = process.env.EMBEDDING_MODEL || 'text-embedding-3-small';
28
- const response = await getClient().embeddings.create({ model, input: text });
29
- return response.data[0].embedding;
30
- }
31
-
32
- module.exports = { isEnabled, embed };
1
+ // model-router/embeddings.js
2
+ //
3
+ // The only embedding backend right now is OpenAI's - which means
4
+ // semantic caching needs an OPENAI_API_KEY configured even for a
5
+ // deployment that only ever talks to Anthropic for chat. That's a real
6
+ // constraint, not hidden: isEnabled() is what semanticCache.js checks
7
+ // before doing anything, and it degrades to "disabled" (not an error)
8
+ // when the key isn't set, same as the Redis cache does when REDIS_URL
9
+ // isn't set.
10
+
11
+ const { OpenAI } = require('openai');
12
+
13
+ let client;
14
+ function getClient() {
15
+ if (!client) client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
16
+ return client;
17
+ }
18
+
19
+ function isEnabled() {
20
+ return Boolean(process.env.OPENAI_API_KEY);
21
+ }
22
+
23
+ async function embed(text) {
24
+ if (!isEnabled()) {
25
+ throw new Error('OPENAI_API_KEY not configured - embeddings unavailable');
26
+ }
27
+ const model = process.env.EMBEDDING_MODEL || 'text-embedding-3-small';
28
+ // A hard timeout per embedding call: these run on the hot request path
29
+ // (semanticCache.js's findMatch/store, one or two per miss), so a hung
30
+ // embedding provider must not be able to stall every chat request -
31
+ // including ones that never touch embeddings at all if the semantic
32
+ // cache is enabled. AbortSignal.timeout aborts the underlying fetch, and
33
+ // the thrown error is caught by semanticCache.js's own try/catch, which
34
+ // degrades to "skip semantic caching" rather than failing the request.
35
+ const response = await getClient().embeddings.create(
36
+ { model, input: text },
37
+ { signal: AbortSignal.timeout(Number(process.env.EMBEDDING_TIMEOUT_MS) || 5000) }
38
+ );
39
+ return response.data[0].embedding;
40
+ }
41
+
42
+ module.exports = { isEnabled, embed };