lynkr 9.13.0 → 9.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/scripts/convert-routellm.py +105 -0
- package/src/api/middleware/load-shedding.js +76 -13
- package/src/api/openai-router.js +39 -7
- package/src/clients/databricks.js +90 -11
- package/src/clients/resilience.js +9 -0
- package/src/orchestrator/index.js +43 -2
- package/src/orchestrator/sse-transformer.js +27 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "lynkr",
|
|
3
|
-
"version": "9.
|
|
3
|
+
"version": "9.14.0",
|
|
4
4
|
"files": [
|
|
5
5
|
"index.js",
|
|
6
6
|
"install.sh",
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
"start": "node index.js 2>&1 | npx pino-pretty --sync",
|
|
36
36
|
"stop": "node -e \"if(process.env.HEADROOM_ENABLED==='true'&&process.env.HEADROOM_DOCKER_ENABLED!=='false'){process.exit(0)}else{process.exit(1)}\" && docker compose --profile headroom down || echo 'Headroom skipped (disabled or Docker not running)'",
|
|
37
37
|
"dev": "nodemon index.js",
|
|
38
|
+
"start:supervised": "while true; do node index.js 2>&1 | npx pino-pretty --sync; code=$?; echo \"[supervisor] lynkr exited ($code) — restarting in 3s\"; sleep 3; done",
|
|
38
39
|
"lint": "eslint src index.js",
|
|
39
40
|
"test": "npm run test:unit && npm run test:performance",
|
|
40
41
|
"test:unit": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com LOG_FILE_ENABLED=false node --test test/routing.test.js test/hybrid-routing-integration.test.js test/retry-logic.test.js test/sse-transformer.test.js test/passthrough-stream.test.js test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js test/azure-openai-config.test.js test/azure-openai-format-conversion.test.js test/azure-openai-routing.test.js test/azure-openai-streaming.test.js test/azure-openai-error-resilience.test.js test/azure-openai-integration.test.js test/openai-integration.test.js test/atlas-integration.test.js test/toon-compression.test.js test/gcf-compression.test.js test/llamacpp-integration.test.js test/resilience.test.js test/telemetry-routing.test.js test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/memory/distiller.test.js test/memory/distiller-freeze.test.js test/memory/wiki.test.js test/memory/skills-cache.test.js test/memory/tencentdb-launcher.test.js test/distill.test.js test/large-payload.test.js test/prompt-cache-injection.test.js test/risk-analyzer.test.js test/interaction-block.test.js test/preflight.test.js test/token-reduction.test.js test/session-affinity.test.js test/cache-state.test.js test/cache-switch-cost.test.js test/lens-recommendations.test.js test/model-registry-cost.test.js test/output-format-guard.test.js test/tier-fallback.test.js test/wrap.test.js test/init.test.js test/tool-call-response-metadata.test.js test/degradation.test.js test/routing-telemetry-columns.test.js test/sticky-routing.test.js test/knn-ambiguous-escalate.test.js test/deescalator.test.js test/client-profiles.test.js test/strip-internal-fields.test.js test/complexity-tool-subtraction.test.js test/bandit.test.js test/routing-propensity.test.js test/reward-pipeline.test.js test/knn-cold-start.test.js test/calibration.test.js test/feedback-loop.test.js test/session-fingerprint.test.js test/side-request-guards.test.js test/verifier.test.js test/intent-score.test.js test/difficulty-classifier.test.js test/classifier-setup.test.js test/usage-stats.test.js test/loop-guard.test.js test/moonshot-model-mapping.test.js test/baidu-model-mapping.test.js",
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Convert routellm/gpt4_dataset (train.jsonl) → Lynkr kNN bootstrap JSONL.
|
|
4
|
+
|
|
5
|
+
Stream-downloads from HuggingFace and maps mixtral_score (1-5) to your
|
|
6
|
+
TIER_* model config so the kNN index learns to recommend YOUR real models,
|
|
7
|
+
not RouteLLM's pool (mixtral / gpt-4).
|
|
8
|
+
|
|
9
|
+
Output format matches scripts/build-knn-index.js bootstrap input:
|
|
10
|
+
{"query": "...", "provider": "ollama", "model": "minimax-m2.5:cloud",
|
|
11
|
+
"quality": 95, "cost": 0.001, "latency": 200, "tier": "SIMPLE"}
|
|
12
|
+
|
|
13
|
+
Usage:
|
|
14
|
+
.venv/bin/python scripts/convert-routellm.py [--limit 30000] -o routellm.jsonl
|
|
15
|
+
"""
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
import sys
|
|
19
|
+
import urllib.request
|
|
20
|
+
import ssl
|
|
21
|
+
|
|
22
|
+
# Map mixtral_score → user's tier config.
|
|
23
|
+
# Spread is intentionally aggressive: only score=5 (Mixtral truly nailed it)
|
|
24
|
+
# stays SIMPLE; score=4 escalates to MEDIUM so the kNN index has enough MEDIUM
|
|
25
|
+
# examples to actually recommend non-SIMPLE tiers (vs the previous mapping
|
|
26
|
+
# where 86% of records were SIMPLE and kNN never disagreed with the heuristic).
|
|
27
|
+
SCORE_TO_ROUTE = {
|
|
28
|
+
5: {"tier": "SIMPLE", "provider": "ollama", "model": "minimax-m2.5:cloud", "quality": 95, "cost": 0.001, "latency": 200},
|
|
29
|
+
4: {"tier": "MEDIUM", "provider": "azure-anthropic", "model": "claude-sonnet-4-6", "quality": 80, "cost": 0.003, "latency": 1500},
|
|
30
|
+
3: {"tier": "MEDIUM", "provider": "azure-anthropic", "model": "claude-sonnet-4-6", "quality": 70, "cost": 0.003, "latency": 1500},
|
|
31
|
+
2: {"tier": "COMPLEX", "provider": "azure-anthropic", "model": "claude-sonnet-4-6", "quality": 55, "cost": 0.003, "latency": 1500},
|
|
32
|
+
1: {"tier": "REASONING", "provider": "azure-anthropic", "model": "claude-opus-4-8", "quality": 40, "cost": 0.015, "latency": 3500},
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
URL = "https://huggingface.co/datasets/routellm/gpt4_dataset/resolve/main/train.jsonl"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def main():
|
|
39
|
+
p = argparse.ArgumentParser()
|
|
40
|
+
p.add_argument("--output", "-o", default="data/routellm-bootstrap.jsonl")
|
|
41
|
+
p.add_argument("--limit", type=int, default=None,
|
|
42
|
+
help="Only convert first N rows")
|
|
43
|
+
args = p.parse_args()
|
|
44
|
+
|
|
45
|
+
print(f"Streaming {URL} …", file=sys.stderr)
|
|
46
|
+
# Some macOS Python builds have SSL issues with huggingface.co — fall back
|
|
47
|
+
# to unverified context if needed. The data is public, no auth.
|
|
48
|
+
ctx = ssl.create_default_context()
|
|
49
|
+
try:
|
|
50
|
+
with urllib.request.urlopen(URL, context=ctx, timeout=60) as resp:
|
|
51
|
+
_stream(resp, args.output, args.limit)
|
|
52
|
+
except (ssl.SSLError, OSError) as e:
|
|
53
|
+
print(f"SSL strict failed ({e}); retrying with unverified context", file=sys.stderr)
|
|
54
|
+
ctx = ssl._create_unverified_context()
|
|
55
|
+
with urllib.request.urlopen(URL, context=ctx, timeout=60) as resp:
|
|
56
|
+
_stream(resp, args.output, args.limit)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _stream(resp, out_path, limit):
|
|
60
|
+
written = 0
|
|
61
|
+
by_score = {s: 0 for s in SCORE_TO_ROUTE}
|
|
62
|
+
skipped = 0
|
|
63
|
+
with open(out_path, "w") as out:
|
|
64
|
+
for line in resp:
|
|
65
|
+
if limit and written >= limit:
|
|
66
|
+
break
|
|
67
|
+
try:
|
|
68
|
+
row = json.loads(line)
|
|
69
|
+
except Exception:
|
|
70
|
+
skipped += 1
|
|
71
|
+
continue
|
|
72
|
+
score = row.get("mixtral_score")
|
|
73
|
+
if score not in SCORE_TO_ROUTE:
|
|
74
|
+
skipped += 1
|
|
75
|
+
continue
|
|
76
|
+
route = SCORE_TO_ROUTE[score]
|
|
77
|
+
rec = {
|
|
78
|
+
"query": row.get("prompt", ""),
|
|
79
|
+
"provider": route["provider"],
|
|
80
|
+
"model": route["model"],
|
|
81
|
+
"quality": route["quality"],
|
|
82
|
+
"cost": route["cost"],
|
|
83
|
+
"latency": route["latency"],
|
|
84
|
+
"tier": route["tier"],
|
|
85
|
+
}
|
|
86
|
+
if not rec["query"]:
|
|
87
|
+
skipped += 1
|
|
88
|
+
continue
|
|
89
|
+
out.write(json.dumps(rec) + "\n")
|
|
90
|
+
written += 1
|
|
91
|
+
by_score[score] += 1
|
|
92
|
+
if written % 5000 == 0:
|
|
93
|
+
print(f" {written} written…", file=sys.stderr)
|
|
94
|
+
|
|
95
|
+
print(f"\nWrote {written} records to {out_path} ({skipped} skipped)", file=sys.stderr)
|
|
96
|
+
print(f"By mixtral_score: {by_score}", file=sys.stderr)
|
|
97
|
+
print(f"\nTier distribution:", file=sys.stderr)
|
|
98
|
+
for score, route in SCORE_TO_ROUTE.items():
|
|
99
|
+
n = by_score[score]
|
|
100
|
+
if n:
|
|
101
|
+
print(f" {route['tier']:>10} {route['provider']}:{route['model']:30s} n={n}", file=sys.stderr)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
if __name__ == "__main__":
|
|
105
|
+
main()
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
const os = require("os");
|
|
2
|
+
const v8 = require("v8");
|
|
2
3
|
const logger = require("../../logger");
|
|
3
4
|
const { ServiceUnavailableError } = require("./error-handling");
|
|
4
5
|
|
|
@@ -18,11 +19,31 @@ class LoadShedder {
|
|
|
18
19
|
this.memoryThreshold = options.memoryThreshold || 0.85; // 85%
|
|
19
20
|
this.heapThreshold = options.heapThreshold || 0.95; // 95% (increased from 90% to prevent false positives from temporary allocation spikes)
|
|
20
21
|
this.activeRequestsThreshold = options.activeRequestsThreshold || 1000;
|
|
22
|
+
// Absolute free-heap floor. Percent-of-heapTotal is unreliable in both
|
|
23
|
+
// directions: heapTotal is V8's CURRENTLY allocated heap (grows on
|
|
24
|
+
// demand toward the real limit), and at the limit "99%" leaves ~40MB on
|
|
25
|
+
// a 4GB heap — nowhere near enough to parse one multi-MB request body.
|
|
26
|
+
// 2026-08-19 OOM: a 5MB compact request passed the percent check and
|
|
27
|
+
// died allocating during JSON.parse. Shed while there is still room to
|
|
28
|
+
// finish in-flight work.
|
|
29
|
+
this.headroomMB = options.headroomMB || 512;
|
|
30
|
+
this.heapLimitMB = v8.getHeapStatistics().heap_size_limit / (1024 * 1024);
|
|
31
|
+
|
|
32
|
+
// Watchdog: how long the heap may stay below the headroom floor with
|
|
33
|
+
// zero active requests before the process self-restarts. Retained
|
|
34
|
+
// memory GC can't reclaim never recovers on its own — without this the
|
|
35
|
+
// server sheds 100% of traffic until someone restarts it by hand.
|
|
36
|
+
this.wedgedRestartMs = options.wedgedRestartMs || 60000;
|
|
37
|
+
this.selfRestart = options.selfRestart !== false;
|
|
38
|
+
this.wedgedSince = null;
|
|
21
39
|
|
|
22
40
|
// State
|
|
23
41
|
this.activeRequests = 0;
|
|
24
42
|
this.totalShed = 0;
|
|
25
|
-
|
|
43
|
+
// 0, not Date.now(): the first isOverloaded() call must actually
|
|
44
|
+
// evaluate instead of returning the constructor's cached "false"
|
|
45
|
+
// for the first checkInterval window.
|
|
46
|
+
this.lastCheck = 0;
|
|
26
47
|
this.checkInterval = options.checkInterval || 1000; // Check every second
|
|
27
48
|
this.cachedOverloadState = false;
|
|
28
49
|
}
|
|
@@ -42,26 +63,32 @@ class LoadShedder {
|
|
|
42
63
|
|
|
43
64
|
// Check memory usage
|
|
44
65
|
const memUsage = process.memoryUsage();
|
|
45
|
-
const heapUsedPercent = memUsage.heapUsed / memUsage.heapTotal;
|
|
46
|
-
|
|
47
|
-
// FIX: Only trigger if BOTH percentage is high AND actual usage is significant
|
|
48
|
-
// This prevents false positives on startup when heapTotal is small but will grow
|
|
49
|
-
const heapUsedMB = memUsage.heapUsed / (1024 * 1024);
|
|
50
|
-
const minHeapThresholdMB = 500; // Only shed load if using more than 500MB
|
|
51
66
|
|
|
52
|
-
|
|
67
|
+
// Absolute headroom against the true V8 limit — the primary guard.
|
|
68
|
+
const heapStats = v8.getHeapStatistics();
|
|
69
|
+
const freeMB = (heapStats.heap_size_limit - heapStats.used_heap_size) / (1024 * 1024);
|
|
70
|
+
if (freeMB < this.headroomMB) {
|
|
53
71
|
logger.warn(
|
|
54
72
|
{
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
73
|
+
freeMB: freeMB.toFixed(0),
|
|
74
|
+
headroomMB: this.headroomMB,
|
|
75
|
+
heapLimitMB: this.heapLimitMB.toFixed(0),
|
|
76
|
+
usedMB: (heapStats.used_heap_size / (1024 * 1024)).toFixed(0),
|
|
59
77
|
},
|
|
60
|
-
"Load shedding:
|
|
78
|
+
"Load shedding: heap headroom below floor"
|
|
61
79
|
);
|
|
80
|
+
this._trackWedged(now);
|
|
62
81
|
this.cachedOverloadState = true;
|
|
63
82
|
return true;
|
|
64
83
|
}
|
|
84
|
+
this.wedgedSince = null;
|
|
85
|
+
|
|
86
|
+
// NOTE: the old heapUsed/heapTotal percent check is gone. heapTotal is
|
|
87
|
+
// V8's *currently allocated* heap, which it keeps close to usage, so the
|
|
88
|
+
// ratio reads ~99% during perfectly normal operation — it shed all
|
|
89
|
+
// traffic at 755MB used with gigabytes of real headroom remaining
|
|
90
|
+
// (2026-08-20 false-positive incident). The absolute-headroom check
|
|
91
|
+
// against heap_size_limit above is the correct form of this guard.
|
|
65
92
|
|
|
66
93
|
// Check RSS / system memory
|
|
67
94
|
const rssPercent = memUsage.rss / os.totalmem();
|
|
@@ -94,6 +121,32 @@ class LoadShedder {
|
|
|
94
121
|
return false;
|
|
95
122
|
}
|
|
96
123
|
|
|
124
|
+
/**
|
|
125
|
+
* Track how long the heap has been wedged below the headroom floor.
|
|
126
|
+
* If it persists past wedgedRestartMs with no in-flight requests, the
|
|
127
|
+
* memory is retained (not transient) and GC cannot recover it — exit so
|
|
128
|
+
* the supervisor restarts a clean process instead of shedding forever.
|
|
129
|
+
*/
|
|
130
|
+
_trackWedged(now) {
|
|
131
|
+
if (!this.selfRestart) return;
|
|
132
|
+
if (this.wedgedSince === null) {
|
|
133
|
+
this.wedgedSince = now;
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
if (now - this.wedgedSince >= this.wedgedRestartMs && this.activeRequests === 0) {
|
|
137
|
+
logger.fatal(
|
|
138
|
+
{
|
|
139
|
+
wedgedForMs: now - this.wedgedSince,
|
|
140
|
+
totalShed: this.totalShed,
|
|
141
|
+
heapLimitMB: this.heapLimitMB.toFixed(0),
|
|
142
|
+
},
|
|
143
|
+
"Load shedding: heap wedged below headroom floor with no active requests — exiting for supervisor restart"
|
|
144
|
+
);
|
|
145
|
+
// Give the fatal log a moment to flush, then exit non-zero.
|
|
146
|
+
setTimeout(() => process.exit(1), 250).unref();
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
97
150
|
/**
|
|
98
151
|
* Get current metrics
|
|
99
152
|
*/
|
|
@@ -107,10 +160,15 @@ class LoadShedder {
|
|
|
107
160
|
heapTotalMB: (memUsage.heapTotal / (1024 * 1024)).toFixed(2),
|
|
108
161
|
rssMB: (memUsage.rss / (1024 * 1024)).toFixed(2),
|
|
109
162
|
rssPercent: ((memUsage.rss / os.totalmem()) * 100).toFixed(2),
|
|
163
|
+
heapLimitMB: this.heapLimitMB.toFixed(2),
|
|
164
|
+
freeHeadroomMB: ((v8.getHeapStatistics().heap_size_limit - v8.getHeapStatistics().used_heap_size) / (1024 * 1024)).toFixed(2),
|
|
110
165
|
thresholds: {
|
|
111
166
|
heapThreshold: (this.heapThreshold * 100).toFixed(2),
|
|
112
167
|
memoryThreshold: (this.memoryThreshold * 100).toFixed(2),
|
|
113
168
|
activeRequestsThreshold: this.activeRequestsThreshold,
|
|
169
|
+
headroomMB: this.headroomMB,
|
|
170
|
+
wedgedRestartMs: this.wedgedRestartMs,
|
|
171
|
+
selfRestart: this.selfRestart,
|
|
114
172
|
},
|
|
115
173
|
};
|
|
116
174
|
}
|
|
@@ -129,6 +187,11 @@ function getLoadShedder(options) {
|
|
|
129
187
|
process.env.LOAD_SHEDDING_ACTIVE_REQUESTS_THRESHOLD || "1000",
|
|
130
188
|
10
|
|
131
189
|
),
|
|
190
|
+
// Not env-configurable by design: these are crash-prevention floors,
|
|
191
|
+
// and there is no deployment where turning them off is correct.
|
|
192
|
+
headroomMB: 512,
|
|
193
|
+
wedgedRestartMs: 60000,
|
|
194
|
+
selfRestart: true,
|
|
132
195
|
};
|
|
133
196
|
instance = new LoadShedder({ ...defaultOptions, ...options });
|
|
134
197
|
}
|
package/src/api/openai-router.js
CHANGED
|
@@ -426,22 +426,42 @@ async function forwardAnthropicStreamAsOpenAIChunks(res, stream, requestedModel)
|
|
|
426
426
|
res.write("data: [DONE]\n\n");
|
|
427
427
|
res.end();
|
|
428
428
|
return;
|
|
429
|
+
case "error":
|
|
430
|
+
// Forward upstream stream errors in OpenAI wire shape
|
|
431
|
+
// (data: {"error": ...}) and terminate WITHOUT finish_reason or
|
|
432
|
+
// [DONE]. Writing "stop" here told agent clients the model chose
|
|
433
|
+
// to end its turn — they stopped mid-task instead of retrying
|
|
434
|
+
// (2026-08-19 incident: opencode stalled between tool calls).
|
|
435
|
+
logger.warn({ error: ev.error?.message }, "Forwarding upstream stream error to OpenAI-surface client");
|
|
436
|
+
res.write(`data: ${JSON.stringify({ error: { message: ev.error?.message || "upstream stream error", type: ev.error?.type || "server_error", code: null } })}\n\n`);
|
|
437
|
+
res.end();
|
|
438
|
+
return;
|
|
429
439
|
default:
|
|
430
440
|
break; // ping, thinking deltas, unknown events — skip
|
|
431
441
|
}
|
|
432
442
|
if (res.writableEnded || res.destroyed) return;
|
|
433
443
|
}
|
|
434
444
|
}
|
|
435
|
-
// Upstream closed without message_stop
|
|
445
|
+
// Upstream closed without message_stop. If a stop_reason already arrived
|
|
446
|
+
// the turn is materially complete — finish normally. Otherwise this is a
|
|
447
|
+
// truncated stream: send an error chunk, never a fabricated "stop".
|
|
436
448
|
if (!res.writableEnded) {
|
|
437
|
-
|
|
438
|
-
|
|
449
|
+
if (finishReason) {
|
|
450
|
+
res.write(chunk({}, finishReason));
|
|
451
|
+
res.write("data: [DONE]\n\n");
|
|
452
|
+
} else {
|
|
453
|
+
logger.warn("Upstream stream ended without message_stop — sending error chunk to OpenAI-surface client");
|
|
454
|
+
res.write(`data: ${JSON.stringify({ error: { message: "Lynkr: upstream stream truncated — retry", type: "server_error", code: null } })}\n\n`);
|
|
455
|
+
}
|
|
439
456
|
res.end();
|
|
440
457
|
}
|
|
441
458
|
} catch (err) {
|
|
442
459
|
logger.warn({ err: err.message }, "OpenAI-surface stream forward failed mid-flight");
|
|
443
460
|
if (!res.writableEnded) {
|
|
444
|
-
try {
|
|
461
|
+
try {
|
|
462
|
+
res.write(`data: ${JSON.stringify({ error: { message: `Lynkr: stream forward failed — retry (${err.message})`, type: "server_error", code: null } })}\n\n`);
|
|
463
|
+
res.end();
|
|
464
|
+
} catch { /* client gone */ }
|
|
445
465
|
}
|
|
446
466
|
}
|
|
447
467
|
}
|
|
@@ -594,13 +614,25 @@ router.post("/chat/completions", async (req, res) => {
|
|
|
594
614
|
// "task complete" and terminate mid-task. Killing the stream without
|
|
595
615
|
// a finish chunk makes the client retry instead.
|
|
596
616
|
if (!content && (!toolCalls || toolCalls.length === 0)) {
|
|
617
|
+
const upstreamErr = result.body?.error || {};
|
|
597
618
|
logger.error({
|
|
598
619
|
finishReason: openaiResponse.choices[0]?.finish_reason,
|
|
599
620
|
terminationReason: result?.terminationReason,
|
|
600
621
|
status: result?.status,
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
res.destroy()
|
|
622
|
+
upstreamError: upstreamErr.message || null,
|
|
623
|
+
}, "Empty completion reached serving path — sending error chunk to client");
|
|
624
|
+
// res.destroy() here surfaced as ECONNRESET ("connection reset by
|
|
625
|
+
// server") in clients, and the error chunk raced the teardown so
|
|
626
|
+
// it often never arrived. A cleanly-ended SSE error chunk (no
|
|
627
|
+
// finish chunk, no [DONE]) delivers the real upstream error and
|
|
628
|
+
// still reads as a failed request that agent clients retry.
|
|
629
|
+
res.write(`data: ${JSON.stringify({ error: {
|
|
630
|
+
message: upstreamErr.message || "Upstream returned an empty completion; retry.",
|
|
631
|
+
type: upstreamErr.type || "server_error",
|
|
632
|
+
code: upstreamErr.code || "empty_completion",
|
|
633
|
+
...(result?.status ? { status: result.status } : {}),
|
|
634
|
+
} })}\n\n`);
|
|
635
|
+
res.end();
|
|
604
636
|
return;
|
|
605
637
|
}
|
|
606
638
|
|
|
@@ -3541,12 +3541,101 @@ async function invokeModel(body, options = {}) {
|
|
|
3541
3541
|
|
|
3542
3542
|
const fallbackStart = Date.now();
|
|
3543
3543
|
|
|
3544
|
+
// Strip the failed primary's routing stamps before re-dispatch: every
|
|
3545
|
+
// provider client prefers body._tierModel over its own configured
|
|
3546
|
+
// model, so leaving the failed tier's model on the body makes the
|
|
3547
|
+
// fallback provider serve a model it doesn't have — azure-openai
|
|
3548
|
+
// treats it as the deployment id and 404s with DeploymentNotFound
|
|
3549
|
+
// (live incident 2026-08-19). The fallback provider must fall through
|
|
3550
|
+
// to its own configured model.
|
|
3551
|
+
const fallbackBody = { ...body };
|
|
3552
|
+
delete fallbackBody._tierModel;
|
|
3553
|
+
delete fallbackBody._suggestionModeModel;
|
|
3554
|
+
|
|
3544
3555
|
const fallbackResult = await fallbackBreaker.execute(async () => {
|
|
3545
|
-
return await invokeProvider(fallbackProvider,
|
|
3556
|
+
return await invokeProvider(fallbackProvider, fallbackBody, incomingHeaders);
|
|
3546
3557
|
});
|
|
3547
3558
|
|
|
3548
3559
|
const fallbackLatency = Date.now() - fallbackStart;
|
|
3549
3560
|
|
|
3561
|
+
// Provider clients return {ok:false} on 4xx/5xx instead of throwing,
|
|
3562
|
+
// so a fallback rung that answered with an error body used to sail
|
|
3563
|
+
// through the success path below — logging "Fallback to cloud
|
|
3564
|
+
// provider succeeded" and recording status_code:200 telemetry for
|
|
3565
|
+
// what was actually a double failure. Treat it as one.
|
|
3566
|
+
if (fallbackResult?.ok === false) {
|
|
3567
|
+
const fbErrMessage = fallbackResult.json?.error?.message || String(fallbackResult.text || "").slice(0, 120) || "error response";
|
|
3568
|
+
|
|
3569
|
+
metricsCollector.recordFallbackFailure();
|
|
3570
|
+
metricsCollector.recordDatabricksRequest(false, retries);
|
|
3571
|
+
healthTracker.recordFailure(fallbackProvider, new Error(fbErrMessage), fallbackResult.status);
|
|
3572
|
+
|
|
3573
|
+
logger.warn({
|
|
3574
|
+
originalProvider: initialProvider,
|
|
3575
|
+
fallbackProvider,
|
|
3576
|
+
status: fallbackResult.status,
|
|
3577
|
+
error: fbErrMessage,
|
|
3578
|
+
}, "Fallback provider answered with an error response");
|
|
3579
|
+
|
|
3580
|
+
// Thread the whole trail into the error payload so the client sees
|
|
3581
|
+
// every failure, not just this one (e.g. "moonshot: quota" matters
|
|
3582
|
+
// more than "azure: 404").
|
|
3583
|
+
failureTrail.push(`${fallbackProvider}: HTTP ${fallbackResult.status} — ${fbErrMessage}`);
|
|
3584
|
+
if (fallbackResult.json && typeof fallbackResult.json === "object") {
|
|
3585
|
+
fallbackResult.json._lynkr_failure_trail = failureTrail;
|
|
3586
|
+
}
|
|
3587
|
+
|
|
3588
|
+
const fbFailLatency = Date.now() - startTime;
|
|
3589
|
+
telemetry.record({
|
|
3590
|
+
request_id: crypto.randomUUID(),
|
|
3591
|
+
session_id: body._sessionId || null,
|
|
3592
|
+
timestamp: Date.now(),
|
|
3593
|
+
complexity_score: routingResult.score ?? null,
|
|
3594
|
+
tier: routingDecision.tier,
|
|
3595
|
+
provider: fallbackProvider,
|
|
3596
|
+
model: routingDecision.model ?? body._tierModel ?? null,
|
|
3597
|
+
routing_method: "fallback",
|
|
3598
|
+
was_fallback: true,
|
|
3599
|
+
latency_ms: fbFailLatency,
|
|
3600
|
+
status_code: fallbackResult.status || null,
|
|
3601
|
+
error_type: "fallback_error_response",
|
|
3602
|
+
quality_score: 0,
|
|
3603
|
+
base_tier: routingResult.base_tier ?? null,
|
|
3604
|
+
escalation_source: routingResult.escalation_source ?? null,
|
|
3605
|
+
propensity: routingResult.propensity ?? null,
|
|
3606
|
+
candidates: routingResult.candidates ?? null,
|
|
3607
|
+
pinned: routingResult.pinned ? 1 : 0,
|
|
3608
|
+
switch_reason: routingResult.switch_reason ?? null,
|
|
3609
|
+
cache_decision: routingResult._cacheDecision ?? null,
|
|
3610
|
+
});
|
|
3611
|
+
|
|
3612
|
+
// WS5.4 — feedback loop (fallback answered with an error). Same
|
|
3613
|
+
// hard-negative signal as a double failure.
|
|
3614
|
+
recordFeedbackOutcome({
|
|
3615
|
+
routingResult,
|
|
3616
|
+
body,
|
|
3617
|
+
outcome: {
|
|
3618
|
+
qualityScore: 0,
|
|
3619
|
+
costUsd: null,
|
|
3620
|
+
latencyMs: fbFailLatency,
|
|
3621
|
+
statusCode: fallbackResult.status || null,
|
|
3622
|
+
errorType: "fallback_error_response",
|
|
3623
|
+
wasFallback: true,
|
|
3624
|
+
},
|
|
3625
|
+
});
|
|
3626
|
+
|
|
3627
|
+
return {
|
|
3628
|
+
...fallbackResult,
|
|
3629
|
+
actualProvider: fallbackProvider,
|
|
3630
|
+
routingDecision: {
|
|
3631
|
+
...routingDecision,
|
|
3632
|
+
provider: fallbackProvider,
|
|
3633
|
+
method: 'fallback',
|
|
3634
|
+
fallbackReason: reason,
|
|
3635
|
+
},
|
|
3636
|
+
};
|
|
3637
|
+
}
|
|
3638
|
+
|
|
3550
3639
|
metricsCollector.recordFallbackSuccess(fallbackLatency);
|
|
3551
3640
|
metricsCollector.recordDatabricksRequest(true, retries);
|
|
3552
3641
|
healthTracker.recordSuccess(fallbackProvider, fallbackLatency);
|
|
@@ -3635,16 +3724,6 @@ async function invokeModel(body, options = {}) {
|
|
|
3635
3724
|
},
|
|
3636
3725
|
});
|
|
3637
3726
|
|
|
3638
|
-
// The last rung also answered with an error — thread the whole trail
|
|
3639
|
-
// into the error payload so the client sees every failure, not just
|
|
3640
|
-
// this one (e.g. "moonshot: quota" matters more than "azure: 404").
|
|
3641
|
-
if (fallbackResult?.ok === false) {
|
|
3642
|
-
failureTrail.push(`${fallbackProvider}: HTTP ${fallbackResult.status} — ${fallbackResult.json?.error?.message || String(fallbackResult.text || "").slice(0, 120) || "error response"}`);
|
|
3643
|
-
if (fallbackResult.json && typeof fallbackResult.json === "object") {
|
|
3644
|
-
fallbackResult.json._lynkr_failure_trail = failureTrail;
|
|
3645
|
-
}
|
|
3646
|
-
}
|
|
3647
|
-
|
|
3648
3727
|
return {
|
|
3649
3728
|
...fallbackResult,
|
|
3650
3729
|
actualProvider: fallbackProvider,
|
|
@@ -381,6 +381,15 @@ async function withCockatielRetry(fn, options = {}) {
|
|
|
381
381
|
const retryAfterDate = new Date(retryAfter);
|
|
382
382
|
delay = retryAfterDate.getTime() - Date.now();
|
|
383
383
|
}
|
|
384
|
+
// Retry-After is a floor, not the full answer: Azure sends
|
|
385
|
+
// "Retry-After: 1" on token-window 429s that need far longer to
|
|
386
|
+
// drain, so honoring it literally re-hammers the window every
|
|
387
|
+
// second and it never recovers (2026-08-19 incident). Back off
|
|
388
|
+
// exponentially on top of the header, with jitter.
|
|
389
|
+
const floorMs = Math.max(0, delay || 0);
|
|
390
|
+
const backoffMs = 2000 * Math.pow(config.backoffMultiplier, attempt);
|
|
391
|
+
const cappedMs = Math.min(Math.max(floorMs, backoffMs), 60000);
|
|
392
|
+
delay = cappedMs + cappedMs * config.jitterFactor * Math.random();
|
|
384
393
|
} else {
|
|
385
394
|
// Exponential backoff with longer delays for rate limiting
|
|
386
395
|
const baseDelay = 2000 * Math.pow(config.backoffMultiplier, attempt);
|
|
@@ -1356,6 +1356,18 @@ function buildNonJsonResponse(databricksResponse) {
|
|
|
1356
1356
|
};
|
|
1357
1357
|
}
|
|
1358
1358
|
|
|
1359
|
+
/**
|
|
1360
|
+
* Visible routing badge for live-streamed responses, built from the actual
|
|
1361
|
+
* post-routing decision (tier fallbacks included). Matches the format of
|
|
1362
|
+
* openai-router's lynkrBadge and the Anthropic router's intent badge so
|
|
1363
|
+
* LYNKR_BADGE_PREFIX_RE strips all three identically from replayed history.
|
|
1364
|
+
*/
|
|
1365
|
+
function buildRoutingBadge(rd) {
|
|
1366
|
+
if (!rd || !rd.tier) return null;
|
|
1367
|
+
const score = typeof rd.score === "number" ? ` · score ${rd.score}` : "";
|
|
1368
|
+
return `*[Lynkr] ${rd.tier} → ${rd.model || "—"} (${rd.provider || "—"})${score}*\n\n`;
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1359
1371
|
function buildStreamingResponse(databricksResponse) {
|
|
1360
1372
|
return {
|
|
1361
1373
|
status: databricksResponse.status,
|
|
@@ -1755,7 +1767,24 @@ IMPORTANT TOOL USAGE RULES:
|
|
|
1755
1767
|
};
|
|
1756
1768
|
const headroomProvider = process.env.HEADROOM_PROVIDER || 'anthropic';
|
|
1757
1769
|
const headroomSafeProviders = headroomProviderMap[headroomProvider] || new Set();
|
|
1758
|
-
|
|
1770
|
+
// Under tier routing the static providerType is just the MODEL_PROVIDER
|
|
1771
|
+
// default (usually "databricks") — the real provider is chosen per-request
|
|
1772
|
+
// later in invokeModel, AFTER this compression point, so gating on it
|
|
1773
|
+
// silently disabled Headroom for every tier-routed setup. Gate on the
|
|
1774
|
+
// configured tier fleet instead: compress only when every routable tier
|
|
1775
|
+
// provider is format-safe, since any request may land on any tier.
|
|
1776
|
+
let headroomCompatible = headroomSafeProviders.has(providerType);
|
|
1777
|
+
if (!headroomCompatible) {
|
|
1778
|
+
try {
|
|
1779
|
+
const { getModelTierSelector } = require('../routing/model-tiers');
|
|
1780
|
+
const sel = getModelTierSelector();
|
|
1781
|
+
const tierProviders = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING']
|
|
1782
|
+
.map((t) => { try { return sel.selectModel(t)?.provider; } catch { return null; } })
|
|
1783
|
+
.filter(Boolean)
|
|
1784
|
+
.map((p) => (p === 'z-ai' ? 'zai' : p));
|
|
1785
|
+
headroomCompatible = tierProviders.length > 0 && tierProviders.every((p) => headroomSafeProviders.has(p));
|
|
1786
|
+
} catch { /* tier routing not configured — keep the static gate result */ }
|
|
1787
|
+
}
|
|
1759
1788
|
|
|
1760
1789
|
if (isHeadroomEnabled() && headroomCompatible && cleanPayload.messages && cleanPayload.messages.length > 0) {
|
|
1761
1790
|
try {
|
|
@@ -2053,7 +2082,16 @@ IMPORTANT TOOL USAGE RULES:
|
|
|
2053
2082
|
const routingDecision = databricksResponse.routingDecision || {};
|
|
2054
2083
|
const transformed = sseTransform.openaiToAnthropicSSE(databricksResponse.stream, {
|
|
2055
2084
|
model: requestedModel,
|
|
2056
|
-
|
|
2085
|
+
// Badge precedence: an explicit streamBadgeText (Anthropic router's
|
|
2086
|
+
// pre-computed intent badge) wins; otherwise build one from the
|
|
2087
|
+
// ACTUAL post-routing decision so OpenAI-surface live streams
|
|
2088
|
+
// (opencode et al.) show which model served the response —
|
|
2089
|
+
// previously this path silently dropped the badge entirely.
|
|
2090
|
+
// History pollution is covered: stripLynkrBadges in invokeModel
|
|
2091
|
+
// removes resubmitted badges before anything reaches a provider.
|
|
2092
|
+
badgeText: config.routing?.visibleInteraction
|
|
2093
|
+
? (options?.streamBadgeText || buildRoutingBadge(routingDecision))
|
|
2094
|
+
: null,
|
|
2057
2095
|
onClose: (stats) => {
|
|
2058
2096
|
try {
|
|
2059
2097
|
const telemetry = require("../routing/telemetry");
|
|
@@ -2080,6 +2118,7 @@ IMPORTANT TOOL USAGE RULES:
|
|
|
2080
2118
|
logger.info({
|
|
2081
2119
|
provider: routingDecision.provider || _streamProvider,
|
|
2082
2120
|
stopReason: stats.stopReason,
|
|
2121
|
+
finishReason: stats.finishReason ?? null,
|
|
2083
2122
|
toolCalls: stats.toolCalls.map((t) => t.name),
|
|
2084
2123
|
outputTokens: stats.usage.output_tokens,
|
|
2085
2124
|
}, "[SSETransform] Stream closed");
|
|
@@ -3191,4 +3230,6 @@ module.exports = {
|
|
|
3191
3230
|
toAnthropicResponse,
|
|
3192
3231
|
// Exported for unit testing of loop trimming (task-preservation contract).
|
|
3193
3232
|
trimLoopMessages,
|
|
3233
|
+
// Exported for unit testing of the live-stream routing badge.
|
|
3234
|
+
buildRoutingBadge,
|
|
3194
3235
|
};
|
|
@@ -256,6 +256,7 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
|
|
|
256
256
|
error: { type: "overloaded_error", message: `Lynkr: upstream stream failed — retry (${err.message})` },
|
|
257
257
|
});
|
|
258
258
|
stats.stopReason = "stream_error";
|
|
259
|
+
stats.finishReason = finishReason ?? null;
|
|
259
260
|
if (opts.onClose) { try { opts.onClose(stats); } catch { /* non-fatal */ } }
|
|
260
261
|
return;
|
|
261
262
|
}
|
|
@@ -263,10 +264,36 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
|
|
|
263
264
|
// Normal end of stream ([DONE] or upstream EOF).
|
|
264
265
|
yield* startMessage();
|
|
265
266
|
yield* closeTextBlock();
|
|
267
|
+
|
|
268
|
+
// An EOF with no finish_reason is a dropped upstream, not a completed
|
|
269
|
+
// turn. Mapping it to end_turn hands the client a success-shaped stop —
|
|
270
|
+
// agent loops halt mid-task believing the model chose to stop (live
|
|
271
|
+
// incident 2026-08-19: opencode stalled between tool calls; the stream
|
|
272
|
+
// closed as end_turn with null output tokens). Spec-compliant upstreams
|
|
273
|
+
// always send finish_reason on the last choice chunk, so signal an error
|
|
274
|
+
// and let the client retry, mirroring the mid-flight failure path above.
|
|
275
|
+
// Accumulated tool fragments are NOT flushed: without finish_reason their
|
|
276
|
+
// argument JSON may be truncated mid-string.
|
|
277
|
+
if (!finishReason) {
|
|
278
|
+
logger.warn(
|
|
279
|
+
{ droppedToolFragments: toolAcc.size, outputTokens: stats.usage.output_tokens },
|
|
280
|
+
"[SSETransform] Upstream ended without finish_reason — signaling stream error"
|
|
281
|
+
);
|
|
282
|
+
yield _sse("error", {
|
|
283
|
+
type: "error",
|
|
284
|
+
error: { type: "overloaded_error", message: "Lynkr: upstream ended without finish_reason — retry" },
|
|
285
|
+
});
|
|
286
|
+
stats.stopReason = "stream_error";
|
|
287
|
+
stats.finishReason = null;
|
|
288
|
+
if (opts.onClose) { try { opts.onClose(stats); } catch { /* non-fatal */ } }
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
|
|
266
292
|
yield* flushToolBlocks();
|
|
267
293
|
|
|
268
294
|
const stopReason = _mapFinishReason(finishReason);
|
|
269
295
|
stats.stopReason = stopReason;
|
|
296
|
+
stats.finishReason = finishReason;
|
|
270
297
|
yield _sse("message_delta", {
|
|
271
298
|
type: "message_delta",
|
|
272
299
|
delta: { stop_reason: stopReason, stop_sequence: null },
|