lynkr 9.13.0 → 9.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "lynkr",
3
- "version": "9.13.0",
3
+ "version": "9.14.0",
4
4
  "files": [
5
5
  "index.js",
6
6
  "install.sh",
@@ -35,6 +35,7 @@
35
35
  "start": "node index.js 2>&1 | npx pino-pretty --sync",
36
36
  "stop": "node -e \"if(process.env.HEADROOM_ENABLED==='true'&&process.env.HEADROOM_DOCKER_ENABLED!=='false'){process.exit(0)}else{process.exit(1)}\" && docker compose --profile headroom down || echo 'Headroom skipped (disabled or Docker not running)'",
37
37
  "dev": "nodemon index.js",
38
+ "start:supervised": "while true; do node index.js 2>&1 | npx pino-pretty --sync; code=$?; echo \"[supervisor] lynkr exited ($code) — restarting in 3s\"; sleep 3; done",
38
39
  "lint": "eslint src index.js",
39
40
  "test": "npm run test:unit && npm run test:performance",
40
41
  "test:unit": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com LOG_FILE_ENABLED=false node --test test/routing.test.js test/hybrid-routing-integration.test.js test/retry-logic.test.js test/sse-transformer.test.js test/passthrough-stream.test.js test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js test/azure-openai-config.test.js test/azure-openai-format-conversion.test.js test/azure-openai-routing.test.js test/azure-openai-streaming.test.js test/azure-openai-error-resilience.test.js test/azure-openai-integration.test.js test/openai-integration.test.js test/atlas-integration.test.js test/toon-compression.test.js test/gcf-compression.test.js test/llamacpp-integration.test.js test/resilience.test.js test/telemetry-routing.test.js test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/memory/distiller.test.js test/memory/distiller-freeze.test.js test/memory/wiki.test.js test/memory/skills-cache.test.js test/memory/tencentdb-launcher.test.js test/distill.test.js test/large-payload.test.js test/prompt-cache-injection.test.js test/risk-analyzer.test.js test/interaction-block.test.js test/preflight.test.js test/token-reduction.test.js test/session-affinity.test.js test/cache-state.test.js test/cache-switch-cost.test.js test/lens-recommendations.test.js test/model-registry-cost.test.js test/output-format-guard.test.js test/tier-fallback.test.js test/wrap.test.js test/init.test.js test/tool-call-response-metadata.test.js test/degradation.test.js test/routing-telemetry-columns.test.js test/sticky-routing.test.js test/knn-ambiguous-escalate.test.js test/deescalator.test.js test/client-profiles.test.js test/strip-internal-fields.test.js test/complexity-tool-subtraction.test.js test/bandit.test.js test/routing-propensity.test.js test/reward-pipeline.test.js test/knn-cold-start.test.js test/calibration.test.js test/feedback-loop.test.js test/session-fingerprint.test.js test/side-request-guards.test.js test/verifier.test.js test/intent-score.test.js test/difficulty-classifier.test.js test/classifier-setup.test.js test/usage-stats.test.js test/loop-guard.test.js test/moonshot-model-mapping.test.js test/baidu-model-mapping.test.js",
@@ -0,0 +1,105 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ Convert routellm/gpt4_dataset (train.jsonl) → Lynkr kNN bootstrap JSONL.
4
+
5
+ Stream-downloads from HuggingFace and maps mixtral_score (1-5) to your
6
+ TIER_* model config so the kNN index learns to recommend YOUR real models,
7
+ not RouteLLM's pool (mixtral / gpt-4).
8
+
9
+ Output format matches scripts/build-knn-index.js bootstrap input:
10
+ {"query": "...", "provider": "ollama", "model": "minimax-m2.5:cloud",
11
+ "quality": 95, "cost": 0.001, "latency": 200, "tier": "SIMPLE"}
12
+
13
+ Usage:
14
+ .venv/bin/python scripts/convert-routellm.py [--limit 30000] -o routellm.jsonl
15
+ """
16
+ import argparse
17
+ import json
18
+ import sys
19
+ import urllib.request
20
+ import ssl
21
+
22
+ # Map mixtral_score → user's tier config.
23
+ # Spread is intentionally aggressive: only score=5 (Mixtral truly nailed it)
24
+ # stays SIMPLE; score=4 escalates to MEDIUM so the kNN index has enough MEDIUM
25
+ # examples to actually recommend non-SIMPLE tiers (vs the previous mapping
26
+ # where 86% of records were SIMPLE and kNN never disagreed with the heuristic).
27
+ SCORE_TO_ROUTE = {
28
+ 5: {"tier": "SIMPLE", "provider": "ollama", "model": "minimax-m2.5:cloud", "quality": 95, "cost": 0.001, "latency": 200},
29
+ 4: {"tier": "MEDIUM", "provider": "azure-anthropic", "model": "claude-sonnet-4-6", "quality": 80, "cost": 0.003, "latency": 1500},
30
+ 3: {"tier": "MEDIUM", "provider": "azure-anthropic", "model": "claude-sonnet-4-6", "quality": 70, "cost": 0.003, "latency": 1500},
31
+ 2: {"tier": "COMPLEX", "provider": "azure-anthropic", "model": "claude-sonnet-4-6", "quality": 55, "cost": 0.003, "latency": 1500},
32
+ 1: {"tier": "REASONING", "provider": "azure-anthropic", "model": "claude-opus-4-8", "quality": 40, "cost": 0.015, "latency": 3500},
33
+ }
34
+
35
+ URL = "https://huggingface.co/datasets/routellm/gpt4_dataset/resolve/main/train.jsonl"
36
+
37
+
38
+ def main():
39
+ p = argparse.ArgumentParser()
40
+ p.add_argument("--output", "-o", default="data/routellm-bootstrap.jsonl")
41
+ p.add_argument("--limit", type=int, default=None,
42
+ help="Only convert first N rows")
43
+ args = p.parse_args()
44
+
45
+ print(f"Streaming {URL} …", file=sys.stderr)
46
+ # Some macOS Python builds have SSL issues with huggingface.co — fall back
47
+ # to unverified context if needed. The data is public, no auth.
48
+ ctx = ssl.create_default_context()
49
+ try:
50
+ with urllib.request.urlopen(URL, context=ctx, timeout=60) as resp:
51
+ _stream(resp, args.output, args.limit)
52
+ except (ssl.SSLError, OSError) as e:
53
+ print(f"SSL strict failed ({e}); retrying with unverified context", file=sys.stderr)
54
+ ctx = ssl._create_unverified_context()
55
+ with urllib.request.urlopen(URL, context=ctx, timeout=60) as resp:
56
+ _stream(resp, args.output, args.limit)
57
+
58
+
59
+ def _stream(resp, out_path, limit):
60
+ written = 0
61
+ by_score = {s: 0 for s in SCORE_TO_ROUTE}
62
+ skipped = 0
63
+ with open(out_path, "w") as out:
64
+ for line in resp:
65
+ if limit and written >= limit:
66
+ break
67
+ try:
68
+ row = json.loads(line)
69
+ except Exception:
70
+ skipped += 1
71
+ continue
72
+ score = row.get("mixtral_score")
73
+ if score not in SCORE_TO_ROUTE:
74
+ skipped += 1
75
+ continue
76
+ route = SCORE_TO_ROUTE[score]
77
+ rec = {
78
+ "query": row.get("prompt", ""),
79
+ "provider": route["provider"],
80
+ "model": route["model"],
81
+ "quality": route["quality"],
82
+ "cost": route["cost"],
83
+ "latency": route["latency"],
84
+ "tier": route["tier"],
85
+ }
86
+ if not rec["query"]:
87
+ skipped += 1
88
+ continue
89
+ out.write(json.dumps(rec) + "\n")
90
+ written += 1
91
+ by_score[score] += 1
92
+ if written % 5000 == 0:
93
+ print(f" {written} written…", file=sys.stderr)
94
+
95
+ print(f"\nWrote {written} records to {out_path} ({skipped} skipped)", file=sys.stderr)
96
+ print(f"By mixtral_score: {by_score}", file=sys.stderr)
97
+ print(f"\nTier distribution:", file=sys.stderr)
98
+ for score, route in SCORE_TO_ROUTE.items():
99
+ n = by_score[score]
100
+ if n:
101
+ print(f" {route['tier']:>10} {route['provider']}:{route['model']:30s} n={n}", file=sys.stderr)
102
+
103
+
104
+ if __name__ == "__main__":
105
+ main()
@@ -1,4 +1,5 @@
1
1
  const os = require("os");
2
+ const v8 = require("v8");
2
3
  const logger = require("../../logger");
3
4
  const { ServiceUnavailableError } = require("./error-handling");
4
5
 
@@ -18,11 +19,31 @@ class LoadShedder {
18
19
  this.memoryThreshold = options.memoryThreshold || 0.85; // 85%
19
20
  this.heapThreshold = options.heapThreshold || 0.95; // 95% (increased from 90% to prevent false positives from temporary allocation spikes)
20
21
  this.activeRequestsThreshold = options.activeRequestsThreshold || 1000;
22
+ // Absolute free-heap floor. Percent-of-heapTotal is unreliable in both
23
+ // directions: heapTotal is V8's CURRENTLY allocated heap (grows on
24
+ // demand toward the real limit), and at the limit "99%" leaves ~40MB on
25
+ // a 4GB heap — nowhere near enough to parse one multi-MB request body.
26
+ // 2026-08-19 OOM: a 5MB compact request passed the percent check and
27
+ // died allocating during JSON.parse. Shed while there is still room to
28
+ // finish in-flight work.
29
+ this.headroomMB = options.headroomMB || 512;
30
+ this.heapLimitMB = v8.getHeapStatistics().heap_size_limit / (1024 * 1024);
31
+
32
+ // Watchdog: how long the heap may stay below the headroom floor with
33
+ // zero active requests before the process self-restarts. Retained
34
+ // memory GC can't reclaim never recovers on its own — without this the
35
+ // server sheds 100% of traffic until someone restarts it by hand.
36
+ this.wedgedRestartMs = options.wedgedRestartMs || 60000;
37
+ this.selfRestart = options.selfRestart !== false;
38
+ this.wedgedSince = null;
21
39
 
22
40
  // State
23
41
  this.activeRequests = 0;
24
42
  this.totalShed = 0;
25
- this.lastCheck = Date.now();
43
+ // 0, not Date.now(): the first isOverloaded() call must actually
44
+ // evaluate instead of returning the constructor's cached "false"
45
+ // for the first checkInterval window.
46
+ this.lastCheck = 0;
26
47
  this.checkInterval = options.checkInterval || 1000; // Check every second
27
48
  this.cachedOverloadState = false;
28
49
  }
@@ -42,26 +63,32 @@ class LoadShedder {
42
63
 
43
64
  // Check memory usage
44
65
  const memUsage = process.memoryUsage();
45
- const heapUsedPercent = memUsage.heapUsed / memUsage.heapTotal;
46
-
47
- // FIX: Only trigger if BOTH percentage is high AND actual usage is significant
48
- // This prevents false positives on startup when heapTotal is small but will grow
49
- const heapUsedMB = memUsage.heapUsed / (1024 * 1024);
50
- const minHeapThresholdMB = 500; // Only shed load if using more than 500MB
51
66
 
52
- if (heapUsedPercent > this.heapThreshold && heapUsedMB > minHeapThresholdMB) {
67
+ // Absolute headroom against the true V8 limit — the primary guard.
68
+ const heapStats = v8.getHeapStatistics();
69
+ const freeMB = (heapStats.heap_size_limit - heapStats.used_heap_size) / (1024 * 1024);
70
+ if (freeMB < this.headroomMB) {
53
71
  logger.warn(
54
72
  {
55
- heapUsedPercent: (heapUsedPercent * 100).toFixed(2),
56
- heapUsedMB: heapUsedMB.toFixed(2),
57
- threshold: (this.heapThreshold * 100).toFixed(2),
58
- minThresholdMB: minHeapThresholdMB,
73
+ freeMB: freeMB.toFixed(0),
74
+ headroomMB: this.headroomMB,
75
+ heapLimitMB: this.heapLimitMB.toFixed(0),
76
+ usedMB: (heapStats.used_heap_size / (1024 * 1024)).toFixed(0),
59
77
  },
60
- "Load shedding: Heap usage exceeded threshold"
78
+ "Load shedding: heap headroom below floor"
61
79
  );
80
+ this._trackWedged(now);
62
81
  this.cachedOverloadState = true;
63
82
  return true;
64
83
  }
84
+ this.wedgedSince = null;
85
+
86
+ // NOTE: the old heapUsed/heapTotal percent check is gone. heapTotal is
87
+ // V8's *currently allocated* heap, which it keeps close to usage, so the
88
+ // ratio reads ~99% during perfectly normal operation — it shed all
89
+ // traffic at 755MB used with gigabytes of real headroom remaining
90
+ // (2026-08-20 false-positive incident). The absolute-headroom check
91
+ // against heap_size_limit above is the correct form of this guard.
65
92
 
66
93
  // Check RSS / system memory
67
94
  const rssPercent = memUsage.rss / os.totalmem();
@@ -94,6 +121,32 @@ class LoadShedder {
94
121
  return false;
95
122
  }
96
123
 
124
+ /**
125
+ * Track how long the heap has been wedged below the headroom floor.
126
+ * If it persists past wedgedRestartMs with no in-flight requests, the
127
+ * memory is retained (not transient) and GC cannot recover it — exit so
128
+ * the supervisor restarts a clean process instead of shedding forever.
129
+ */
130
+ _trackWedged(now) {
131
+ if (!this.selfRestart) return;
132
+ if (this.wedgedSince === null) {
133
+ this.wedgedSince = now;
134
+ return;
135
+ }
136
+ if (now - this.wedgedSince >= this.wedgedRestartMs && this.activeRequests === 0) {
137
+ logger.fatal(
138
+ {
139
+ wedgedForMs: now - this.wedgedSince,
140
+ totalShed: this.totalShed,
141
+ heapLimitMB: this.heapLimitMB.toFixed(0),
142
+ },
143
+ "Load shedding: heap wedged below headroom floor with no active requests — exiting for supervisor restart"
144
+ );
145
+ // Give the fatal log a moment to flush, then exit non-zero.
146
+ setTimeout(() => process.exit(1), 250).unref();
147
+ }
148
+ }
149
+
97
150
  /**
98
151
  * Get current metrics
99
152
  */
@@ -107,10 +160,15 @@ class LoadShedder {
107
160
  heapTotalMB: (memUsage.heapTotal / (1024 * 1024)).toFixed(2),
108
161
  rssMB: (memUsage.rss / (1024 * 1024)).toFixed(2),
109
162
  rssPercent: ((memUsage.rss / os.totalmem()) * 100).toFixed(2),
163
+ heapLimitMB: this.heapLimitMB.toFixed(2),
164
+ freeHeadroomMB: ((v8.getHeapStatistics().heap_size_limit - v8.getHeapStatistics().used_heap_size) / (1024 * 1024)).toFixed(2),
110
165
  thresholds: {
111
166
  heapThreshold: (this.heapThreshold * 100).toFixed(2),
112
167
  memoryThreshold: (this.memoryThreshold * 100).toFixed(2),
113
168
  activeRequestsThreshold: this.activeRequestsThreshold,
169
+ headroomMB: this.headroomMB,
170
+ wedgedRestartMs: this.wedgedRestartMs,
171
+ selfRestart: this.selfRestart,
114
172
  },
115
173
  };
116
174
  }
@@ -129,6 +187,11 @@ function getLoadShedder(options) {
129
187
  process.env.LOAD_SHEDDING_ACTIVE_REQUESTS_THRESHOLD || "1000",
130
188
  10
131
189
  ),
190
+ // Not env-configurable by design: these are crash-prevention floors,
191
+ // and there is no deployment where turning them off is correct.
192
+ headroomMB: 512,
193
+ wedgedRestartMs: 60000,
194
+ selfRestart: true,
132
195
  };
133
196
  instance = new LoadShedder({ ...defaultOptions, ...options });
134
197
  }
@@ -426,22 +426,42 @@ async function forwardAnthropicStreamAsOpenAIChunks(res, stream, requestedModel)
426
426
  res.write("data: [DONE]\n\n");
427
427
  res.end();
428
428
  return;
429
+ case "error":
430
+ // Forward upstream stream errors in OpenAI wire shape
431
+ // (data: {"error": ...}) and terminate WITHOUT finish_reason or
432
+ // [DONE]. Writing "stop" here told agent clients the model chose
433
+ // to end its turn — they stopped mid-task instead of retrying
434
+ // (2026-08-19 incident: opencode stalled between tool calls).
435
+ logger.warn({ error: ev.error?.message }, "Forwarding upstream stream error to OpenAI-surface client");
436
+ res.write(`data: ${JSON.stringify({ error: { message: ev.error?.message || "upstream stream error", type: ev.error?.type || "server_error", code: null } })}\n\n`);
437
+ res.end();
438
+ return;
429
439
  default:
430
440
  break; // ping, thinking deltas, unknown events — skip
431
441
  }
432
442
  if (res.writableEnded || res.destroyed) return;
433
443
  }
434
444
  }
435
- // Upstream closed without message_stop — still terminate the client stream cleanly.
445
+ // Upstream closed without message_stop. If a stop_reason already arrived
446
+ // the turn is materially complete — finish normally. Otherwise this is a
447
+ // truncated stream: send an error chunk, never a fabricated "stop".
436
448
  if (!res.writableEnded) {
437
- res.write(chunk({}, finishReason || "stop"));
438
- res.write("data: [DONE]\n\n");
449
+ if (finishReason) {
450
+ res.write(chunk({}, finishReason));
451
+ res.write("data: [DONE]\n\n");
452
+ } else {
453
+ logger.warn("Upstream stream ended without message_stop — sending error chunk to OpenAI-surface client");
454
+ res.write(`data: ${JSON.stringify({ error: { message: "Lynkr: upstream stream truncated — retry", type: "server_error", code: null } })}\n\n`);
455
+ }
439
456
  res.end();
440
457
  }
441
458
  } catch (err) {
442
459
  logger.warn({ err: err.message }, "OpenAI-surface stream forward failed mid-flight");
443
460
  if (!res.writableEnded) {
444
- try { res.write(chunk({}, "stop")); res.write("data: [DONE]\n\n"); res.end(); } catch { /* client gone */ }
461
+ try {
462
+ res.write(`data: ${JSON.stringify({ error: { message: `Lynkr: stream forward failed — retry (${err.message})`, type: "server_error", code: null } })}\n\n`);
463
+ res.end();
464
+ } catch { /* client gone */ }
445
465
  }
446
466
  }
447
467
  }
@@ -594,13 +614,25 @@ router.post("/chat/completions", async (req, res) => {
594
614
  // "task complete" and terminate mid-task. Killing the stream without
595
615
  // a finish chunk makes the client retry instead.
596
616
  if (!content && (!toolCalls || toolCalls.length === 0)) {
617
+ const upstreamErr = result.body?.error || {};
597
618
  logger.error({
598
619
  finishReason: openaiResponse.choices[0]?.finish_reason,
599
620
  terminationReason: result?.terminationReason,
600
621
  status: result?.status,
601
- }, "Empty completion reached serving path — aborting stream to force client retry");
602
- res.write(`data: ${JSON.stringify({ error: { message: "Upstream returned an empty completion; retry.", type: "server_error", code: "empty_completion" } })}\n\n`);
603
- res.destroy();
622
+ upstreamError: upstreamErr.message || null,
623
+ }, "Empty completion reached serving path — sending error chunk to client");
624
+ // res.destroy() here surfaced as ECONNRESET ("connection reset by
625
+ // server") in clients, and the error chunk raced the teardown so
626
+ // it often never arrived. A cleanly-ended SSE error chunk (no
627
+ // finish chunk, no [DONE]) delivers the real upstream error and
628
+ // still reads as a failed request that agent clients retry.
629
+ res.write(`data: ${JSON.stringify({ error: {
630
+ message: upstreamErr.message || "Upstream returned an empty completion; retry.",
631
+ type: upstreamErr.type || "server_error",
632
+ code: upstreamErr.code || "empty_completion",
633
+ ...(result?.status ? { status: result.status } : {}),
634
+ } })}\n\n`);
635
+ res.end();
604
636
  return;
605
637
  }
606
638
 
@@ -3541,12 +3541,101 @@ async function invokeModel(body, options = {}) {
3541
3541
 
3542
3542
  const fallbackStart = Date.now();
3543
3543
 
3544
+ // Strip the failed primary's routing stamps before re-dispatch: every
3545
+ // provider client prefers body._tierModel over its own configured
3546
+ // model, so leaving the failed tier's model on the body makes the
3547
+ // fallback provider serve a model it doesn't have — azure-openai
3548
+ // treats it as the deployment id and 404s with DeploymentNotFound
3549
+ // (live incident 2026-08-19). The fallback provider must fall through
3550
+ // to its own configured model.
3551
+ const fallbackBody = { ...body };
3552
+ delete fallbackBody._tierModel;
3553
+ delete fallbackBody._suggestionModeModel;
3554
+
3544
3555
  const fallbackResult = await fallbackBreaker.execute(async () => {
3545
- return await invokeProvider(fallbackProvider, body, incomingHeaders);
3556
+ return await invokeProvider(fallbackProvider, fallbackBody, incomingHeaders);
3546
3557
  });
3547
3558
 
3548
3559
  const fallbackLatency = Date.now() - fallbackStart;
3549
3560
 
3561
+ // Provider clients return {ok:false} on 4xx/5xx instead of throwing,
3562
+ // so a fallback rung that answered with an error body used to sail
3563
+ // through the success path below — logging "Fallback to cloud
3564
+ // provider succeeded" and recording status_code:200 telemetry for
3565
+ // what was actually a double failure. Treat it as one.
3566
+ if (fallbackResult?.ok === false) {
3567
+ const fbErrMessage = fallbackResult.json?.error?.message || String(fallbackResult.text || "").slice(0, 120) || "error response";
3568
+
3569
+ metricsCollector.recordFallbackFailure();
3570
+ metricsCollector.recordDatabricksRequest(false, retries);
3571
+ healthTracker.recordFailure(fallbackProvider, new Error(fbErrMessage), fallbackResult.status);
3572
+
3573
+ logger.warn({
3574
+ originalProvider: initialProvider,
3575
+ fallbackProvider,
3576
+ status: fallbackResult.status,
3577
+ error: fbErrMessage,
3578
+ }, "Fallback provider answered with an error response");
3579
+
3580
+ // Thread the whole trail into the error payload so the client sees
3581
+ // every failure, not just this one (e.g. "moonshot: quota" matters
3582
+ // more than "azure: 404").
3583
+ failureTrail.push(`${fallbackProvider}: HTTP ${fallbackResult.status} — ${fbErrMessage}`);
3584
+ if (fallbackResult.json && typeof fallbackResult.json === "object") {
3585
+ fallbackResult.json._lynkr_failure_trail = failureTrail;
3586
+ }
3587
+
3588
+ const fbFailLatency = Date.now() - startTime;
3589
+ telemetry.record({
3590
+ request_id: crypto.randomUUID(),
3591
+ session_id: body._sessionId || null,
3592
+ timestamp: Date.now(),
3593
+ complexity_score: routingResult.score ?? null,
3594
+ tier: routingDecision.tier,
3595
+ provider: fallbackProvider,
3596
+ model: routingDecision.model ?? body._tierModel ?? null,
3597
+ routing_method: "fallback",
3598
+ was_fallback: true,
3599
+ latency_ms: fbFailLatency,
3600
+ status_code: fallbackResult.status || null,
3601
+ error_type: "fallback_error_response",
3602
+ quality_score: 0,
3603
+ base_tier: routingResult.base_tier ?? null,
3604
+ escalation_source: routingResult.escalation_source ?? null,
3605
+ propensity: routingResult.propensity ?? null,
3606
+ candidates: routingResult.candidates ?? null,
3607
+ pinned: routingResult.pinned ? 1 : 0,
3608
+ switch_reason: routingResult.switch_reason ?? null,
3609
+ cache_decision: routingResult._cacheDecision ?? null,
3610
+ });
3611
+
3612
+ // WS5.4 — feedback loop (fallback answered with an error). Same
3613
+ // hard-negative signal as a double failure.
3614
+ recordFeedbackOutcome({
3615
+ routingResult,
3616
+ body,
3617
+ outcome: {
3618
+ qualityScore: 0,
3619
+ costUsd: null,
3620
+ latencyMs: fbFailLatency,
3621
+ statusCode: fallbackResult.status || null,
3622
+ errorType: "fallback_error_response",
3623
+ wasFallback: true,
3624
+ },
3625
+ });
3626
+
3627
+ return {
3628
+ ...fallbackResult,
3629
+ actualProvider: fallbackProvider,
3630
+ routingDecision: {
3631
+ ...routingDecision,
3632
+ provider: fallbackProvider,
3633
+ method: 'fallback',
3634
+ fallbackReason: reason,
3635
+ },
3636
+ };
3637
+ }
3638
+
3550
3639
  metricsCollector.recordFallbackSuccess(fallbackLatency);
3551
3640
  metricsCollector.recordDatabricksRequest(true, retries);
3552
3641
  healthTracker.recordSuccess(fallbackProvider, fallbackLatency);
@@ -3635,16 +3724,6 @@ async function invokeModel(body, options = {}) {
3635
3724
  },
3636
3725
  });
3637
3726
 
3638
- // The last rung also answered with an error — thread the whole trail
3639
- // into the error payload so the client sees every failure, not just
3640
- // this one (e.g. "moonshot: quota" matters more than "azure: 404").
3641
- if (fallbackResult?.ok === false) {
3642
- failureTrail.push(`${fallbackProvider}: HTTP ${fallbackResult.status} — ${fallbackResult.json?.error?.message || String(fallbackResult.text || "").slice(0, 120) || "error response"}`);
3643
- if (fallbackResult.json && typeof fallbackResult.json === "object") {
3644
- fallbackResult.json._lynkr_failure_trail = failureTrail;
3645
- }
3646
- }
3647
-
3648
3727
  return {
3649
3728
  ...fallbackResult,
3650
3729
  actualProvider: fallbackProvider,
@@ -381,6 +381,15 @@ async function withCockatielRetry(fn, options = {}) {
381
381
  const retryAfterDate = new Date(retryAfter);
382
382
  delay = retryAfterDate.getTime() - Date.now();
383
383
  }
384
+ // Retry-After is a floor, not the full answer: Azure sends
385
+ // "Retry-After: 1" on token-window 429s that need far longer to
386
+ // drain, so honoring it literally re-hammers the window every
387
+ // second and it never recovers (2026-08-19 incident). Back off
388
+ // exponentially on top of the header, with jitter.
389
+ const floorMs = Math.max(0, delay || 0);
390
+ const backoffMs = 2000 * Math.pow(config.backoffMultiplier, attempt);
391
+ const cappedMs = Math.min(Math.max(floorMs, backoffMs), 60000);
392
+ delay = cappedMs + cappedMs * config.jitterFactor * Math.random();
384
393
  } else {
385
394
  // Exponential backoff with longer delays for rate limiting
386
395
  const baseDelay = 2000 * Math.pow(config.backoffMultiplier, attempt);
@@ -1356,6 +1356,18 @@ function buildNonJsonResponse(databricksResponse) {
1356
1356
  };
1357
1357
  }
1358
1358
 
1359
+ /**
1360
+ * Visible routing badge for live-streamed responses, built from the actual
1361
+ * post-routing decision (tier fallbacks included). Matches the format of
1362
+ * openai-router's lynkrBadge and the Anthropic router's intent badge so
1363
+ * LYNKR_BADGE_PREFIX_RE strips all three identically from replayed history.
1364
+ */
1365
+ function buildRoutingBadge(rd) {
1366
+ if (!rd || !rd.tier) return null;
1367
+ const score = typeof rd.score === "number" ? ` · score ${rd.score}` : "";
1368
+ return `*[Lynkr] ${rd.tier} → ${rd.model || "—"} (${rd.provider || "—"})${score}*\n\n`;
1369
+ }
1370
+
1359
1371
  function buildStreamingResponse(databricksResponse) {
1360
1372
  return {
1361
1373
  status: databricksResponse.status,
@@ -1755,7 +1767,24 @@ IMPORTANT TOOL USAGE RULES:
1755
1767
  };
1756
1768
  const headroomProvider = process.env.HEADROOM_PROVIDER || 'anthropic';
1757
1769
  const headroomSafeProviders = headroomProviderMap[headroomProvider] || new Set();
1758
- const headroomCompatible = headroomSafeProviders.has(providerType);
1770
+ // Under tier routing the static providerType is just the MODEL_PROVIDER
1771
+ // default (usually "databricks") — the real provider is chosen per-request
1772
+ // later in invokeModel, AFTER this compression point, so gating on it
1773
+ // silently disabled Headroom for every tier-routed setup. Gate on the
1774
+ // configured tier fleet instead: compress only when every routable tier
1775
+ // provider is format-safe, since any request may land on any tier.
1776
+ let headroomCompatible = headroomSafeProviders.has(providerType);
1777
+ if (!headroomCompatible) {
1778
+ try {
1779
+ const { getModelTierSelector } = require('../routing/model-tiers');
1780
+ const sel = getModelTierSelector();
1781
+ const tierProviders = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING']
1782
+ .map((t) => { try { return sel.selectModel(t)?.provider; } catch { return null; } })
1783
+ .filter(Boolean)
1784
+ .map((p) => (p === 'z-ai' ? 'zai' : p));
1785
+ headroomCompatible = tierProviders.length > 0 && tierProviders.every((p) => headroomSafeProviders.has(p));
1786
+ } catch { /* tier routing not configured — keep the static gate result */ }
1787
+ }
1759
1788
 
1760
1789
  if (isHeadroomEnabled() && headroomCompatible && cleanPayload.messages && cleanPayload.messages.length > 0) {
1761
1790
  try {
@@ -2053,7 +2082,16 @@ IMPORTANT TOOL USAGE RULES:
2053
2082
  const routingDecision = databricksResponse.routingDecision || {};
2054
2083
  const transformed = sseTransform.openaiToAnthropicSSE(databricksResponse.stream, {
2055
2084
  model: requestedModel,
2056
- badgeText: config.routing?.visibleInteraction ? options?.streamBadgeText || null : null,
2085
+ // Badge precedence: an explicit streamBadgeText (Anthropic router's
2086
+ // pre-computed intent badge) wins; otherwise build one from the
2087
+ // ACTUAL post-routing decision so OpenAI-surface live streams
2088
+ // (opencode et al.) show which model served the response —
2089
+ // previously this path silently dropped the badge entirely.
2090
+ // History pollution is covered: stripLynkrBadges in invokeModel
2091
+ // removes resubmitted badges before anything reaches a provider.
2092
+ badgeText: config.routing?.visibleInteraction
2093
+ ? (options?.streamBadgeText || buildRoutingBadge(routingDecision))
2094
+ : null,
2057
2095
  onClose: (stats) => {
2058
2096
  try {
2059
2097
  const telemetry = require("../routing/telemetry");
@@ -2080,6 +2118,7 @@ IMPORTANT TOOL USAGE RULES:
2080
2118
  logger.info({
2081
2119
  provider: routingDecision.provider || _streamProvider,
2082
2120
  stopReason: stats.stopReason,
2121
+ finishReason: stats.finishReason ?? null,
2083
2122
  toolCalls: stats.toolCalls.map((t) => t.name),
2084
2123
  outputTokens: stats.usage.output_tokens,
2085
2124
  }, "[SSETransform] Stream closed");
@@ -3191,4 +3230,6 @@ module.exports = {
3191
3230
  toAnthropicResponse,
3192
3231
  // Exported for unit testing of loop trimming (task-preservation contract).
3193
3232
  trimLoopMessages,
3233
+ // Exported for unit testing of the live-stream routing badge.
3234
+ buildRoutingBadge,
3194
3235
  };
@@ -256,6 +256,7 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
256
256
  error: { type: "overloaded_error", message: `Lynkr: upstream stream failed — retry (${err.message})` },
257
257
  });
258
258
  stats.stopReason = "stream_error";
259
+ stats.finishReason = finishReason ?? null;
259
260
  if (opts.onClose) { try { opts.onClose(stats); } catch { /* non-fatal */ } }
260
261
  return;
261
262
  }
@@ -263,10 +264,36 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
263
264
  // Normal end of stream ([DONE] or upstream EOF).
264
265
  yield* startMessage();
265
266
  yield* closeTextBlock();
267
+
268
+ // An EOF with no finish_reason is a dropped upstream, not a completed
269
+ // turn. Mapping it to end_turn hands the client a success-shaped stop —
270
+ // agent loops halt mid-task believing the model chose to stop (live
271
+ // incident 2026-08-19: opencode stalled between tool calls; the stream
272
+ // closed as end_turn with null output tokens). Spec-compliant upstreams
273
+ // always send finish_reason on the last choice chunk, so signal an error
274
+ // and let the client retry, mirroring the mid-flight failure path above.
275
+ // Accumulated tool fragments are NOT flushed: without finish_reason their
276
+ // argument JSON may be truncated mid-string.
277
+ if (!finishReason) {
278
+ logger.warn(
279
+ { droppedToolFragments: toolAcc.size, outputTokens: stats.usage.output_tokens },
280
+ "[SSETransform] Upstream ended without finish_reason — signaling stream error"
281
+ );
282
+ yield _sse("error", {
283
+ type: "error",
284
+ error: { type: "overloaded_error", message: "Lynkr: upstream ended without finish_reason — retry" },
285
+ });
286
+ stats.stopReason = "stream_error";
287
+ stats.finishReason = null;
288
+ if (opts.onClose) { try { opts.onClose(stats); } catch { /* non-fatal */ } }
289
+ return;
290
+ }
291
+
266
292
  yield* flushToolBlocks();
267
293
 
268
294
  const stopReason = _mapFinishReason(finishReason);
269
295
  stats.stopReason = stopReason;
296
+ stats.finishReason = finishReason;
270
297
  yield _sse("message_delta", {
271
298
  type: "message_delta",
272
299
  delta: { stop_reason: stopReason, stop_sequence: null },