lynkr 9.14.9 → 9.14.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/model-capability-seeds.json +31 -4
- package/package.json +1 -1
- package/src/api/middleware/session.js +10 -1
- package/src/cache/prompt.js +40 -1
- package/src/cache/semantic.js +7 -0
- package/src/clients/databricks.js +86 -2
- package/src/clients/openrouter-utils.js +84 -14
- package/src/logger/audit-logger.js +6 -0
- package/src/logger/deduplicator.js +10 -0
- package/src/orchestrator/index.js +26 -7
- package/src/routing/client-profiles.js +19 -0
- package/src/routing/index.js +43 -19
- package/src/routing/model-tiers.js +10 -11
- package/src/routing/risk-classifier.js +13 -1
- package/src/routing/tokenizer.js +22 -5
- package/src/routing/vision.js +246 -0
- package/src/training/trajectory-compressor.js +8 -0
- package/src/utils/tokens.js +3 -3
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 1,
|
|
3
|
-
"updatedAt": "2026-09-
|
|
4
|
-
"notes": "Reviewed per-family capability seeds. Values are curated estimates triangulated from SWE-Bench Verified, TerminalBench, LiveCodeBench/BigCodeBench and Artificial Analysis coding index (see scripts/seed-capabilities.js to regenerate from live sources). Heads: reasoning/codegen/debugging/tool_use in [0,1]. Keys are normalized family ids (see capability-seeds/family.js) with trailing-* wildcards matched longest-first. Quantized self-hosted servings take an automatic -0.02 haircut at resolve time. Unknown families fall through to the family heuristic, then tier-slot caps.",
|
|
3
|
+
"updatedAt": "2026-09-16",
|
|
4
|
+
"notes": "Reviewed per-family capability seeds. Values are curated estimates triangulated from SWE-Bench Verified, TerminalBench, LiveCodeBench/BigCodeBench and Artificial Analysis coding index (see scripts/seed-capabilities.js to regenerate from live sources). Heads: reasoning/codegen/debugging/tool_use in [0,1]. Keys are normalized family ids (see capability-seeds/family.js) with trailing-* wildcards matched longest-first. Quantized self-hosted servings take an automatic -0.02 haircut at resolve time. Unknown families fall through to the family heuristic, then tier-slot caps. 2026-09-16: added local-runtime + tier-catalog gaps (ornith, qwen2.5 base sizes, qwen3.5 sizes, spark-x2.5, gemma4, minimax-m3, versioned claude-3.5/gpt-4o/gpt-4.1 names, mistral bare, deepseek-coder, codellama, llama-3.1-8b, gemini-1.5-flash, amazon nova, gpt-4-turbo, gpt-3.5-turbo). Entries without benchmark rows are marked provisional — promote on data.",
|
|
5
5
|
"seeds": {
|
|
6
6
|
"claude-opus-4*": { "caps": { "reasoning": 0.9, "codegen": 0.9, "debugging": 0.9, "tool_use": 0.9 }, "sources": ["swe-bench-verified", "artificial-analysis"], "note": "curated-estimate: flagship tier" },
|
|
7
7
|
"claude-opus-3*": { "caps": { "reasoning": 0.8, "codegen": 0.82, "debugging": 0.8, "tool_use": 0.78 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
8
8
|
"claude-sonnet-4*": { "caps": { "reasoning": 0.78, "codegen": 0.8, "debugging": 0.78, "tool_use": 0.78 }, "sources": ["swe-bench-verified", "artificial-analysis"], "note": "curated-estimate" },
|
|
9
9
|
"claude-sonnet-3.5": { "caps": { "reasoning": 0.72, "codegen": 0.75, "debugging": 0.72, "tool_use": 0.7 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
10
|
+
"claude-3.5-sonnet*": { "caps": { "reasoning": 0.72, "codegen": 0.75, "debugging": 0.72, "tool_use": 0.7 }, "sources": ["swe-bench-verified"], "note": "alias: dated/provider form (claude-3-5-sonnet-20241022) shares claude-sonnet-3.5 caps" },
|
|
10
11
|
"claude-haiku*": { "caps": { "reasoning": 0.42, "codegen": 0.48, "debugging": 0.42, "tool_use": 0.45 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
12
|
+
"claude-3.5-haiku*": { "caps": { "reasoning": 0.42, "codegen": 0.48, "debugging": 0.42, "tool_use": 0.45 }, "sources": ["swe-bench-verified"], "note": "alias: dated/provider form (claude-3-5-haiku-20241022) shares claude-haiku caps" },
|
|
11
13
|
"gpt-5.6*": { "caps": { "reasoning": 0.85, "codegen": 0.85, "debugging": 0.85, "tool_use": 0.82 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: flagship tier" },
|
|
12
14
|
"gpt-5.4*": { "caps": { "reasoning": 0.85, "codegen": 0.85, "debugging": 0.85, "tool_use": 0.82 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: flagship tier" },
|
|
13
15
|
"gpt-5.3*": { "caps": { "reasoning": 0.82, "codegen": 0.85, "debugging": 0.82, "tool_use": 0.8 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
@@ -15,10 +17,12 @@
|
|
|
15
17
|
"gpt-5.1*": { "caps": { "reasoning": 0.7, "codegen": 0.72, "debugging": 0.7, "tool_use": 0.68 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
16
18
|
"gpt-5-mini*": { "caps": { "reasoning": 0.6, "codegen": 0.65, "debugging": 0.6, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
17
19
|
"gpt-5-nano*": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.45 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
18
|
-
"gpt-4o": { "caps": { "reasoning": 0.65, "codegen": 0.7, "debugging": 0.65, "tool_use": 0.68 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate" },
|
|
20
|
+
"gpt-4o*": { "caps": { "reasoning": 0.65, "codegen": 0.7, "debugging": 0.65, "tool_use": 0.68 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate; wildcard also covers dated snapshots (gpt-4o-2024-*). Exact gpt-4o-mini still wins via exact-first lookup." },
|
|
19
21
|
"gpt-4o-mini": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.48 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
20
|
-
"gpt-4.1": { "caps": { "reasoning": 0.68, "codegen": 0.72, "debugging": 0.68, "tool_use": 0.7 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
22
|
+
"gpt-4.1*": { "caps": { "reasoning": 0.68, "codegen": 0.72, "debugging": 0.68, "tool_use": 0.7 }, "sources": ["swe-bench-verified"], "note": "curated-estimate; wildcard also covers dated snapshots. Exact gpt-4.1-mini still wins via exact-first lookup." },
|
|
21
23
|
"gpt-4.1-mini": { "caps": { "reasoning": 0.5, "codegen": 0.55, "debugging": 0.5, "tool_use": 0.52 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
24
|
+
"gpt-4-turbo*": { "caps": { "reasoning": 0.62, "codegen": 0.68, "debugging": 0.62, "tool_use": 0.64 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (MEDIUM openai slot); sized just below gpt-4o — promote on benchmark rows" },
|
|
25
|
+
"gpt-3.5-turbo": { "caps": { "reasoning": 0.3, "codegen": 0.35, "debugging": 0.3, "tool_use": 0.33 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (SIMPLE openai slot); matches tiny/small ladder rung" },
|
|
22
26
|
"o3": { "caps": { "reasoning": 0.88, "codegen": 0.8, "debugging": 0.85, "tool_use": 0.7 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: reasoning tilt" },
|
|
23
27
|
"o3-mini": { "caps": { "reasoning": 0.72, "codegen": 0.68, "debugging": 0.7, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: reasoning tilt" },
|
|
24
28
|
"o4-mini": { "caps": { "reasoning": 0.7, "codegen": 0.66, "debugging": 0.68, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
@@ -28,11 +32,16 @@
|
|
|
28
32
|
"gemini-3-pro*": { "caps": { "reasoning": 0.85, "codegen": 0.83, "debugging": 0.85, "tool_use": 0.78 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
29
33
|
"gemini-2.5-flash": { "caps": { "reasoning": 0.55, "codegen": 0.6, "debugging": 0.55, "tool_use": 0.58 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
30
34
|
"gemini-2.0-flash": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.5 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
35
|
+
"gemini-1.5-flash*": { "caps": { "reasoning": 0.42, "codegen": 0.48, "debugging": 0.42, "tool_use": 0.46 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (openrouter google/gemini-flash-1.5); sized just below gemini-2.0-flash" },
|
|
31
36
|
"gemini-1.5-pro": { "caps": { "reasoning": 0.6, "codegen": 0.62, "debugging": 0.6, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
37
|
+
"amazon.nova-lite*": { "caps": { "reasoning": 0.35, "codegen": 0.4, "debugging": 0.35, "tool_use": 0.38 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (bedrock SIMPLE slot); dotted key matches normalized amazon.nova-lite-v1-0 form" },
|
|
38
|
+
"amazon.nova-pro*": { "caps": { "reasoning": 0.5, "codegen": 0.55, "debugging": 0.5, "tool_use": 0.52 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (bedrock MEDIUM slot)" },
|
|
32
39
|
"deepseek-r1": { "caps": { "reasoning": 0.82, "codegen": 0.78, "debugging": 0.8, "tool_use": 0.6 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate: reasoning tilt, weak tool-use" },
|
|
33
40
|
"deepseek-reasoner": { "caps": { "reasoning": 0.82, "codegen": 0.78, "debugging": 0.8, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
34
41
|
"deepseek-chat": { "caps": { "reasoning": 0.65, "codegen": 0.7, "debugging": 0.65, "tool_use": 0.62 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
35
42
|
"deepseek-v3*": { "caps": { "reasoning": 0.68, "codegen": 0.72, "debugging": 0.68, "tool_use": 0.62 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate" },
|
|
43
|
+
"deepseek-coder*": { "caps": { "reasoning": 0.5, "codegen": 0.6, "debugging": 0.55, "tool_use": 0.5 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog legacy (MEDIUM/COMPLEX ollama slots: 33b, v2:236b); coder tilt, sized below qwen2.5-coder-32b" },
|
|
44
|
+
"codellama*": { "caps": { "reasoning": 0.45, "codegen": 0.55, "debugging": 0.5, "tool_use": 0.45 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog legacy (MEDIUM ollama slot: 34b); coder tilt" },
|
|
36
45
|
"qwen3-max": { "caps": { "reasoning": 0.78, "codegen": 0.8, "debugging": 0.78, "tool_use": 0.72 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate" },
|
|
37
46
|
"qwen3-235b*": { "caps": { "reasoning": 0.78, "codegen": 0.8, "debugging": 0.78, "tool_use": 0.7 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
38
47
|
"qwen3-32b": { "caps": { "reasoning": 0.65, "codegen": 0.7, "debugging": 0.65, "tool_use": 0.6 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
@@ -40,8 +49,18 @@
|
|
|
40
49
|
"qwen2.5-coder-32b*": { "caps": { "reasoning": 0.55, "codegen": 0.68, "debugging": 0.6, "tool_use": 0.55 }, "sources": ["livecodebench"], "note": "curated-estimate: coder tilt" },
|
|
41
50
|
"qwen2.5-coder-14b*": { "caps": { "reasoning": 0.45, "codegen": 0.58, "debugging": 0.5, "tool_use": 0.45 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
42
51
|
"qwen2.5-coder-7b*": { "caps": { "reasoning": 0.35, "codegen": 0.48, "debugging": 0.4, "tool_use": 0.38 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
52
|
+
"qwen2.5-72b*": { "caps": { "reasoning": 0.55, "codegen": 0.62, "debugging": 0.55, "tool_use": 0.52 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (COMPLEX ollama slot); sized at llama-3.1-70b rung" },
|
|
53
|
+
"qwen2.5-32b*": { "caps": { "reasoning": 0.5, "codegen": 0.6, "debugging": 0.55, "tool_use": 0.5 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (MEDIUM ollama slot); sized just below qwen2.5-coder-32b" },
|
|
54
|
+
"qwen2.5-14b*": { "caps": { "reasoning": 0.42, "codegen": 0.55, "debugging": 0.48, "tool_use": 0.42 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: sized between qwen2.5-7b and qwen2.5-32b rungs" },
|
|
55
|
+
"qwen2.5-7b*": { "caps": { "reasoning": 0.33, "codegen": 0.45, "debugging": 0.38, "tool_use": 0.36 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (SIMPLE ollama slot); sized just below qwen2.5-coder-7b" },
|
|
56
|
+
"qwen2.5-3b*": { "caps": { "reasoning": 0.28, "codegen": 0.35, "debugging": 0.3, "tool_use": 0.3 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: local runtime (ollama qwen2.5:3b); small-ladder rung" },
|
|
57
|
+
"qwen2.5-1.5b*": { "caps": { "reasoning": 0.22, "codegen": 0.28, "debugging": 0.24, "tool_use": 0.24 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: local runtime (ollama qwen2.5:1.5b); tiny-ladder rung" },
|
|
58
|
+
"qwen2.5-0.5b*": { "caps": { "reasoning": 0.15, "codegen": 0.2, "debugging": 0.18, "tool_use": 0.18 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: local runtime (ollama qwen2.5:0.5b); floor rung, trivial turns only" },
|
|
43
59
|
"qwen3-8b": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.45 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
44
60
|
"qwen3-4b": { "caps": { "reasoning": 0.35, "codegen": 0.42, "debugging": 0.38, "tool_use": 0.38 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
61
|
+
"qwen3.5-9b*": { "caps": { "reasoning": 0.42, "codegen": 0.5, "debugging": 0.44, "tool_use": 0.42 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: Qwen3.5 9B class (ornith arch family); sized just below qwen3-8b+ — promote on benchmark rows" },
|
|
62
|
+
"qwen3.5-4b*": { "caps": { "reasoning": 0.35, "codegen": 0.42, "debugging": 0.38, "tool_use": 0.38 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: Qwen3.5 4B class; mirrors qwen3-4b — promote on benchmark rows" },
|
|
63
|
+
"qwen3.5-2b*": { "caps": { "reasoning": 0.26, "codegen": 0.3, "debugging": 0.27, "tool_use": 0.28 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: Qwen3.5 2B class; tiny/small rung" },
|
|
45
64
|
"glm-5.2": { "caps": { "reasoning": 0.68, "codegen": 0.72, "debugging": 0.68, "tool_use": 0.66 }, "sources": ["swe-bench-verified", "artificial-analysis"], "note": "curated-estimate: provider-agnostic (z.ai/Baidu/local share this entry)" },
|
|
46
65
|
"glm-5*": { "caps": { "reasoning": 0.65, "codegen": 0.68, "debugging": 0.65, "tool_use": 0.62 }, "sources": ["artificial-analysis"], "note": "curated-estimate: GLM-5 family default" },
|
|
47
66
|
"glm-4.7": { "caps": { "reasoning": 0.62, "codegen": 0.66, "debugging": 0.62, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
@@ -54,20 +73,28 @@
|
|
|
54
73
|
"kimi-k2*": { "caps": { "reasoning": 0.6, "codegen": 0.63, "debugging": 0.6, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: Kimi-K2 family default" },
|
|
55
74
|
"llama-3.3-70b*": { "caps": { "reasoning": 0.6, "codegen": 0.65, "debugging": 0.6, "tool_use": 0.58 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate" },
|
|
56
75
|
"llama-3.1-70b*": { "caps": { "reasoning": 0.55, "codegen": 0.6, "debugging": 0.55, "tool_use": 0.52 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
76
|
+
"llama-3.1-8b*": { "caps": { "reasoning": 0.35, "codegen": 0.4, "debugging": 0.35, "tool_use": 0.36 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: tier-catalog entry (fireworks llama-3.1-8b-instruct); small rung" },
|
|
57
77
|
"llama-4-maverick*": { "caps": { "reasoning": 0.6, "codegen": 0.62, "debugging": 0.6, "tool_use": 0.58 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
58
78
|
"llama-4-scout*": { "caps": { "reasoning": 0.5, "codegen": 0.52, "debugging": 0.5, "tool_use": 0.5 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
59
79
|
"llama-3.2*": { "caps": { "reasoning": 0.25, "codegen": 0.3, "debugging": 0.25, "tool_use": 0.28 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
60
80
|
"mistral-large*": { "caps": { "reasoning": 0.6, "codegen": 0.65, "debugging": 0.6, "tool_use": 0.62 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
61
81
|
"codestral*": { "caps": { "reasoning": 0.5, "codegen": 0.62, "debugging": 0.55, "tool_use": 0.5 }, "sources": ["livecodebench"], "note": "curated-estimate: coder tilt" },
|
|
62
82
|
"mistral-small*": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.48 }, "sources": ["artificial-analysis"], "note": "curated-estimate" },
|
|
83
|
+
"mistral*": { "caps": { "reasoning": 0.35, "codegen": 0.4, "debugging": 0.35, "tool_use": 0.36 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: bare/7b-class mistral (ollama SIMPLE slot); longer keys (mistral-large/small) still win via longest-prefix" },
|
|
63
84
|
"devstral*": { "caps": { "reasoning": 0.5, "codegen": 0.6, "debugging": 0.55, "tool_use": 0.5 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: coder tilt" },
|
|
64
85
|
"gpt-oss-120b": { "caps": { "reasoning": 0.65, "codegen": 0.68, "debugging": 0.65, "tool_use": 0.55 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
65
86
|
"gpt-oss-20b": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.42 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
66
87
|
"gemma-3-27b": { "caps": { "reasoning": 0.45, "codegen": 0.48, "debugging": 0.45, "tool_use": 0.45 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
88
|
+
"gemma4*": { "caps": { "reasoning": 0.42, "codegen": 0.46, "debugging": 0.42, "tool_use": 0.42 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: local runtime (ollama gemma4:12b normalizes to gemma4-12b, no dash); sized just below gemma-3-27b" },
|
|
67
89
|
"gemma-2*": { "caps": { "reasoning": 0.28, "codegen": 0.32, "debugging": 0.28, "tool_use": 0.3 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
68
90
|
"phi-4": { "caps": { "reasoning": 0.45, "codegen": 0.48, "debugging": 0.45, "tool_use": 0.4 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
69
91
|
"phi-3*": { "caps": { "reasoning": 0.28, "codegen": 0.32, "debugging": 0.28, "tool_use": 0.28 }, "sources": ["livecodebench"], "note": "curated-estimate" },
|
|
70
92
|
"minimax-m2*": { "caps": { "reasoning": 0.65, "codegen": 0.68, "debugging": 0.65, "tool_use": 0.65 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" },
|
|
93
|
+
"minimax-m3*": { "caps": { "reasoning": 0.65, "codegen": 0.68, "debugging": 0.65, "tool_use": 0.65 }, "sources": ["lynkr-operator-estimate"], "note": "provisional: local runtime (OLLAMA_MODEL minimax-m3:cloud); mirrors minimax-m2 pending m3 rows" },
|
|
94
|
+
"ornith*": { "caps": { "reasoning": 0.44, "codegen": 0.52, "debugging": 0.46, "tool_use": 0.47 }, "sources": ["ollama-library", "lynkr-operator-estimate"], "note": "provisional: 9B Qwen3.5-based agentic coder (tools+thinking, 262k ctx); no public SWE-bench row — coder/agentic tilt, revisit on data" },
|
|
95
|
+
"spark-x2.5-4b*": { "caps": { "reasoning": 0.42, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.48 }, "sources": ["vendor-reported", "lynkr-operator-estimate"], "note": "provisional: vendor claims SOTA-among-size on agent/code; capped per self-report policy (cf. muse-spark-1.3) — revisit on independent rows" },
|
|
96
|
+
"spark-x2.5-1.7b*": { "caps": { "reasoning": 0.32, "codegen": 0.38, "debugging": 0.34, "tool_use": 0.36 }, "sources": ["vendor-reported", "lynkr-operator-estimate"], "note": "provisional: 1.7B sibling; small rung with agent tilt — revisit on independent rows" },
|
|
97
|
+
"spark-x2.5*": { "caps": { "reasoning": 0.35, "codegen": 0.42, "debugging": 0.38, "tool_use": 0.4 }, "sources": ["lynkr-operator-estimate"], "note": "family default; dated/suffixed forms fall here, longer 4b/1.7b keys win via longest-prefix" },
|
|
71
98
|
"muse-spark-1.3": { "caps": { "reasoning": 0.85, "codegen": 0.87, "debugging": 0.85, "tool_use": 0.75 }, "sources": ["deepswe-1.1@75.4% (vendor-reported, max setting)", "terminal-bench-2.1@88.8% (vendor-reported)"], "note": "curated-estimate: full 1.3 at max reasoning. Vendor self-reported; absent from public DeepSWE leaderboard (only 1.2 listed); hands-on gap reported by third parties. tool_use capped — no direct tool-use data. Revisit on independent rows." },
|
|
72
99
|
"muse-spark-1.2*": { "caps": { "reasoning": 0.42, "codegen": 0.46, "debugging": 0.42, "tool_use": 0.44 }, "sources": ["lynkr-telemetry:4176 scored rows, avg quality 52.7 @ complexity 51.7"], "note": "measured: operator's own traffic. Below the 70 evidence bar — economize trivial turns only." },
|
|
73
100
|
"muse-spark-1.3-contributor*": { "caps": { "reasoning": 0.4, "codegen": 0.45, "debugging": 0.4, "tool_use": 0.45 }, "sources": ["lynkr-telemetry:29 scored rows (thin) + 1.2 family history"], "note": "provisional: free/contributor serving grades far below full-1.3 published scores (variant, effort setting, or benchmark inflation — undetermined). Do NOT cover with muse-spark*: the full model and the free serving must never share caps." }
|
package/package.json
CHANGED
|
@@ -8,12 +8,21 @@ const FALLBACK_HEADERS = [
|
|
|
8
8
|
"x-claude-session",
|
|
9
9
|
"x-claude-conversation-id",
|
|
10
10
|
"anthropic-session-id",
|
|
11
|
+
// open-code-review per-bundle affinity: internal/llm/sessionkey.go sends
|
|
12
|
+
// `x-session-affinity: <ocr_session_key>` (real task key, or an
|
|
13
|
+
// auto-generated fallback). Each file bundle gets its own key so bundles
|
|
14
|
+
// route independently instead of sharing one PR-wide pin.
|
|
15
|
+
"x-session-affinity",
|
|
11
16
|
];
|
|
12
17
|
|
|
13
18
|
function normaliseSessionId(value) {
|
|
14
19
|
if (typeof value !== "string") return null;
|
|
15
20
|
const trimmed = value.trim();
|
|
16
|
-
|
|
21
|
+
if (!trimmed.length) return null;
|
|
22
|
+
// Ignore unsubstituted templates (OCR sends literal "{ocr_session_key}"
|
|
23
|
+
// when no key is configured — its own client falls back to auto-generated).
|
|
24
|
+
if (trimmed.includes("{") && trimmed.includes("}")) return null;
|
|
25
|
+
return trimmed;
|
|
17
26
|
}
|
|
18
27
|
|
|
19
28
|
function _contentText(content) {
|
package/src/cache/prompt.js
CHANGED
|
@@ -37,6 +37,45 @@ function stableStringify(value) {
|
|
|
37
37
|
return JSON.stringify(normaliseObject(value));
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Replace image bytes with short hashes for cache keys.
|
|
42
|
+
* Prevents multi-MB base64 from entering SQLite keys/values index path and
|
|
43
|
+
* gives stable keys for identical images without hashing megabytes per request.
|
|
44
|
+
*/
|
|
45
|
+
function stripImageBytesForKey(messages) {
|
|
46
|
+
if (!Array.isArray(messages)) return messages;
|
|
47
|
+
let hashImageData = null;
|
|
48
|
+
try {
|
|
49
|
+
hashImageData = require('../routing/vision').hashImageData;
|
|
50
|
+
} catch { hashImageData = null; }
|
|
51
|
+
const shortHash = (s) => {
|
|
52
|
+
if (!hashImageData || typeof s !== 'string') return `[image:${String(s ?? '').length}]`;
|
|
53
|
+
try { return `[image:${hashImageData(s)}]`; } catch { return `[image:${String(s).length}]`; }
|
|
54
|
+
};
|
|
55
|
+
return messages.map((msg) => {
|
|
56
|
+
if (!msg || !Array.isArray(msg.content)) return msg;
|
|
57
|
+
return {
|
|
58
|
+
...msg,
|
|
59
|
+
content: msg.content.map((block) => {
|
|
60
|
+
if (!block || typeof block !== 'object') return block;
|
|
61
|
+
if (block.type === 'image' && typeof block.source?.data === 'string') {
|
|
62
|
+
return { ...block, source: { ...block.source, data: shortHash(block.source.data) } };
|
|
63
|
+
}
|
|
64
|
+
if (typeof block.image_url?.url === 'string' && block.image_url.url.startsWith('data:image')) {
|
|
65
|
+
return { ...block, image_url: { ...block.image_url, url: shortHash(block.image_url.url) } };
|
|
66
|
+
}
|
|
67
|
+
if (typeof block.inlineData?.data === 'string') {
|
|
68
|
+
return { ...block, inlineData: { ...block.inlineData, data: shortHash(block.inlineData.data) } };
|
|
69
|
+
}
|
|
70
|
+
if (block.type === 'tool_result' && Array.isArray(block.content)) {
|
|
71
|
+
return { ...block, content: stripImageBytesForKey([{ content: block.content }])[0].content };
|
|
72
|
+
}
|
|
73
|
+
return block;
|
|
74
|
+
}),
|
|
75
|
+
};
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
40
79
|
class PromptCache {
|
|
41
80
|
constructor(options = {}) {
|
|
42
81
|
this.enabled = options.enabled === true;
|
|
@@ -158,7 +197,7 @@ class PromptCache {
|
|
|
158
197
|
const canonical = {
|
|
159
198
|
model: payload.model ?? null,
|
|
160
199
|
input: payload.input ?? null,
|
|
161
|
-
messages: payload.messages ? normaliseObject(payload.messages) : null,
|
|
200
|
+
messages: payload.messages ? normaliseObject(stripImageBytesForKey(payload.messages)) : null,
|
|
162
201
|
tools: payload.tools ? normaliseObject(payload.tools) : null,
|
|
163
202
|
tool_choice: payload.tool_choice ? normaliseObject(payload.tool_choice) : null,
|
|
164
203
|
temperature: payload.temperature ?? null,
|
package/src/cache/semantic.js
CHANGED
|
@@ -162,6 +162,13 @@ class SemanticCache {
|
|
|
162
162
|
}
|
|
163
163
|
return count;
|
|
164
164
|
}, 0),
|
|
165
|
+
// Vision hash: same text + different image must NOT hit. Null when no
|
|
166
|
+
// images (backwards compatible with existing entries).
|
|
167
|
+
visionHash: (() => {
|
|
168
|
+
try {
|
|
169
|
+
return require('../routing/vision').visionContextHash(messages);
|
|
170
|
+
} catch { return null; }
|
|
171
|
+
})(),
|
|
165
172
|
};
|
|
166
173
|
|
|
167
174
|
hash.update(JSON.stringify(conversationState));
|
|
@@ -1810,6 +1810,62 @@ async function invokeLMStudio(body, _incomingHeaders = {}) {
|
|
|
1810
1810
|
return performJsonRequest(endpoint, { headers, body: lmstudioBody }, "LM Studio");
|
|
1811
1811
|
}
|
|
1812
1812
|
|
|
1813
|
+
/**
|
|
1814
|
+
* Map a single Anthropic content block to Bedrock Converse content.
|
|
1815
|
+
* Text -> {text}; image base64 -> {image:{format, source:{bytes}}};
|
|
1816
|
+
* image URL / unsupported -> text placeholder (Converse has no URL fetch).
|
|
1817
|
+
* Returns null for cache_control-only / empty blocks.
|
|
1818
|
+
*
|
|
1819
|
+
* @param {object|string} block
|
|
1820
|
+
* @returns {object|null}
|
|
1821
|
+
*/
|
|
1822
|
+
function anthropicBlockToConverseContent(block) {
|
|
1823
|
+
if (block == null) return null;
|
|
1824
|
+
if (typeof block === 'string') return block ? { text: block } : null;
|
|
1825
|
+
if (block.type === 'text' && typeof block.text === 'string') {
|
|
1826
|
+
return block.text ? { text: block.text } : null;
|
|
1827
|
+
}
|
|
1828
|
+
if (block.type === 'image' && block.source) {
|
|
1829
|
+
const src = block.source;
|
|
1830
|
+
if (src.type === 'base64' && src.data) {
|
|
1831
|
+
const mediaType = (src.media_type || 'image/jpeg').toLowerCase();
|
|
1832
|
+
const format = mediaType.includes('png') ? 'png'
|
|
1833
|
+
: mediaType.includes('gif') ? 'gif'
|
|
1834
|
+
: mediaType.includes('webp') ? 'webp'
|
|
1835
|
+
: 'jpeg';
|
|
1836
|
+
try {
|
|
1837
|
+
return { image: { format, source: { bytes: src.data } } };
|
|
1838
|
+
} catch { return null; }
|
|
1839
|
+
}
|
|
1840
|
+
if (src.type === 'url' && src.url) {
|
|
1841
|
+
return { text: `[Image URL: ${src.url}]` };
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
// OpenAI-shaped image_url (may already be converted upstream)
|
|
1845
|
+
const imageUrl = block.image_url?.url || (typeof block.url === 'string' ? block.url : null);
|
|
1846
|
+
if ((block.type === 'image_url' || block.type === 'input_image' || imageUrl) && imageUrl) {
|
|
1847
|
+
const dataMatch = String(imageUrl).match(/^data:(image\/[^;]+);base64,(.+)$/);
|
|
1848
|
+
if (dataMatch) {
|
|
1849
|
+
const mediaType = dataMatch[1].toLowerCase();
|
|
1850
|
+
const format = mediaType.includes('png') ? 'png'
|
|
1851
|
+
: mediaType.includes('gif') ? 'gif'
|
|
1852
|
+
: mediaType.includes('webp') ? 'webp'
|
|
1853
|
+
: 'jpeg';
|
|
1854
|
+
return { image: { format, source: { bytes: dataMatch[2] } } };
|
|
1855
|
+
}
|
|
1856
|
+
return { text: `[Image URL: ${imageUrl}]` };
|
|
1857
|
+
}
|
|
1858
|
+
if (block.type === 'tool_use' || block.type === 'tool_result' || block.type === 'thinking') {
|
|
1859
|
+
// Tool/thinking blocks are handled by the toolConfig path or dropped for
|
|
1860
|
+
// pure-text turns — preserve text content when present.
|
|
1861
|
+
const text = block.text || (typeof block.content === 'string' ? block.content : null);
|
|
1862
|
+
return text ? { text } : null;
|
|
1863
|
+
}
|
|
1864
|
+
if (typeof block.text === 'string' && block.text) return { text: block.text };
|
|
1865
|
+
if (typeof block.content === 'string' && block.content) return { text: block.content };
|
|
1866
|
+
return null;
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1813
1869
|
/**
|
|
1814
1870
|
* Flatten an Anthropic-style content value into a plain string for the
|
|
1815
1871
|
* Bedrock Converse API.
|
|
@@ -1919,9 +1975,10 @@ async function invokeBedrock(body, _incomingHeaders = {}) {
|
|
|
1919
1975
|
.map(msg => ({
|
|
1920
1976
|
role: msg.role,
|
|
1921
1977
|
content: Array.isArray(msg.content)
|
|
1922
|
-
? msg.content.map(c => (
|
|
1923
|
-
: [{ text: msg.content }]
|
|
1978
|
+
? msg.content.map(c => anthropicBlockToConverseContent(c)).filter(Boolean)
|
|
1979
|
+
: [{ text: String(msg.content ?? '') }]
|
|
1924
1980
|
}))
|
|
1981
|
+
.map(m => ({ ...m, content: m.content.length > 0 ? m.content : [{ text: ' ' }] }))
|
|
1925
1982
|
};
|
|
1926
1983
|
|
|
1927
1984
|
// Add system prompt (from Anthropic system field OR extracted from messages)
|
|
@@ -2997,6 +3054,33 @@ IMPORTANT TOOL USAGE RULES:
|
|
|
2997
3054
|
for (const block of msg.content) {
|
|
2998
3055
|
if (block.type === "text") {
|
|
2999
3056
|
parts.push({ text: block.text });
|
|
3057
|
+
} else if (block.type === "image" && block.source?.type === "base64" && block.source?.data) {
|
|
3058
|
+
parts.push({
|
|
3059
|
+
inlineData: {
|
|
3060
|
+
mimeType: block.source.media_type || "image/jpeg",
|
|
3061
|
+
data: block.source.data,
|
|
3062
|
+
},
|
|
3063
|
+
});
|
|
3064
|
+
} else if (block.type === "image" && block.source?.type === "url" && block.source?.url) {
|
|
3065
|
+
parts.push({ text: `[Image URL: ${block.source.url}]` });
|
|
3066
|
+
} else if ((block.type === "image_url" || block.type === "input_image") && block.image_url?.url) {
|
|
3067
|
+
const url = block.image_url.url;
|
|
3068
|
+
const dataMatch = String(url).match(/^data:(image\/[^;]+);base64,(.+)$/);
|
|
3069
|
+
if (dataMatch) {
|
|
3070
|
+
parts.push({ inlineData: { mimeType: dataMatch[1], data: dataMatch[2] } });
|
|
3071
|
+
} else {
|
|
3072
|
+
parts.push({ text: `[Image URL: ${url}]` });
|
|
3073
|
+
}
|
|
3074
|
+
} else if (block.image_url?.url) {
|
|
3075
|
+
const url = block.image_url.url;
|
|
3076
|
+
const dataMatch = String(url).match(/^data:(image\/[^;]+);base64,(.+)$/);
|
|
3077
|
+
if (dataMatch) {
|
|
3078
|
+
parts.push({ inlineData: { mimeType: dataMatch[1], data: dataMatch[2] } });
|
|
3079
|
+
} else {
|
|
3080
|
+
parts.push({ text: `[Image URL: ${url}]` });
|
|
3081
|
+
}
|
|
3082
|
+
} else if (block.inlineData?.data) {
|
|
3083
|
+
parts.push({ inlineData: block.inlineData });
|
|
3000
3084
|
} else if (block.type === "tool_use") {
|
|
3001
3085
|
// Assistant's tool call
|
|
3002
3086
|
parts.push({
|
|
@@ -55,6 +55,25 @@ function convertAnthropicToolsToOpenRouter(anthropicTools) {
|
|
|
55
55
|
* - User messages with tool_result blocks → OpenRouter tool role messages
|
|
56
56
|
* - Regular text content → OpenRouter text content
|
|
57
57
|
*/
|
|
58
|
+
function anthropicImageToOpenAIUrl(block) {
|
|
59
|
+
if (!block || typeof block !== 'object') return null;
|
|
60
|
+
// Anthropic: {type:'image', source:{type:'base64', media_type, data}}
|
|
61
|
+
if (block.type === 'image' && block.source?.type === 'base64' && block.source?.data) {
|
|
62
|
+
const mediaType = block.source.media_type || 'image/jpeg';
|
|
63
|
+
return `data:${mediaType};base64,${block.source.data}`;
|
|
64
|
+
}
|
|
65
|
+
if (block.type === 'image' && block.source?.type === 'url' && block.source?.url) {
|
|
66
|
+
return block.source.url;
|
|
67
|
+
}
|
|
68
|
+
// Already OpenAI-shaped
|
|
69
|
+
if ((block.type === 'image_url' || block.type === 'input_image') && block.image_url?.url) {
|
|
70
|
+
return block.image_url.url;
|
|
71
|
+
}
|
|
72
|
+
if (block.image_url?.url) return block.image_url.url;
|
|
73
|
+
if (typeof block.url === 'string' && block.url.startsWith('data:image')) return block.url;
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
|
|
58
77
|
function convertAnthropicMessagesToOpenRouter(anthropicMessages) {
|
|
59
78
|
if (!Array.isArray(anthropicMessages)) return [];
|
|
60
79
|
|
|
@@ -124,32 +143,80 @@ function convertAnthropicMessagesToOpenRouter(anthropicMessages) {
|
|
|
124
143
|
// providers 400 the unpaired function_call.
|
|
125
144
|
else if (msg.role === 'user' && toolResultBlocks.length > 0) {
|
|
126
145
|
// Tool results first — adjacent to their assistant tool_calls.
|
|
146
|
+
// Preserve nested screenshots: emit text to the tool message and
|
|
147
|
+
// re-attach images as a following user message with image_url parts.
|
|
148
|
+
const pendingImages = [];
|
|
127
149
|
for (const toolResult of toolResultBlocks) {
|
|
150
|
+
let toolText = '';
|
|
151
|
+
const trContent = toolResult.content;
|
|
152
|
+
if (typeof trContent === 'string') {
|
|
153
|
+
toolText = trContent;
|
|
154
|
+
} else if (Array.isArray(trContent)) {
|
|
155
|
+
const texts = [];
|
|
156
|
+
for (const b of trContent) {
|
|
157
|
+
if (b?.type === 'text' && b.text) texts.push(b.text);
|
|
158
|
+
else {
|
|
159
|
+
const url = anthropicImageToOpenAIUrl(b);
|
|
160
|
+
if (url) pendingImages.push(url);
|
|
161
|
+
else if (b) texts.push(typeof b === 'string' ? b : JSON.stringify(b));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
toolText = texts.join('\n');
|
|
165
|
+
} else {
|
|
166
|
+
try { toolText = JSON.stringify(trContent || {}); } catch { toolText = ''; }
|
|
167
|
+
}
|
|
128
168
|
converted.push({
|
|
129
169
|
role: 'tool',
|
|
130
170
|
tool_call_id: toolResult.tool_use_id || `call_${Date.now()}`,
|
|
131
|
-
content:
|
|
132
|
-
? toolResult.content
|
|
133
|
-
: JSON.stringify(toolResult.content || {})
|
|
171
|
+
content: toolText
|
|
134
172
|
});
|
|
135
173
|
}
|
|
136
174
|
|
|
137
|
-
// Then any user-authored/injected text as a user message.
|
|
175
|
+
// Then any user-authored/injected text + images as a user message.
|
|
138
176
|
const textContent = textBlocks.map(block => block.text || '').join('\n');
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
177
|
+
for (const b of content) {
|
|
178
|
+
const url = anthropicImageToOpenAIUrl(b);
|
|
179
|
+
if (url) pendingImages.push(url);
|
|
180
|
+
}
|
|
181
|
+
if (textContent || pendingImages.length > 0) {
|
|
182
|
+
if (pendingImages.length === 0) {
|
|
183
|
+
converted.push({ role: 'user', content: textContent });
|
|
184
|
+
} else {
|
|
185
|
+
const parts = [];
|
|
186
|
+
if (textContent) parts.push({ type: 'text', text: textContent });
|
|
187
|
+
for (const url of pendingImages) {
|
|
188
|
+
parts.push({ type: 'image_url', image_url: { url } });
|
|
189
|
+
}
|
|
190
|
+
converted.push({ role: 'user', content: parts });
|
|
191
|
+
}
|
|
144
192
|
}
|
|
145
193
|
}
|
|
146
|
-
// Regular message
|
|
194
|
+
// Regular message (may carry text + images)
|
|
147
195
|
else {
|
|
148
196
|
const textContent = textBlocks.map(block => block.text || '').join('\n');
|
|
149
|
-
const
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
197
|
+
const imageUrls = [];
|
|
198
|
+
for (const b of content) {
|
|
199
|
+
const url = anthropicImageToOpenAIUrl(b);
|
|
200
|
+
if (url) imageUrls.push(url);
|
|
201
|
+
}
|
|
202
|
+
// Document blocks with embedded base64 images
|
|
203
|
+
for (const b of content) {
|
|
204
|
+
if (b?.type === 'document' && b.source?.data && imageUrls.length === 0) {
|
|
205
|
+
// PDFs aren't image_url-compatible — keep a text placeholder so the
|
|
206
|
+
// turn isn't silently emptied (becomes ' ' downstream if nothing else).
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
let message;
|
|
210
|
+
if (imageUrls.length === 0) {
|
|
211
|
+
message = { role: msg.role, content: textContent || '' };
|
|
212
|
+
} else {
|
|
213
|
+
const parts = [];
|
|
214
|
+
if (textContent) parts.push({ type: 'text', text: textContent });
|
|
215
|
+
for (const url of imageUrls) {
|
|
216
|
+
parts.push({ type: 'image_url', image_url: { url } });
|
|
217
|
+
}
|
|
218
|
+
message = { role: msg.role, content: parts };
|
|
219
|
+
}
|
|
153
220
|
if (msg.role === 'assistant' && thinkingText) {
|
|
154
221
|
message.reasoning_content = thinkingText;
|
|
155
222
|
}
|
|
@@ -188,6 +255,9 @@ function convertAnthropicMessagesToOpenRouter(anthropicMessages) {
|
|
|
188
255
|
if (m.role === 'tool') continue;
|
|
189
256
|
const hasToolCalls = Array.isArray(m.tool_calls) && m.tool_calls.length > 0;
|
|
190
257
|
if (hasToolCalls) continue;
|
|
258
|
+
// Array content (text + image_url parts) must be preserved verbatim —
|
|
259
|
+
// only string content gets the empty-string guard.
|
|
260
|
+
if (Array.isArray(m.content)) continue;
|
|
191
261
|
if (typeof m.content !== 'string' || m.content.trim() === '') {
|
|
192
262
|
m.content = ' ';
|
|
193
263
|
}
|
|
@@ -127,6 +127,12 @@ function hashAndTruncate(content, maxLength, deduplicator) {
|
|
|
127
127
|
return { hash: null, content: null, truncated: false, originalLength: null };
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
+
// Redact image bytes before hashing/storing — never persist base64 to disk.
|
|
131
|
+
try {
|
|
132
|
+
const { redactVisionBytes } = require('../routing/vision');
|
|
133
|
+
content = redactVisionBytes(content);
|
|
134
|
+
} catch { /* redaction is best-effort */ }
|
|
135
|
+
|
|
130
136
|
// Hash the ORIGINAL content before any truncation
|
|
131
137
|
const hash = deduplicator ? deduplicator.hashContent(content) : null;
|
|
132
138
|
|
|
@@ -244,6 +244,11 @@ class ContentDeduplicator {
|
|
|
244
244
|
return null;
|
|
245
245
|
}
|
|
246
246
|
|
|
247
|
+
// Redact image bytes before anything touches disk.
|
|
248
|
+
try {
|
|
249
|
+
content = require('../routing/vision').redactVisionBytes(content);
|
|
250
|
+
} catch { /* best-effort */ }
|
|
251
|
+
|
|
247
252
|
let stringContent = typeof content === "string" ? content : JSON.stringify(content);
|
|
248
253
|
|
|
249
254
|
// Sanitize content before hashing (if enabled)
|
|
@@ -292,6 +297,11 @@ class ContentDeduplicator {
|
|
|
292
297
|
return null;
|
|
293
298
|
}
|
|
294
299
|
|
|
300
|
+
// Redact image bytes before anything touches disk.
|
|
301
|
+
try {
|
|
302
|
+
content = require('../routing/vision').redactVisionBytes(content);
|
|
303
|
+
} catch { /* best-effort */ }
|
|
304
|
+
|
|
295
305
|
let stringContent = typeof content === "string" ? content : JSON.stringify(content);
|
|
296
306
|
|
|
297
307
|
// Sanitize content (if enabled)
|
|
@@ -948,7 +948,14 @@ function sanitizePayload(payload) {
|
|
|
948
948
|
|| config.modelProvider?.type
|
|
949
949
|
|| "databricks";
|
|
950
950
|
const willFlatten = providerType !== "azure-anthropic";
|
|
951
|
-
|
|
951
|
+
// Vision payloads must keep image bytes through the clone: flatten-aware
|
|
952
|
+
// cloning replaces them with {_skipped:true} placeholders that downstream
|
|
953
|
+
// converters can no longer recover. Full-clone when images are present.
|
|
954
|
+
let needsVisionForClone = false;
|
|
955
|
+
try {
|
|
956
|
+
needsVisionForClone = require("../routing/vision").payloadNeedsVision(payload);
|
|
957
|
+
} catch { needsVisionForClone = false; }
|
|
958
|
+
const clean = clonePayloadSmart(payload ?? {}, { willFlatten: willFlatten && !needsVisionForClone });
|
|
952
959
|
const requestedModel =
|
|
953
960
|
(typeof payload?.model === "string" && payload.model.trim().length > 0
|
|
954
961
|
? payload.model.trim()
|
|
@@ -1031,17 +1038,28 @@ function sanitizePayload(payload) {
|
|
|
1031
1038
|
const { modelNameSupportsTools } = require("../clients/ollama-utils");
|
|
1032
1039
|
const modelSupportsTools = modelNameSupportsTools(config.ollama?.model);
|
|
1033
1040
|
|
|
1041
|
+
// Preserve image blocks for vision-capable Ollama models (e.g. llava,
|
|
1042
|
+
// llama3.2-vision): the native /v1/messages path forwards them verbatim,
|
|
1043
|
+
// the legacy /api/chat path drops what it can't handle. Text-only
|
|
1044
|
+
// filtering below must not blind the native path.
|
|
1045
|
+
const msgHasVision = (msg) => Array.isArray(msg?.content) && msg.content.some((b) =>
|
|
1046
|
+
b?.type === 'image' || b?.type === 'image_url' || b?.image_url?.url || b?.source?.data
|
|
1047
|
+
);
|
|
1034
1048
|
if (!modelSupportsTools) {
|
|
1035
1049
|
// Filter out tool_result content blocks for models without tool support
|
|
1050
|
+
// (but keep text + images so vision still works).
|
|
1036
1051
|
clean.messages = clean.messages
|
|
1037
1052
|
.map((msg) => {
|
|
1038
1053
|
if (Array.isArray(msg.content)) {
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
(block) => block.type === "text" && block.text
|
|
1054
|
+
const keepBlocks = msg.content.filter(
|
|
1055
|
+
(block) => (block.type === "text" && block.text) || msgHasVision({ content: [block] })
|
|
1042
1056
|
);
|
|
1043
|
-
if (
|
|
1044
|
-
|
|
1057
|
+
if (keepBlocks.length > 0) {
|
|
1058
|
+
const textBlocks = keepBlocks.filter((b) => b.type === "text" && b.text);
|
|
1059
|
+
// If only images remain, keep array form for the native path
|
|
1060
|
+
if (textBlocks.length === 0) return { role: msg.role, content: keepBlocks };
|
|
1061
|
+
// Mixed text+images: keep array form (native) — legacy drops images
|
|
1062
|
+
if (keepBlocks.some((b) => b.type !== "text")) return { role: msg.role, content: keepBlocks };
|
|
1045
1063
|
return {
|
|
1046
1064
|
role: msg.role,
|
|
1047
1065
|
content: textBlocks.map((b) => b.text).join("\n"),
|
|
@@ -1054,9 +1072,10 @@ function sanitizePayload(payload) {
|
|
|
1054
1072
|
.filter(Boolean);
|
|
1055
1073
|
} else {
|
|
1056
1074
|
// Keep tool blocks for tool-capable models
|
|
1057
|
-
//
|
|
1075
|
+
// Preserve image arrays; flatten pure-text arrays for compatibility.
|
|
1058
1076
|
clean.messages = clean.messages.map((msg) => {
|
|
1059
1077
|
if (Array.isArray(msg.content)) {
|
|
1078
|
+
if (msgHasVision(msg)) return msg;
|
|
1060
1079
|
const textBlocks = msg.content.filter(
|
|
1061
1080
|
(block) => block.type === "text" && block.text
|
|
1062
1081
|
);
|
|
@@ -158,6 +158,25 @@ const PROFILES = {
|
|
|
158
158
|
minToolFingerprintMatch: 0.8,
|
|
159
159
|
},
|
|
160
160
|
},
|
|
161
|
+
'open-code-review': {
|
|
162
|
+
name: 'open-code-review',
|
|
163
|
+
baselineTools: new Set([
|
|
164
|
+
// Alibaba open-code-review focused review toolset —
|
|
165
|
+
// internal/tool/definitions.go: task_done, code_comment, file_read,
|
|
166
|
+
// file_read_diff (+ search variants). These attach to every review
|
|
167
|
+
// bundle by design; counting them as agentic intent would floor every
|
|
168
|
+
// bundle at ITERATIVE/COMPLEX even for a 10-line typo fix. Real
|
|
169
|
+
// complexity still comes through diff size + file_read chains.
|
|
170
|
+
'task_done', 'code_comment', 'file_read', 'file_read_diff',
|
|
171
|
+
]),
|
|
172
|
+
detect: {
|
|
173
|
+
// UA shape: `open-code-review/<ver> | <provider>` (internal/llm/client.go
|
|
174
|
+
// userAgent). Fingerprint fallback at 0.5: minimal bundles may carry
|
|
175
|
+
// only code_comment + task_done.
|
|
176
|
+
headerPatterns: [/open-code-review/i],
|
|
177
|
+
minToolFingerprintMatch: 0.5,
|
|
178
|
+
},
|
|
179
|
+
},
|
|
161
180
|
};
|
|
162
181
|
|
|
163
182
|
// Union of every baseline tool across all known profiles — used for the
|
package/src/routing/index.js
CHANGED
|
@@ -61,16 +61,23 @@ const LOCAL_PROVIDERS = ['ollama', 'llamacpp', 'lmstudio'];
|
|
|
61
61
|
|
|
62
62
|
/**
|
|
63
63
|
* Returns true when any message content block is an image.
|
|
64
|
-
*
|
|
64
|
+
* Delegates to the central vision detector (covers base64/url/document/
|
|
65
|
+
* tool_result-nested/Gemini inlineData). Kept as a thin wrapper for
|
|
66
|
+
* backwards compatibility with existing call sites and tests.
|
|
65
67
|
*/
|
|
66
68
|
function _payloadHasImages(payload) {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
69
|
+
try {
|
|
70
|
+
const { payloadNeedsVision } = require('./vision');
|
|
71
|
+
return payloadNeedsVision(payload);
|
|
72
|
+
} catch {
|
|
73
|
+
const messages = payload?.messages;
|
|
74
|
+
if (!Array.isArray(messages)) return false;
|
|
75
|
+
return messages.some(msg => {
|
|
76
|
+
const content = msg?.content;
|
|
77
|
+
if (!Array.isArray(content)) return false;
|
|
78
|
+
return content.some(block => block?.type === 'image' || block?.type === 'image_url');
|
|
79
|
+
});
|
|
80
|
+
}
|
|
74
81
|
}
|
|
75
82
|
|
|
76
83
|
/**
|
|
@@ -215,10 +222,8 @@ function _runPinGuards(payload, pin) {
|
|
|
215
222
|
|
|
216
223
|
if (_payloadHasImages(payload)) {
|
|
217
224
|
try {
|
|
218
|
-
const {
|
|
219
|
-
|
|
220
|
-
const modelInfo = pin.model ? registry.getCost(pin.model) : null;
|
|
221
|
-
if (!modelInfo?.vision) {
|
|
225
|
+
const { providerSupportsVision } = require('./vision');
|
|
226
|
+
if (!providerSupportsVision(pin.provider, pin.model)) {
|
|
222
227
|
return { ok: false, reason: 'vision', risk, promptTokensEst };
|
|
223
228
|
}
|
|
224
229
|
} catch (err) {
|
|
@@ -758,6 +763,26 @@ function checkSessionPin(payload, options = {}) {
|
|
|
758
763
|
}
|
|
759
764
|
}
|
|
760
765
|
} catch { /* never block the pin-serve path */ }
|
|
766
|
+
// Vision escape hatch. Same safe pattern as context-overflow above: an
|
|
767
|
+
// image arriving mid-tool-loop must not stay pinned to a text-only model
|
|
768
|
+
// (upstream 400). Serve the pin THIS turn (tool-call IDs aren't portable),
|
|
769
|
+
// drop it so the next boundary re-routes to a vision-capable model.
|
|
770
|
+
try {
|
|
771
|
+
if (_payloadHasImages(payload)) {
|
|
772
|
+
const { providerSupportsVision } = require('./vision');
|
|
773
|
+
if (!providerSupportsVision(pin.provider, pin.model)) {
|
|
774
|
+
sessionAffinity.removePin(sessionId);
|
|
775
|
+
logger.warn({
|
|
776
|
+
sessionId,
|
|
777
|
+
trigger: 'vision_mid_loop',
|
|
778
|
+
pinnedModel: pin.model,
|
|
779
|
+
pinnedProvider: pin.provider,
|
|
780
|
+
pinnedTier: pin.tier,
|
|
781
|
+
}, '[Routing] Image arrived mid-tool-loop on text-only pin — pin dropped, next boundary re-routes to vision-capable model');
|
|
782
|
+
return { serve: true, pin, reason: 'tool_history_pin_dropped', sessionId };
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
} catch { /* never block the pin-serve path */ }
|
|
761
786
|
if (refreshOk) {
|
|
762
787
|
sessionAffinity.setPin(sessionId, pin, {
|
|
763
788
|
// Monotonic within a pin's lifetime: refreshes must never SHRINK the
|
|
@@ -1451,15 +1476,14 @@ async function _determineProviderSmartInner(payload, options = {}) {
|
|
|
1451
1476
|
}
|
|
1452
1477
|
|
|
1453
1478
|
// Phase 1.4 — vision capability guard.
|
|
1454
|
-
// If the payload contains image content blocks but the selected
|
|
1455
|
-
// vision
|
|
1456
|
-
// above the current tier. Prevents silent
|
|
1479
|
+
// If the payload contains image content blocks but the selected
|
|
1480
|
+
// provider/model combo cannot carry vision, swap to the cheapest
|
|
1481
|
+
// vision-capable model at or above the current tier. Prevents silent
|
|
1482
|
+
// upstream failures. Uses per-provider gate, not registry flag alone.
|
|
1457
1483
|
if (_payloadHasImages(payload)) {
|
|
1458
1484
|
try {
|
|
1459
|
-
const {
|
|
1460
|
-
|
|
1461
|
-
const modelInfo = registry.getCost(selectedModel);
|
|
1462
|
-
if (!modelInfo?.vision) {
|
|
1485
|
+
const { providerSupportsVision } = require('./vision');
|
|
1486
|
+
if (!providerSupportsVision(provider, selectedModel)) {
|
|
1463
1487
|
const visionModel = selector.findVisionCapable(tier);
|
|
1464
1488
|
if (visionModel) {
|
|
1465
1489
|
const fromTier = tier;
|
|
@@ -184,24 +184,23 @@ class ModelTierSelector {
|
|
|
184
184
|
|
|
185
185
|
/**
|
|
186
186
|
* Find a vision-capable model at or above `preferredTier`.
|
|
187
|
-
* Walks
|
|
187
|
+
* Walks tiers in priority order (cheapest upward); returns null when none.
|
|
188
|
+
* Uses per-provider gate so registry-true but transportless combos are skipped.
|
|
188
189
|
*/
|
|
189
190
|
findVisionCapable(preferredTier = null) {
|
|
190
|
-
const {
|
|
191
|
-
const
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
191
|
+
const { providerSupportsVision } = require('./vision');
|
|
192
|
+
const priority = { SIMPLE: 1, MEDIUM: 2, COMPLEX: 3, REASONING: 4 };
|
|
193
|
+
const allTiers = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
|
|
194
|
+
const minPriority = preferredTier ? (priority[preferredTier] ?? 1) : 1;
|
|
195
|
+
const tierOrder = allTiers.filter((t) => (priority[t] ?? 99) >= minPriority);
|
|
196
|
+
// Fall back to full sweep (handles unknown tier names gracefully)
|
|
197
|
+
if (preferredTier && !tierOrder.includes(preferredTier)) tierOrder.unshift(preferredTier);
|
|
196
198
|
for (const t of tierOrder) {
|
|
197
|
-
if (seen.has(t)) continue;
|
|
198
|
-
seen.add(t);
|
|
199
199
|
const tierConfig = this.tierConfig[t];
|
|
200
200
|
if (!tierConfig?.preferred) continue;
|
|
201
201
|
for (const [provider, models] of Object.entries(tierConfig.preferred)) {
|
|
202
202
|
for (const model of models) {
|
|
203
|
-
|
|
204
|
-
if (info?.vision) return { provider, model, tier: t };
|
|
203
|
+
if (providerSupportsVision(provider, model)) return { provider, model, tier: t };
|
|
205
204
|
}
|
|
206
205
|
}
|
|
207
206
|
}
|
|
@@ -102,7 +102,19 @@ function analyzeRisk(payload) {
|
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
104
|
text = stripSystemReminders(text);
|
|
105
|
-
|
|
105
|
+
// Harness-aware classification: OCR ships its review ruleset (NPE,
|
|
106
|
+
// thread-safety, XSS, SQL injection) as system text on every bundle. Those
|
|
107
|
+
// rule words are risk keywords ("security", "credential", "secret") the
|
|
108
|
+
// diff never contained — including system would force-escalate every
|
|
109
|
+
// bundle to high/REASONING. Classify on the task (diff bundle), execute
|
|
110
|
+
// on the full request. Fall back to tool fingerprint when the profile
|
|
111
|
+
// hasn't been attached yet (direct analyzeRisk calls in tests/guards).
|
|
112
|
+
const isOCR = payload?._clientProfile?.name === 'open-code-review'
|
|
113
|
+
|| (Array.isArray(payload?.tools) && payload.tools.some((t) => {
|
|
114
|
+
const n = t?.name || t?.function?.name;
|
|
115
|
+
return n === 'code_comment' || n === 'task_done';
|
|
116
|
+
}));
|
|
117
|
+
if (!isOCR && typeof payload?.system === 'string') text += ' ' + payload.system;
|
|
106
118
|
|
|
107
119
|
const prob = _predict(text, model);
|
|
108
120
|
let level;
|
package/src/routing/tokenizer.js
CHANGED
|
@@ -114,14 +114,31 @@ function _extractText(content) {
|
|
|
114
114
|
|
|
115
115
|
function _imageTokenEstimate(content) {
|
|
116
116
|
if (!Array.isArray(content)) return 0;
|
|
117
|
-
let
|
|
117
|
+
let imageCount = 0;
|
|
118
118
|
for (const block of content) {
|
|
119
|
-
if (block
|
|
120
|
-
|
|
119
|
+
if (!block || typeof block !== 'object') continue;
|
|
120
|
+
// Fixed per-image estimate (Anthropic ~1k-1.5k/image, OpenAI tiles similar).
|
|
121
|
+
// Never count base64 bytes as text — 1MB of base64 is not 170k tokens.
|
|
122
|
+
if (block.type === 'image' || block.type === 'image_url' || block.type === 'input_image') {
|
|
123
|
+
imageCount += 1;
|
|
124
|
+
} else if (block.image_url?.url || block.inlineData?.data || block.inline_data?.data) {
|
|
125
|
+
imageCount += 1;
|
|
126
|
+
} else if (block.type === 'tool_result' && block.content) {
|
|
127
|
+
// Nested screenshots inside tool results
|
|
128
|
+
const nested = Array.isArray(block.content) ? block.content : [block.content];
|
|
129
|
+
for (const nb of nested) {
|
|
130
|
+
if (nb && typeof nb === 'object' && (nb.type === 'image' || nb.type === 'image_url' || nb.image_url?.url)) {
|
|
131
|
+
imageCount += 1;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
121
134
|
}
|
|
122
135
|
}
|
|
123
|
-
|
|
124
|
-
|
|
136
|
+
try {
|
|
137
|
+
const { VISION_IMAGE_TOKEN_ESTIMATE } = require('./vision');
|
|
138
|
+
return imageCount * VISION_IMAGE_TOKEN_ESTIMATE;
|
|
139
|
+
} catch {
|
|
140
|
+
return imageCount * 1500;
|
|
141
|
+
}
|
|
125
142
|
}
|
|
126
143
|
|
|
127
144
|
/**
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Central vision detection + capability gating.
|
|
3
|
+
*
|
|
4
|
+
* Single source of truth for "does this payload need a vision-capable model?"
|
|
5
|
+
* Covers Anthropic image blocks, OpenAI image_url parts, documents with
|
|
6
|
+
* embedded images, Gemini inlineData, and nested tool_result screenshots.
|
|
7
|
+
*
|
|
8
|
+
* @module routing/vision
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const crypto = require('crypto');
|
|
12
|
+
|
|
13
|
+
const VISION_IMAGE_TOKEN_ESTIMATE = 1500;
|
|
14
|
+
|
|
15
|
+
// Providers that can never transport image bytes (no vision path, even after
|
|
16
|
+
// converter fixes). Everything else either forwards natively or converts.
|
|
17
|
+
const TRANSPORTLESS_PROVIDERS = new Set(['llamacpp', 'lmstudio', 'codex']);
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Does a single content block carry image data?
|
|
21
|
+
* @param {object|string|null} block
|
|
22
|
+
* @returns {boolean}
|
|
23
|
+
*/
|
|
24
|
+
function blockNeedsVision(block) {
|
|
25
|
+
if (!block || typeof block !== 'object') {
|
|
26
|
+
if (typeof block === 'string') {
|
|
27
|
+
return block.includes('data:image');
|
|
28
|
+
}
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
// Anthropic native: {type:'image', source:{type:'base64'|'url', data|url}}
|
|
32
|
+
if (block.type === 'image') return true;
|
|
33
|
+
// OpenAI native: {type:'image_url', image_url:{url}} or {type:'input_image'}
|
|
34
|
+
if (block.type === 'image_url' || block.type === 'input_image') return true;
|
|
35
|
+
if (block.image_url?.url) return true;
|
|
36
|
+
// Gemini: {inlineData:{mimeType,data}} / {inline_data:{...}} / {fileData}
|
|
37
|
+
if (block.inlineData?.data || block.inline_data?.data || block.fileData) return true;
|
|
38
|
+
// Anthropic document that embeds an image (PDF with scans)
|
|
39
|
+
if (block.type === 'document' && block.source?.data) return true;
|
|
40
|
+
// Generic base64 source marker
|
|
41
|
+
if (block.source?.type === 'base64' && block.source?.data) return true;
|
|
42
|
+
if (typeof block.source?.data === 'string' && block.source.data.length > 100) {
|
|
43
|
+
// Heuristic: large opaque source payloads are media, not text
|
|
44
|
+
if (block.type !== 'text') return true;
|
|
45
|
+
}
|
|
46
|
+
// data: URL embedded in url/text fields
|
|
47
|
+
if (typeof block.url === 'string' && block.url.startsWith('data:image')) return true;
|
|
48
|
+
if (typeof block.text === 'string' && block.text.includes('data:image')) return true;
|
|
49
|
+
// Nested tool_result content: {type:'tool_result', content:[{type:'image'...}]}
|
|
50
|
+
if (block.type === 'tool_result' && block.content) {
|
|
51
|
+
return contentNeedsVision(block.content);
|
|
52
|
+
}
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Does message content (string | array | object) contain image data?
|
|
58
|
+
* @param {*} content
|
|
59
|
+
* @returns {boolean}
|
|
60
|
+
*/
|
|
61
|
+
function contentNeedsVision(content) {
|
|
62
|
+
if (!content) return false;
|
|
63
|
+
if (typeof content === 'string') return content.includes('data:image');
|
|
64
|
+
if (Array.isArray(content)) return content.some(blockNeedsVision);
|
|
65
|
+
if (typeof content === 'object') return blockNeedsVision(content);
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Does the full request payload need a vision-capable model?
|
|
71
|
+
* Scans messages + system for any image signal.
|
|
72
|
+
* @param {object} payload
|
|
73
|
+
* @returns {boolean}
|
|
74
|
+
*/
|
|
75
|
+
function payloadNeedsVision(payload) {
|
|
76
|
+
if (!payload || typeof payload !== 'object') return false;
|
|
77
|
+
const messages = payload.messages;
|
|
78
|
+
if (Array.isArray(messages)) {
|
|
79
|
+
for (const msg of messages) {
|
|
80
|
+
if (!msg) continue;
|
|
81
|
+
if (contentNeedsVision(msg.content)) return true;
|
|
82
|
+
// OpenAI tool_calls with image args (rare) — stringify check, cheap
|
|
83
|
+
if (Array.isArray(msg.tool_calls)) {
|
|
84
|
+
for (const tc of msg.tool_calls) {
|
|
85
|
+
try {
|
|
86
|
+
const args = typeof tc?.function?.arguments === 'string'
|
|
87
|
+
? tc.function.arguments
|
|
88
|
+
: JSON.stringify(tc?.function?.arguments ?? '');
|
|
89
|
+
if (args && args.includes('data:image')) return true;
|
|
90
|
+
} catch { /* ignore */ }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
if (payload.system && contentNeedsVision(payload.system)) return true;
|
|
96
|
+
if (payload.input && contentNeedsVision(payload.input)) return true;
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Count image blocks in a payload (for token budgets + logging).
|
|
102
|
+
* @param {object} payload
|
|
103
|
+
* @returns {number}
|
|
104
|
+
*/
|
|
105
|
+
function countVisionImages(payload) {
|
|
106
|
+
if (!payload || !Array.isArray(payload.messages)) return 0;
|
|
107
|
+
let count = 0;
|
|
108
|
+
const walk = (content) => {
|
|
109
|
+
if (!content) return;
|
|
110
|
+
if (typeof content === 'string') {
|
|
111
|
+
// Count data: URLs in strings (rough)
|
|
112
|
+
const matches = content.match(/data:image/g);
|
|
113
|
+
if (matches) count += matches.length;
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
if (Array.isArray(content)) {
|
|
117
|
+
for (const b of content) {
|
|
118
|
+
if (!b || typeof b !== 'object') continue;
|
|
119
|
+
if (b.type === 'image' || b.type === 'image_url' || b.type === 'input_image' || b.image_url?.url || b.inlineData?.data || b.inline_data?.data) {
|
|
120
|
+
count += 1;
|
|
121
|
+
} else if (b.type === 'tool_result' && b.content) {
|
|
122
|
+
walk(b.content);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
if (typeof content === 'object' && blockNeedsVision(content)) count += 1;
|
|
128
|
+
};
|
|
129
|
+
for (const msg of payload.messages) {
|
|
130
|
+
if (msg) walk(msg.content);
|
|
131
|
+
}
|
|
132
|
+
return count;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Hash image bytes for cache keys (avoids hashing multi-MB base64 twice and
|
|
137
|
+
* gives stable keys for identical images).
|
|
138
|
+
* @param {string} data - base64 or URL string
|
|
139
|
+
* @returns {string} short hash
|
|
140
|
+
*/
|
|
141
|
+
function hashImageData(data) {
|
|
142
|
+
if (!data || typeof data !== 'string') return 'empty';
|
|
143
|
+
// URLs are short — hash full string; base64 is huge — hash first+last 4k
|
|
144
|
+
const sample = data.length > 8192 ? data.slice(0, 4096) + data.slice(-4096) + data.length : data;
|
|
145
|
+
return crypto.createHash('sha256').update(sample).digest('hex').substring(0, 16);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Collect per-image hashes for semantic-cache context keying.
|
|
150
|
+
* @param {object} payload
|
|
151
|
+
* @returns {string|null} combined hash or null when no images
|
|
152
|
+
*/
|
|
153
|
+
function visionContextHash(payload) {
|
|
154
|
+
if (!payload || !Array.isArray(payload.messages)) return null;
|
|
155
|
+
const hashes = [];
|
|
156
|
+
const collect = (content) => {
|
|
157
|
+
if (!content || typeof content === 'string') return;
|
|
158
|
+
const arr = Array.isArray(content) ? content : [content];
|
|
159
|
+
for (const b of arr) {
|
|
160
|
+
if (!b || typeof b !== 'object') continue;
|
|
161
|
+
if (b.type === 'image' && b.source?.data) hashes.push(hashImageData(b.source.data));
|
|
162
|
+
else if (b.type === 'image' && b.source?.url) hashes.push(hashImageData(b.source.url));
|
|
163
|
+
else if ((b.type === 'image_url' || b.type === 'input_image') && b.image_url?.url) hashes.push(hashImageData(b.image_url.url));
|
|
164
|
+
else if (b.image_url?.url) hashes.push(hashImageData(b.image_url.url));
|
|
165
|
+
else if (b.inlineData?.data) hashes.push(hashImageData(b.inlineData.data));
|
|
166
|
+
else if (b.inline_data?.data) hashes.push(hashImageData(b.inline_data.data));
|
|
167
|
+
else if (b.type === 'tool_result' && b.content) collect(b.content);
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
for (const msg of payload.messages) {
|
|
171
|
+
if (msg) collect(msg.content);
|
|
172
|
+
}
|
|
173
|
+
if (hashes.length === 0) return null;
|
|
174
|
+
return crypto.createHash('sha256').update(hashes.sort().join(',')).digest('hex').substring(0, 16);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Per-provider transport + model capability gate.
|
|
179
|
+
* Returns true only when the provider can carry image bytes AND the model
|
|
180
|
+
* advertises vision support in the registry.
|
|
181
|
+
*
|
|
182
|
+
* @param {string} provider
|
|
183
|
+
* @param {string|null} model
|
|
184
|
+
* @returns {boolean}
|
|
185
|
+
*/
|
|
186
|
+
function providerSupportsVision(provider, model) {
|
|
187
|
+
if (!provider || TRANSPORTLESS_PROVIDERS.has(provider)) return false;
|
|
188
|
+
if (!model) return false;
|
|
189
|
+
try {
|
|
190
|
+
const { getModelRegistrySync } = require('./model-registry');
|
|
191
|
+
const registry = getModelRegistrySync();
|
|
192
|
+
const info = registry.getCost(model);
|
|
193
|
+
return info?.vision === true;
|
|
194
|
+
} catch {
|
|
195
|
+
return false;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Redact image bytes for logging / trajectory export.
|
|
201
|
+
* Replaces source.data / image_url data: URLs with [REDACTED len=N hash=H].
|
|
202
|
+
* Mutates a deep-cloned structure — never call on live payloads.
|
|
203
|
+
* @param {*} value
|
|
204
|
+
* @returns {*}
|
|
205
|
+
*/
|
|
206
|
+
function redactVisionBytes(value) {
|
|
207
|
+
if (typeof value === 'string') {
|
|
208
|
+
if (value.length > 2000 && value.includes('data:image')) {
|
|
209
|
+
return value.replace(/data:image\/[^;,]+;base64,[A-Za-z0-9+/=]{100,}/g, (m) => `[REDACTED_IMAGE len=${m.length} hash=${hashImageData(m)}]`);
|
|
210
|
+
}
|
|
211
|
+
return value;
|
|
212
|
+
}
|
|
213
|
+
if (Array.isArray(value)) return value.map(redactVisionBytes);
|
|
214
|
+
if (value && typeof value === 'object') {
|
|
215
|
+
const out = { ...value };
|
|
216
|
+
if (typeof out.source?.data === 'string' && out.source.data.length > 500) {
|
|
217
|
+
out.source = { ...out.source, data: `[REDACTED_IMAGE len=${out.source.data.length} hash=${hashImageData(out.source.data)}]` };
|
|
218
|
+
}
|
|
219
|
+
if (typeof out.image_url?.url === 'string' && out.image_url.url.startsWith('data:image') && out.image_url.url.length > 500) {
|
|
220
|
+
out.image_url = { ...out.image_url, url: `[REDACTED_IMAGE len=${out.image_url.url.length} hash=${hashImageData(out.image_url.url)}]` };
|
|
221
|
+
}
|
|
222
|
+
if (typeof out.inlineData?.data === 'string' && out.inlineData.data.length > 500) {
|
|
223
|
+
out.inlineData = { ...out.inlineData, data: `[REDACTED_IMAGE len=${out.inlineData.data.length}]` };
|
|
224
|
+
}
|
|
225
|
+
// Recurse into nested content
|
|
226
|
+
if (out.content !== undefined) out.content = redactVisionBytes(out.content);
|
|
227
|
+
if (out.text !== undefined && typeof out.text === 'string' && out.text.includes('data:image')) {
|
|
228
|
+
out.text = redactVisionBytes(out.text);
|
|
229
|
+
}
|
|
230
|
+
return out;
|
|
231
|
+
}
|
|
232
|
+
return value;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
module.exports = {
|
|
236
|
+
VISION_IMAGE_TOKEN_ESTIMATE,
|
|
237
|
+
TRANSPORTLESS_PROVIDERS,
|
|
238
|
+
blockNeedsVision,
|
|
239
|
+
contentNeedsVision,
|
|
240
|
+
payloadNeedsVision,
|
|
241
|
+
countVisionImages,
|
|
242
|
+
hashImageData,
|
|
243
|
+
visionContextHash,
|
|
244
|
+
providerSupportsVision,
|
|
245
|
+
redactVisionBytes,
|
|
246
|
+
};
|
|
@@ -50,11 +50,19 @@ const ANONYMIZE_PATTERNS = [
|
|
|
50
50
|
[/\/home\/[^/\s]+/g, "/home/<USER>"],
|
|
51
51
|
// IPs
|
|
52
52
|
[/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, "<IP>"],
|
|
53
|
+
// Inline base64 image bytes (never persist image data to training JSONL)
|
|
54
|
+
[/data:image\/[^;,]+;base64,[A-Za-z0-9+/=]{200,}/g, "<REDACTED_IMAGE>"],
|
|
53
55
|
// Hostnames containing service-now / corporate domains (configurable)
|
|
54
56
|
[/[A-Za-z0-9-]+\.service-now\.com/gi, "<SERVICENOW_HOST>"],
|
|
55
57
|
];
|
|
56
58
|
|
|
57
59
|
function anonymize(value) {
|
|
60
|
+
// Strip image bytes at the structural level first (raw base64 source.data
|
|
61
|
+
// has no data: prefix for the regex to catch).
|
|
62
|
+
try {
|
|
63
|
+
const { redactVisionBytes } = require("../routing/vision");
|
|
64
|
+
value = redactVisionBytes(value);
|
|
65
|
+
} catch { /* best-effort */ }
|
|
58
66
|
if (typeof value === "string") {
|
|
59
67
|
let out = value;
|
|
60
68
|
for (const [re, replacement] of ANONYMIZE_PATTERNS) {
|
package/src/utils/tokens.js
CHANGED
|
@@ -50,9 +50,9 @@ function countPayloadTokens(payload) {
|
|
|
50
50
|
return sum + estimateTokens(block.text || '');
|
|
51
51
|
} else if (block.type === 'tool_result') {
|
|
52
52
|
return sum + estimateTokens(block.content || '');
|
|
53
|
-
} else if (block.type === 'image') {
|
|
54
|
-
//
|
|
55
|
-
return sum +
|
|
53
|
+
} else if (block.type === 'image' || block.type === 'image_url' || block.type === 'input_image' || block.image_url?.url || block.inlineData?.data || block.inline_data?.data) {
|
|
54
|
+
// Fixed per-image estimate — never count base64 bytes as text.
|
|
55
|
+
return sum + 1500;
|
|
56
56
|
}
|
|
57
57
|
return sum + estimateTokens(JSON.stringify(block));
|
|
58
58
|
}, 0);
|