@aibranch/llm-meta-widget 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -61,7 +61,7 @@ Option 3 if it does not.
|
|
|
61
61
|
|
|
62
62
|
```ruby
|
|
63
63
|
# Gemfile
|
|
64
|
-
gem "llm_meta_widget", "~> 0.
|
|
64
|
+
gem "llm_meta_widget", "~> 0.12.0"
|
|
65
65
|
```
|
|
66
66
|
|
|
67
67
|
```bash
|
|
@@ -152,17 +152,17 @@ The widget is a custom element in a single self-contained ES module, so the
|
|
|
152
152
|
whole integration is two lines of HTML:
|
|
153
153
|
|
|
154
154
|
```html
|
|
155
|
-
<script type="module" src="https://cdn.jsdelivr.net/npm/@aibranch/llm-meta-widget@0.
|
|
155
|
+
<script type="module" src="https://cdn.jsdelivr.net/npm/@aibranch/llm-meta-widget@0.12"></script>
|
|
156
156
|
<llm-meta-widget llm-url="https://your-meta-server.example"
|
|
157
157
|
model="gpt-oss-120b"
|
|
158
158
|
greeting="Hi — ask me anything about this page."></llm-meta-widget>
|
|
159
159
|
```
|
|
160
160
|
|
|
161
|
-
**`@0.
|
|
161
|
+
**`@0.12` is a version range, and it stops at 0.12.x on purpose.** You receive
|
|
162
162
|
bug fixes without editing your page, and a release that breaks your page cannot
|
|
163
163
|
arrive on its own.
|
|
164
164
|
|
|
165
|
-
Moving to 0.
|
|
165
|
+
Moving to 0.13 is therefore a change you make yourself. Read
|
|
166
166
|
[CHANGELOG.md](CHANGELOG.md) before you do, because this is where what your page
|
|
167
167
|
must provide can change. For example, 0.10.0 changed how the widget finds a page
|
|
168
168
|
card: by media type rather than by element id. A page that declared its card the
|
|
@@ -698,8 +698,12 @@ the model's context. The widget reads two optional fields the
|
|
|
698
698
|
`_meta`; the widget still reads that when `size` is absent, but it will be
|
|
699
699
|
removed, so emit `size`.
|
|
700
700
|
The widget decides whether it can afford the resource *before* fetching it. If
|
|
701
|
-
the resource is over
|
|
702
|
-
|
|
701
|
+
the resource is over that budget, the bytes never cross the wire, and the LLM
|
|
702
|
+
asks for what it needs through your tools instead.
|
|
703
|
+
The budget is a share of the model's own context window, which the widget
|
|
704
|
+
reads from the hub: about 8 KB for a 32,000-token model, about 32 KB for a
|
|
705
|
+
128,000-token one, and at most 64 KB. When the hub does not report a window,
|
|
706
|
+
the budget is 8 KB.
|
|
703
707
|
- **`volatility`** — `"stable"` (read once, re-used) or `"volatile"` (re-read
|
|
704
708
|
on every turn that uses it).
|
|
705
709
|
- **`autoAttach`** — may the widget put this in the model's context without
|
|
@@ -168,6 +168,16 @@ async function runChatLoop(opts) {
|
|
|
168
168
|
remoteTools = [],
|
|
169
169
|
hostWideTools = [],
|
|
170
170
|
maxRounds = 10,
|
|
171
|
+
// The model's own context window, from /api/llms. Without it the loop
|
|
172
|
+
// cannot know how much room is left, so it budgets nothing and a long
|
|
173
|
+
// tool-heavy turn is refused by the provider instead of degrading.
|
|
174
|
+
contextWindow = null,
|
|
175
|
+
// Tokens held back for the answer: a request that fills the window leaves
|
|
176
|
+
// nothing to generate into.
|
|
177
|
+
outputReserve = 2048,
|
|
178
|
+
// Called when a tool result is dropped to make room, so the panel can say
|
|
179
|
+
// so rather than the context quietly thinning.
|
|
180
|
+
onContextEvicted,
|
|
171
181
|
// "hub" (default) talks to llm_meta_server; "ollama" talks straight to an
|
|
172
182
|
// Ollama, with no server component of ours in between. The loop is the
|
|
173
183
|
// same either way — only the call is swapped.
|
|
@@ -189,7 +199,7 @@ async function runChatLoop(opts) {
|
|
|
189
199
|
onPhase,
|
|
190
200
|
...singleOpts
|
|
191
201
|
} = opts;
|
|
192
|
-
|
|
202
|
+
let messages = [...singleOpts.messages || []];
|
|
193
203
|
const remoteByName = Object.fromEntries((remoteTools || []).map((t) => [t.name, t]));
|
|
194
204
|
const hostWideByName = Object.fromEntries((hostWideTools || []).map((t) => [t.name, t]));
|
|
195
205
|
const toolIds = [...singleOpts.toolIds || [], ...(remoteTools || []).map((t) => t.id)];
|
|
@@ -200,6 +210,12 @@ async function runChatLoop(opts) {
|
|
|
200
210
|
}
|
|
201
211
|
const mergedLocalTools = Object.values(inlineByName);
|
|
202
212
|
const rounds = [];
|
|
213
|
+
const evicted = [];
|
|
214
|
+
const toolsTokens = estimateToolsTokens([].concat(
|
|
215
|
+
singleOpts.localTools || [],
|
|
216
|
+
remoteTools || [],
|
|
217
|
+
hostWideTools || []
|
|
218
|
+
));
|
|
203
219
|
const allDispatched = [];
|
|
204
220
|
const allSkipped = [];
|
|
205
221
|
let lastResult = null;
|
|
@@ -230,6 +246,17 @@ async function runChatLoop(opts) {
|
|
|
230
246
|
for (let round = 0; round < maxRounds; round++) {
|
|
231
247
|
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
|
232
248
|
onRoundStart?.(round);
|
|
249
|
+
const room = evictOldToolResults({
|
|
250
|
+
messages,
|
|
251
|
+
window: contextWindow,
|
|
252
|
+
dropped: evicted,
|
|
253
|
+
// The tools take their share of the window before any message does.
|
|
254
|
+
reserve: outputReserve + toolsTokens
|
|
255
|
+
});
|
|
256
|
+
if (room.messages !== messages) {
|
|
257
|
+
messages = room.messages;
|
|
258
|
+
onContextEvicted?.(evicted.slice());
|
|
259
|
+
}
|
|
233
260
|
const call = provider === "ollama" ? ollamaChatCall : singleLlmCall;
|
|
234
261
|
const turnResult = await call({
|
|
235
262
|
...singleOpts,
|
|
@@ -561,6 +588,58 @@ function parseSseFrame(raw) {
|
|
|
561
588
|
}
|
|
562
589
|
var STATIC_PRIMITIVES_META = "io.modelcontextprotocol/static-primitives";
|
|
563
590
|
var PAGE_CARD_MEDIA_TYPE = "application/mcp-static-primitives+json";
|
|
591
|
+
var RESOURCE_SHARE_OF_WINDOW = 0.0625;
|
|
592
|
+
function resourceBudgetBytes(window2, { floor = 8e3, cap = 64e3 } = {}) {
|
|
593
|
+
if (!window2 || !Number.isFinite(window2)) return floor;
|
|
594
|
+
const share = Math.round(window2 * RESOURCE_SHARE_OF_WINDOW * CHARS_PER_TOKEN);
|
|
595
|
+
return Math.min(cap, Math.max(floor, share));
|
|
596
|
+
}
|
|
597
|
+
var CHARS_PER_TOKEN = 4;
|
|
598
|
+
var MESSAGE_OVERHEAD_TOKENS = 4;
|
|
599
|
+
function estimateTokens(messages) {
|
|
600
|
+
return (messages || []).reduce((sum, m) => {
|
|
601
|
+
const content = typeof m?.content === "string" ? m.content : JSON.stringify(m?.content ?? "");
|
|
602
|
+
return sum + Math.ceil(content.length / CHARS_PER_TOKEN) + MESSAGE_OVERHEAD_TOKENS;
|
|
603
|
+
}, 0);
|
|
604
|
+
}
|
|
605
|
+
function estimateToolsTokens(tools) {
|
|
606
|
+
const text = (tools || []).map((t) => JSON.stringify({
|
|
607
|
+
name: t?.name,
|
|
608
|
+
description: t?.description,
|
|
609
|
+
schema: t?.input_schema || t?.inputSchema
|
|
610
|
+
})).join("");
|
|
611
|
+
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
|
612
|
+
}
|
|
613
|
+
function evictOldToolResults({ messages, window: window2, reserve = 0, dropped = [] }) {
|
|
614
|
+
if (!window2 || !Number.isFinite(window2)) return { messages, dropped };
|
|
615
|
+
const budget = window2 - reserve;
|
|
616
|
+
if (estimateTokens(messages) <= budget) return { messages, dropped };
|
|
617
|
+
const out = messages.slice();
|
|
618
|
+
for (let i = 0; i < out.length; i++) {
|
|
619
|
+
if (estimateTokens(out) <= budget) break;
|
|
620
|
+
const m = out[i];
|
|
621
|
+
if (m?.role !== "tool") continue;
|
|
622
|
+
if (typeof m.content === "string" && m.content.startsWith("[dropped")) continue;
|
|
623
|
+
dropped.push({ name: m.name, tokens: estimateTokens([m]) });
|
|
624
|
+
out[i] = Object.assign({}, m, {
|
|
625
|
+
content: "[dropped to make room in the context window. Call the tool again if you still need it, with a narrower query.]"
|
|
626
|
+
});
|
|
627
|
+
}
|
|
628
|
+
return { messages: out, dropped };
|
|
629
|
+
}
|
|
630
|
+
function modelWindowFrom(payload, modelId) {
|
|
631
|
+
for (const family of payload && payload.llms || []) {
|
|
632
|
+
const entries = [].concat(family.available_models || [], family.models || []);
|
|
633
|
+
for (const entry of entries) {
|
|
634
|
+
if (!entry) continue;
|
|
635
|
+
const id = entry.value || entry.name;
|
|
636
|
+
if (id !== modelId) continue;
|
|
637
|
+
const window2 = Number(entry.max_input_tokens);
|
|
638
|
+
return Number.isFinite(window2) && window2 > 0 ? window2 : null;
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
return null;
|
|
642
|
+
}
|
|
564
643
|
var PAGE_GLOBAL_TRANSPORT = "page-global";
|
|
565
644
|
function declaredPageGlobal(card) {
|
|
566
645
|
for (const remote of card && card.remotes || []) {
|
|
@@ -3237,12 +3316,28 @@ var conversation_default = `/*
|
|
|
3237
3316
|
white-space: pre;
|
|
3238
3317
|
}
|
|
3239
3318
|
.llm-meta-conversation .message-content pre code { background: none; padding: 0; }
|
|
3319
|
+
/* The scroller is the WRAPPER, not the table: a table that is itself the
|
|
3320
|
+
* scrolling box gets capped at the bubble width and squeezes its columns
|
|
3321
|
+
* instead of overflowing, and .message's word-wrap then breaks every word
|
|
3322
|
+
* one character per line. The wrapper is capped; the table inside keeps its
|
|
3323
|
+
* natural width and scrolls sideways. */
|
|
3324
|
+
.llm-meta-conversation .message-content .lmw-table-scroll {
|
|
3325
|
+
overflow-x: auto;
|
|
3326
|
+
margin: 0.5em 0;
|
|
3327
|
+
}
|
|
3240
3328
|
.llm-meta-conversation .message-content table {
|
|
3241
3329
|
border-collapse: collapse;
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3330
|
+
/* min-width so a narrow table still fills the bubble, width:max-content so
|
|
3331
|
+
* a wide one is laid out at its natural size and the wrapper scrolls. */
|
|
3332
|
+
width: max-content;
|
|
3333
|
+
min-width: 100%;
|
|
3334
|
+
}
|
|
3335
|
+
/* Inside a table, breaking mid-token is what produced the one-letter columns;
|
|
3336
|
+
* cells keep their words whole and the wrapper scrolls instead. */
|
|
3337
|
+
.llm-meta-conversation .message-content th,
|
|
3338
|
+
.llm-meta-conversation .message-content td {
|
|
3339
|
+
word-wrap: normal;
|
|
3340
|
+
overflow-wrap: normal;
|
|
3246
3341
|
}
|
|
3247
3342
|
.llm-meta-conversation .message-content th,
|
|
3248
3343
|
.llm-meta-conversation .message-content td {
|
|
@@ -3844,6 +3939,20 @@ function boot(cfg) {
|
|
|
3844
3939
|
var root = document.getElementById("llm-meta-widget-chat");
|
|
3845
3940
|
var toggleBtn = document.getElementById("llm-meta-widget-toggle");
|
|
3846
3941
|
var historyEl = root.querySelector(".lmw-messages");
|
|
3942
|
+
var SCROLL_PIN_SLACK = 40;
|
|
3943
|
+
var followingOutput = true;
|
|
3944
|
+
function atBottom() {
|
|
3945
|
+
return historyEl.scrollHeight - historyEl.scrollTop - historyEl.clientHeight <= SCROLL_PIN_SLACK;
|
|
3946
|
+
}
|
|
3947
|
+
function pinToBottom() {
|
|
3948
|
+
historyEl.scrollTop = historyEl.scrollHeight;
|
|
3949
|
+
}
|
|
3950
|
+
function pinIfFollowing() {
|
|
3951
|
+
if (followingOutput) pinToBottom();
|
|
3952
|
+
}
|
|
3953
|
+
historyEl.addEventListener("scroll", function() {
|
|
3954
|
+
followingOutput = atBottom();
|
|
3955
|
+
});
|
|
3847
3956
|
var formEl = root.querySelector(".lmw-form");
|
|
3848
3957
|
var inputEl = root.querySelector(".lmw-input");
|
|
3849
3958
|
var clearBtn = root.querySelector(".lmw-clear");
|
|
@@ -4069,6 +4178,8 @@ function boot(cfg) {
|
|
|
4069
4178
|
tasks.push(fetch(LLM_BASE + "/api/llms", { headers: { "Accept": "application/json" } }).then(function(r) {
|
|
4070
4179
|
return r.ok ? r.json() : { llms: [] };
|
|
4071
4180
|
}).then(function(payload) {
|
|
4181
|
+
llmsPayload = payload;
|
|
4182
|
+
modelWindow = modelWindowFrom(payload, MODEL);
|
|
4072
4183
|
var flat = [];
|
|
4073
4184
|
(payload.llms || []).forEach(function(llm) {
|
|
4074
4185
|
if (llm.family !== "ollama") return;
|
|
@@ -4189,6 +4300,7 @@ function boot(cfg) {
|
|
|
4189
4300
|
if (modelPicker) {
|
|
4190
4301
|
modelPicker.addEventListener("change", function() {
|
|
4191
4302
|
MODEL = modelPicker.value;
|
|
4303
|
+
modelWindow = modelWindowFrom(llmsPayload, MODEL);
|
|
4192
4304
|
});
|
|
4193
4305
|
}
|
|
4194
4306
|
if (toolsPicker) {
|
|
@@ -4213,16 +4325,29 @@ function boot(cfg) {
|
|
|
4213
4325
|
var hostWidePrompts = pageCard.prompts.map(function(p) {
|
|
4214
4326
|
return Object.assign({}, p, { source: "page" });
|
|
4215
4327
|
});
|
|
4328
|
+
var modelWindow = null;
|
|
4329
|
+
var llmsPayload = null;
|
|
4216
4330
|
var resourceContext = null;
|
|
4217
4331
|
var resourcePlan = null;
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4332
|
+
function resourceBudget() {
|
|
4333
|
+
return resourceBudgetBytes(modelWindow);
|
|
4334
|
+
}
|
|
4335
|
+
var pageResourcePlans = null;
|
|
4336
|
+
var pageResourcePlansBudget = null;
|
|
4337
|
+
function pageResourcePlansNow() {
|
|
4338
|
+
var budget = resourceBudget();
|
|
4339
|
+
if (pageResourcePlans === null || budget !== pageResourcePlansBudget) {
|
|
4340
|
+
pageResourcePlansBudget = budget;
|
|
4341
|
+
pageResourcePlans = planPageResources({
|
|
4342
|
+
entries: pageCard.resources,
|
|
4343
|
+
budgetBytes: budget,
|
|
4344
|
+
onSkip: function(plan) {
|
|
4345
|
+
console.info("[llm_meta_widget] not attaching " + plan.uri + " (" + plan.reason + ", size=" + plan.size + ")");
|
|
4346
|
+
}
|
|
4347
|
+
});
|
|
4224
4348
|
}
|
|
4225
|
-
|
|
4349
|
+
return pageResourcePlans;
|
|
4350
|
+
}
|
|
4226
4351
|
var pageResourceCache = {};
|
|
4227
4352
|
restoreConversation();
|
|
4228
4353
|
var promptsEl = root.querySelector(".lmw-prompts");
|
|
@@ -4243,7 +4368,7 @@ function boot(cfg) {
|
|
|
4243
4368
|
if (resourcePlan === null) {
|
|
4244
4369
|
var loaded = await loadHostResource({
|
|
4245
4370
|
endpoint,
|
|
4246
|
-
budgetBytes:
|
|
4371
|
+
budgetBytes: resourceBudget(),
|
|
4247
4372
|
list: listMcpResources,
|
|
4248
4373
|
read: readMcpResource,
|
|
4249
4374
|
onSkip: function(plan) {
|
|
@@ -4305,7 +4430,7 @@ function boot(cfg) {
|
|
|
4305
4430
|
});
|
|
4306
4431
|
if (body.parentNode) body.parentNode.appendChild(row);
|
|
4307
4432
|
});
|
|
4308
|
-
|
|
4433
|
+
pinToBottom();
|
|
4309
4434
|
return true;
|
|
4310
4435
|
}
|
|
4311
4436
|
function renderWelcome() {
|
|
@@ -4406,7 +4531,7 @@ function boot(cfg) {
|
|
|
4406
4531
|
div.appendChild(document.createTextNode(" "));
|
|
4407
4532
|
div.appendChild(body);
|
|
4408
4533
|
historyEl.appendChild(div);
|
|
4409
|
-
|
|
4534
|
+
pinIfFollowing();
|
|
4410
4535
|
body.roleLabel = label;
|
|
4411
4536
|
return body;
|
|
4412
4537
|
}
|
|
@@ -4416,6 +4541,17 @@ function boot(cfg) {
|
|
|
4416
4541
|
} catch (e) {
|
|
4417
4542
|
el.textContent = mdText;
|
|
4418
4543
|
}
|
|
4544
|
+
wrapTablesForScrolling(el);
|
|
4545
|
+
}
|
|
4546
|
+
function wrapTablesForScrolling(el) {
|
|
4547
|
+
var tables = el.querySelectorAll("table");
|
|
4548
|
+
for (var i = 0; i < tables.length; i++) {
|
|
4549
|
+
var table = tables[i];
|
|
4550
|
+
var wrap = document.createElement("div");
|
|
4551
|
+
wrap.className = "lmw-table-scroll";
|
|
4552
|
+
table.parentNode.insertBefore(wrap, table);
|
|
4553
|
+
wrap.appendChild(table);
|
|
4554
|
+
}
|
|
4419
4555
|
}
|
|
4420
4556
|
function roleLabel(role) {
|
|
4421
4557
|
if (role === "user") return "\u{1F464} You";
|
|
@@ -4453,7 +4589,7 @@ function boot(cfg) {
|
|
|
4453
4589
|
chip.dataset.tool = toolCall.name;
|
|
4454
4590
|
row.appendChild(chip);
|
|
4455
4591
|
pendingChips.push(chip);
|
|
4456
|
-
|
|
4592
|
+
pinIfFollowing();
|
|
4457
4593
|
}
|
|
4458
4594
|
function resolveToolCall(outcome) {
|
|
4459
4595
|
var idx = pendingChips.findIndex(function(c) {
|
|
@@ -4493,7 +4629,7 @@ function boot(cfg) {
|
|
|
4493
4629
|
} else {
|
|
4494
4630
|
historyEl.appendChild(details);
|
|
4495
4631
|
}
|
|
4496
|
-
|
|
4632
|
+
pinIfFollowing();
|
|
4497
4633
|
currentThinkingBlock = details;
|
|
4498
4634
|
currentThinkingBody = body;
|
|
4499
4635
|
return body;
|
|
@@ -4573,13 +4709,13 @@ function boot(cfg) {
|
|
|
4573
4709
|
cached: resourceContext,
|
|
4574
4710
|
endpoint: resourcePlan && resourcePlan.endpoint,
|
|
4575
4711
|
read: readMcpResource,
|
|
4576
|
-
budgetBytes:
|
|
4712
|
+
budgetBytes: resourceBudget()
|
|
4577
4713
|
});
|
|
4578
4714
|
resourceContext = turn.cached;
|
|
4579
4715
|
var page = await pageResourceLinesForTurn({
|
|
4580
|
-
plans:
|
|
4716
|
+
plans: pageResourcePlansNow(),
|
|
4581
4717
|
resources: handlers.resources,
|
|
4582
|
-
budgetBytes:
|
|
4718
|
+
budgetBytes: resourceBudget(),
|
|
4583
4719
|
cache: pageResourceCache
|
|
4584
4720
|
});
|
|
4585
4721
|
pageResourceCache = page.cache;
|
|
@@ -4610,6 +4746,7 @@ function boot(cfg) {
|
|
|
4610
4746
|
var userText = inputEl.value.trim();
|
|
4611
4747
|
if (!userText) return;
|
|
4612
4748
|
inputEl.value = "";
|
|
4749
|
+
followingOutput = true;
|
|
4613
4750
|
appendTurn("user", userText, pendingTurnLabel);
|
|
4614
4751
|
pendingTurnLabel = null;
|
|
4615
4752
|
if (currentAbort) {
|
|
@@ -4646,8 +4783,14 @@ function boot(cfg) {
|
|
|
4646
4783
|
hostWideTools,
|
|
4647
4784
|
aiActions: handlers.tools,
|
|
4648
4785
|
maxRounds: MAX_ROUNDS,
|
|
4786
|
+
contextWindow: modelWindow,
|
|
4649
4787
|
provider: LLM_PROVIDER === "ollama" ? "ollama" : "hub",
|
|
4650
4788
|
signal: currentAbort.signal,
|
|
4789
|
+
onContextEvicted: function(dropped) {
|
|
4790
|
+
var last = dropped[dropped.length - 1];
|
|
4791
|
+
var name = last && last.name ? last.name : "tool";
|
|
4792
|
+
appendTurn("system", "Dropped an earlier " + name + " result to make room in the model's context. Ask again if you need it.");
|
|
4793
|
+
},
|
|
4651
4794
|
onToolCall: function(toolCall) {
|
|
4652
4795
|
announceToolCall(toolCall);
|
|
4653
4796
|
persistConversation();
|
|
@@ -4669,7 +4812,7 @@ function boot(cfg) {
|
|
|
4669
4812
|
var body = ensureThinkingBlock();
|
|
4670
4813
|
body.appendChild(document.createTextNode(delta));
|
|
4671
4814
|
body.scrollTop = body.scrollHeight;
|
|
4672
|
-
|
|
4815
|
+
pinIfFollowing();
|
|
4673
4816
|
},
|
|
4674
4817
|
onTextDelta: function(delta) {
|
|
4675
4818
|
markDone(assistantBody.roleLabel);
|
|
@@ -4678,7 +4821,7 @@ function boot(cfg) {
|
|
|
4678
4821
|
renderMarkdownInto(assistantBody, assistantMarkdown);
|
|
4679
4822
|
liveTurn.content = assistantMarkdown;
|
|
4680
4823
|
persistSoon();
|
|
4681
|
-
|
|
4824
|
+
pinIfFollowing();
|
|
4682
4825
|
}
|
|
4683
4826
|
});
|
|
4684
4827
|
liveTurn.content = result.content;
|
|
@@ -325,6 +325,16 @@ export async function runChatLoop(opts) {
|
|
|
325
325
|
remoteTools = [],
|
|
326
326
|
hostWideTools = [],
|
|
327
327
|
maxRounds = 10,
|
|
328
|
+
// The model's own context window, from /api/llms. Without it the loop
|
|
329
|
+
// cannot know how much room is left, so it budgets nothing and a long
|
|
330
|
+
// tool-heavy turn is refused by the provider instead of degrading.
|
|
331
|
+
contextWindow = null,
|
|
332
|
+
// Tokens held back for the answer: a request that fills the window leaves
|
|
333
|
+
// nothing to generate into.
|
|
334
|
+
outputReserve = 2048,
|
|
335
|
+
// Called when a tool result is dropped to make room, so the panel can say
|
|
336
|
+
// so rather than the context quietly thinning.
|
|
337
|
+
onContextEvicted,
|
|
328
338
|
// "hub" (default) talks to llm_meta_server; "ollama" talks straight to an
|
|
329
339
|
// Ollama, with no server component of ours in between. The loop is the
|
|
330
340
|
// same either way — only the call is swapped.
|
|
@@ -347,7 +357,7 @@ export async function runChatLoop(opts) {
|
|
|
347
357
|
...singleOpts
|
|
348
358
|
} = opts
|
|
349
359
|
|
|
350
|
-
|
|
360
|
+
let messages = [ ...(singleOpts.messages || []) ]
|
|
351
361
|
const remoteByName = Object.fromEntries((remoteTools || []).map((t) => [ t.name, t ]))
|
|
352
362
|
// Class 2: host-wide MCP tools (well-known). Map name→{endpoint, ...} for
|
|
353
363
|
// dispatch. If the same name appears in aiActions, Class 3 wins there
|
|
@@ -366,6 +376,10 @@ export async function runChatLoop(opts) {
|
|
|
366
376
|
const mergedLocalTools = Object.values(inlineByName)
|
|
367
377
|
|
|
368
378
|
const rounds = []
|
|
379
|
+
const evicted = []
|
|
380
|
+
// Fixed for the turn: the declared tool set does not change between rounds.
|
|
381
|
+
const toolsTokens = estimateToolsTokens([].concat(
|
|
382
|
+
singleOpts.localTools || [], remoteTools || [], hostWideTools || []))
|
|
369
383
|
const allDispatched = []
|
|
370
384
|
const allSkipped = []
|
|
371
385
|
let lastResult = null
|
|
@@ -408,6 +422,18 @@ export async function runChatLoop(opts) {
|
|
|
408
422
|
onRoundStart?.(round)
|
|
409
423
|
|
|
410
424
|
// The loop is the same whoever answers; only the call differs.
|
|
425
|
+
// Make room before sending, rather than after being refused. Only old
|
|
426
|
+
// tool results are given up; see evictOldToolResults.
|
|
427
|
+
const room = evictOldToolResults({
|
|
428
|
+
messages, window: contextWindow, dropped: evicted,
|
|
429
|
+
// The tools take their share of the window before any message does.
|
|
430
|
+
reserve: outputReserve + toolsTokens
|
|
431
|
+
});
|
|
432
|
+
if (room.messages !== messages) {
|
|
433
|
+
messages = room.messages;
|
|
434
|
+
onContextEvicted?.(evicted.slice());
|
|
435
|
+
}
|
|
436
|
+
|
|
411
437
|
const call = provider === "ollama" ? ollamaChatCall : singleLlmCall
|
|
412
438
|
const turnResult = await call({
|
|
413
439
|
...singleOpts,
|
|
@@ -891,6 +917,111 @@ export const PAGE_CARD_MEDIA_TYPE = "application/mcp-static-primitives+json"
|
|
|
891
917
|
// card, find the handlers and dispatch. The three legacy globals
|
|
892
918
|
// (aiActions/aiPrompts/aiResources) are names only this widget knows, and are
|
|
893
919
|
// configurable per page, so nothing outside could rely on them.
|
|
920
|
+
// How much room a model has, read from /api/llms — the one call the widget
|
|
921
|
+
// already makes before a turn. Scans every family, because the picker shows
|
|
922
|
+
// only the local ones while the configured model is often hosted, and the two
|
|
923
|
+
// families use different shapes for the same thing (`value` vs `name`).
|
|
924
|
+
//
|
|
925
|
+
// Returns null when the hub does not say. Null means "no budget to work with",
|
|
926
|
+
// which is not the same as zero, and every caller below treats it as "do not
|
|
927
|
+
// guess" rather than "no room".
|
|
928
|
+
// How many bytes of reference resources a turn may attach, derived from the
|
|
929
|
+
// model's window instead of guessed. The old fixed 8000 was chosen as "~2k
|
|
930
|
+
// tokens, beyond which a resource crowds out the conversation on a 32k local
|
|
931
|
+
// model" — which is 6.25% of that window, so the same share reproduces 8000 at
|
|
932
|
+
// 32k exactly and gives a 128k model the 32 KB it can afford.
|
|
933
|
+
//
|
|
934
|
+
// The floor keeps the previous behaviour whenever the window is unknown. The
|
|
935
|
+
// cap is there because these bytes are fetched and inlined on every turn: the
|
|
936
|
+
// largest real catalog (PubDictionaries, 218 dictionaries) is ~35 KB, so 64 KB
|
|
937
|
+
// is already generous and a 1M-token model does not need 250 KB of it.
|
|
938
|
+
export const RESOURCE_SHARE_OF_WINDOW = 0.0625
|
|
939
|
+
|
|
940
|
+
export function resourceBudgetBytes(window, { floor = 8000, cap = 64_000 } = {}) {
|
|
941
|
+
if (!window || !Number.isFinite(window)) return floor
|
|
942
|
+
const share = Math.round(window * RESOURCE_SHARE_OF_WINDOW * CHARS_PER_TOKEN)
|
|
943
|
+
return Math.min(cap, Math.max(floor, share))
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// A rough token count, the same shape the hub's own estimator uses: characters
|
|
947
|
+
// over four, plus a fixed cost per message. Deliberately approximate — it is
|
|
948
|
+
// used to decide "drop the oldest tool result", never to fill the window
|
|
949
|
+
// exactly, and it errs high so the decision is made early rather than late.
|
|
950
|
+
export const CHARS_PER_TOKEN = 4
|
|
951
|
+
export const MESSAGE_OVERHEAD_TOKENS = 4
|
|
952
|
+
|
|
953
|
+
export function estimateTokens(messages) {
|
|
954
|
+
return (messages || []).reduce((sum, m) => {
|
|
955
|
+
const content = typeof m?.content === "string" ? m.content : JSON.stringify(m?.content ?? "")
|
|
956
|
+
return sum + Math.ceil(content.length / CHARS_PER_TOKEN) + MESSAGE_OVERHEAD_TOKENS
|
|
957
|
+
}, 0)
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
// What the declared tools cost in the window. They are part of every request:
|
|
961
|
+
// names, descriptions and JSON schemas. On the hub path the browser sends only
|
|
962
|
+
// `tool_ids` and the hub expands the schemas, but the browser listed those
|
|
963
|
+
// tools to build its picker, so it still has what it needs to price them.
|
|
964
|
+
//
|
|
965
|
+
// This is not a detail. TogoMCP declares around twenty tools, which is a few
|
|
966
|
+
// thousand tokens standing in the window before the conversation starts; a
|
|
967
|
+
// budget counting only messages runs low and evicts too late to help.
|
|
968
|
+
export function estimateToolsTokens(tools) {
|
|
969
|
+
const text = (tools || []).map((t) => JSON.stringify({
|
|
970
|
+
name: t?.name, description: t?.description, schema: t?.input_schema || t?.inputSchema
|
|
971
|
+
})).join("")
|
|
972
|
+
return Math.ceil(text.length / CHARS_PER_TOKEN)
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
// Makes room for the next round by dropping the OLDEST tool results, which is
|
|
976
|
+
// the one eviction a chat loop can justify: the model has already read them and
|
|
977
|
+
// said what it made of them in the assistant turn that follows, so its own
|
|
978
|
+
// summary survives. Nothing else is touched — not the system prompt, not the
|
|
979
|
+
// questions, not the assistant's words.
|
|
980
|
+
//
|
|
981
|
+
// Each dropped result leaves a marker in its place, because a tool message must
|
|
982
|
+
// still answer its tool_call_id: a provider that sees a call with no result
|
|
983
|
+
// rejects the request, and a model that sees no mention of the result would
|
|
984
|
+
// happily call the same tool again.
|
|
985
|
+
//
|
|
986
|
+
// Returns the messages unchanged when the window is unknown. Guessing a window
|
|
987
|
+
// would be worse than not budgeting: too low throws away context the turn
|
|
988
|
+
// needed, and too high fails at the provider anyway.
|
|
989
|
+
export function evictOldToolResults({ messages, window, reserve = 0, dropped = [] }) {
|
|
990
|
+
if (!window || !Number.isFinite(window)) return { messages, dropped }
|
|
991
|
+
|
|
992
|
+
const budget = window - reserve
|
|
993
|
+
if (estimateTokens(messages) <= budget) return { messages, dropped }
|
|
994
|
+
|
|
995
|
+
const out = messages.slice()
|
|
996
|
+
for (let i = 0; i < out.length; i++) {
|
|
997
|
+
if (estimateTokens(out) <= budget) break
|
|
998
|
+
const m = out[i]
|
|
999
|
+
if (m?.role !== "tool") continue
|
|
1000
|
+
if (typeof m.content === "string" && m.content.startsWith("[dropped")) continue
|
|
1001
|
+
|
|
1002
|
+
dropped.push({ name: m.name, tokens: estimateTokens([ m ]) })
|
|
1003
|
+
out[i] = Object.assign({}, m, {
|
|
1004
|
+
content: "[dropped to make room in the context window. Call the tool again " +
|
|
1005
|
+
"if you still need it, with a narrower query.]"
|
|
1006
|
+
})
|
|
1007
|
+
}
|
|
1008
|
+
return { messages: out, dropped }
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
export function modelWindowFrom(payload, modelId) {
|
|
1012
|
+
for (const family of (payload && payload.llms) || []) {
|
|
1013
|
+
const entries = [].concat(family.available_models || [], family.models || [])
|
|
1014
|
+
for (const entry of entries) {
|
|
1015
|
+
if (!entry) continue
|
|
1016
|
+
const id = entry.value || entry.name
|
|
1017
|
+
if (id !== modelId) continue
|
|
1018
|
+
const window = Number(entry.max_input_tokens)
|
|
1019
|
+
return Number.isFinite(window) && window > 0 ? window : null
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
return null
|
|
1023
|
+
}
|
|
1024
|
+
|
|
894
1025
|
export const PAGE_GLOBAL_TRANSPORT = "page-global"
|
|
895
1026
|
|
|
896
1027
|
export function declaredPageGlobal(card) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aibranch/llm-meta-widget",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0",
|
|
4
4
|
"//version": "Must stay in step with lib/llm_meta_widget/version.rb. Two independently versioned copies of the same client is the drift docs/custom-element-distribution.md warns about: bump both, release both.",
|
|
5
5
|
"description": "Embeddable browser chat widget: a custom element that discovers tools at page, origin and hub scope and dispatches each tool call to the right one. Two lines of HTML on any host, Rails or not.",
|
|
6
6
|
"keywords": [
|