@mlx-node/agent 0.0.12 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.d.ts +10 -1
- package/dist/catalog.d.ts.map +1 -1
- package/dist/catalog.js +11 -2
- package/dist/delegate.d.ts +29 -0
- package/dist/delegate.d.ts.map +1 -0
- package/dist/delegate.js +106 -0
- package/dist/extensions/delegation.d.ts +15 -0
- package/dist/extensions/delegation.d.ts.map +1 -0
- package/dist/extensions/delegation.js +93 -0
- package/dist/paths.d.ts +6 -0
- package/dist/paths.d.ts.map +1 -1
- package/dist/paths.js +16 -0
- package/dist/provider/chat-config.d.ts +6 -5
- package/dist/provider/chat-config.d.ts.map +1 -1
- package/dist/provider/chat-config.js +21 -7
- package/dist/provider/index.d.ts.map +1 -1
- package/dist/provider/index.js +8 -1
- package/dist/provider/model-host.d.ts +1 -1
- package/dist/provider/model-host.d.ts.map +1 -1
- package/dist/provider/model-host.js +25 -7
- package/dist/provider/models.d.ts +3 -14
- package/dist/provider/models.d.ts.map +1 -1
- package/dist/provider/models.js +17 -239
- package/dist/provider/stream-adapter.d.ts +2 -2
- package/dist/provider/stream-adapter.d.ts.map +1 -1
- package/dist/provider/stream-adapter.js +8 -5
- package/dist/run-agent.d.ts +4 -0
- package/dist/run-agent.d.ts.map +1 -1
- package/dist/run-agent.js +8 -2
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +23 -5
- package/src/catalog.ts +194 -0
- package/src/cold-tier.ts +152 -0
- package/src/delegate.ts +136 -0
- package/src/extensions/approval-detail.ts +57 -0
- package/src/extensions/delegation.ts +109 -0
- package/src/extensions/local-image-input.ts +132 -0
- package/src/extensions/permission-gate.ts +347 -0
- package/src/extensions/subagent.ts +743 -0
- package/src/extensions/terminal-title.ts +53 -0
- package/src/extensions/trace-notice.ts +37 -0
- package/src/index.ts +23 -0
- package/src/paths.ts +36 -0
- package/src/provider/chat-config.ts +132 -0
- package/src/provider/convert-messages.ts +273 -0
- package/src/provider/error-coercion.ts +36 -0
- package/src/provider/events.ts +341 -0
- package/src/provider/index.ts +255 -0
- package/src/provider/metrics-trace.ts +380 -0
- package/src/provider/mlx-identity.ts +16 -0
- package/src/provider/model-host.ts +276 -0
- package/src/provider/model-registry-filter.ts +336 -0
- package/src/provider/models.ts +48 -0
- package/src/provider/performance-status.ts +112 -0
- package/src/provider/reasoning-tag-buffer.ts +67 -0
- package/src/provider/stream-adapter.ts +515 -0
- package/src/provider/tool-call-buffer.ts +82 -0
- package/src/provider/warm-reuse.ts +125 -0
- package/src/run-agent.ts +178 -0
- package/src/types.ts +10 -0
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `TurnEmitter` — maps one native `ChatStreamEvent` turn onto pi's
|
|
3
|
+
* `AssistantMessageEvent` protocol.
|
|
4
|
+
*
|
|
5
|
+
* Correctness contract (spike-proven):
|
|
6
|
+
* - pi's agent loop takes the final message from `stream.result()` and
|
|
7
|
+
* reads `stopReason` off THAT message — the `done` event's `reason`
|
|
8
|
+
* field is discarded. Both are still emitted protocol-correct.
|
|
9
|
+
* - Aborted native streams end with NO final event, so the caller must
|
|
10
|
+
* invoke {@link TurnEmitter.onAborted} to synthesize the terminal
|
|
11
|
+
* AssistantMessage (stopReason 'aborted', accumulated deltas intact).
|
|
12
|
+
* - Every method is throw-safe: the pi StreamFn contract does not allow
|
|
13
|
+
* the emitter to throw, so internal failures route to {@link onError}.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { randomUUID } from 'node:crypto';
|
|
17
|
+
|
|
18
|
+
import type {
|
|
19
|
+
Api,
|
|
20
|
+
AssistantMessage,
|
|
21
|
+
AssistantMessageEventStream,
|
|
22
|
+
Model,
|
|
23
|
+
TextContent,
|
|
24
|
+
ThinkingContent,
|
|
25
|
+
ToolCall,
|
|
26
|
+
Usage,
|
|
27
|
+
} from '@earendil-works/pi-ai';
|
|
28
|
+
import type { ChatStreamDelta, ChatStreamFinal, PerformanceMetrics, ToolCallResult } from '@mlx-node/lm';
|
|
29
|
+
|
|
30
|
+
import { coerceErrorMessage } from './error-coercion.js';
|
|
31
|
+
import { ReasoningTagBuffer } from './reasoning-tag-buffer.js';
|
|
32
|
+
import { ToolCallTagBuffer } from './tool-call-buffer.js';
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* All-zero usage. Shared with the stream adapter's TurnEmitter-independent
|
|
36
|
+
* failsafe terminal, so it must stay trivially non-throwing.
|
|
37
|
+
*/
|
|
38
|
+
export function emptyUsage(): Usage {
|
|
39
|
+
return {
|
|
40
|
+
input: 0,
|
|
41
|
+
output: 0,
|
|
42
|
+
cacheRead: 0,
|
|
43
|
+
cacheWrite: 0,
|
|
44
|
+
totalTokens: 0,
|
|
45
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* pi auto-compaction thresholds on `usage.totalTokens` vs
|
|
51
|
+
* `model.contextWindow`, so these numbers are load-bearing:
|
|
52
|
+
* input excludes the cache-served prefix, totalTokens is the full
|
|
53
|
+
* prompt + completion. All costs are 0 for local inference.
|
|
54
|
+
*/
|
|
55
|
+
function usageFromFinal(final: ChatStreamFinal): Usage {
|
|
56
|
+
const cachedTokens = final.cachedTokens ?? 0;
|
|
57
|
+
return {
|
|
58
|
+
input: Math.max(0, final.promptTokens - cachedTokens),
|
|
59
|
+
output: final.numTokens,
|
|
60
|
+
cacheRead: cachedTokens,
|
|
61
|
+
cacheWrite: 0,
|
|
62
|
+
reasoning: final.reasoningTokens,
|
|
63
|
+
totalTokens: final.promptTokens + final.numTokens,
|
|
64
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Native `ToolCallResult` → pi `ToolCall`.
|
|
70
|
+
*
|
|
71
|
+
* Non-ok results (invalid_json / missing_name / parse_error) become a
|
|
72
|
+
* deliberately-invalid pi ToolCall so pi's own tool validation fails it
|
|
73
|
+
* and feeds an error tool result back to the model — pi owns that retry
|
|
74
|
+
* loop (spike-proven).
|
|
75
|
+
*/
|
|
76
|
+
function toPiToolCall(call: ToolCallResult): ToolCall {
|
|
77
|
+
if (call.status === 'ok') {
|
|
78
|
+
const args =
|
|
79
|
+
typeof call.arguments === 'object' && call.arguments !== null ? (call.arguments as Record<string, unknown>) : {};
|
|
80
|
+
return { type: 'toolCall', id: call.id, name: call.name, arguments: args };
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
type: 'toolCall',
|
|
84
|
+
id: call.id,
|
|
85
|
+
name: call.name || 'malformed_tool_call',
|
|
86
|
+
arguments: { raw: call.rawContent, error: call.error },
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export class TurnEmitter {
|
|
91
|
+
/**
|
|
92
|
+
* Correlation id for this turn. Minted here, stamped on the partial message
|
|
93
|
+
* as the custom `mlxTraceId` field (survives pi's JSONL round-trip like
|
|
94
|
+
* `mlxThinkingEnabled`), and surfaced so the stream adapter can join a
|
|
95
|
+
* `MetricsTrace` record to the persisted pi message. The pi entry id is not
|
|
96
|
+
* knowable in-turn (pi emits `message_end` before persisting), so this
|
|
97
|
+
* minted id is the durable join key.
|
|
98
|
+
*/
|
|
99
|
+
readonly traceId: string;
|
|
100
|
+
private readonly stream: AssistantMessageEventStream;
|
|
101
|
+
private readonly partial: AssistantMessage;
|
|
102
|
+
private readonly textBuffer = new ToolCallTagBuffer();
|
|
103
|
+
private readonly thinkingBuffer = new ReasoningTagBuffer();
|
|
104
|
+
/**
|
|
105
|
+
* Leading whitespace-only text parked before any text block exists, so
|
|
106
|
+
* a `"\n\n"` emitted right before `<tool_call>` markup never ratifies a
|
|
107
|
+
* whitespace-only text content block (mirrors the server endpoints).
|
|
108
|
+
* Joined onto the first non-whitespace text; dropped at terminal time.
|
|
109
|
+
*/
|
|
110
|
+
private pendingLeadingWhitespace = '';
|
|
111
|
+
private openBlock: TextContent | ThinkingContent | null = null;
|
|
112
|
+
private finished = false;
|
|
113
|
+
|
|
114
|
+
constructor(
|
|
115
|
+
stream: AssistantMessageEventStream,
|
|
116
|
+
model: Model<Api>,
|
|
117
|
+
private readonly onPerformance?: (message: AssistantMessage, performance: PerformanceMetrics) => void,
|
|
118
|
+
thinkingEnabled?: boolean,
|
|
119
|
+
) {
|
|
120
|
+
this.stream = stream;
|
|
121
|
+
const traceId = randomUUID();
|
|
122
|
+
this.traceId = traceId;
|
|
123
|
+
this.partial = {
|
|
124
|
+
role: 'assistant',
|
|
125
|
+
content: [],
|
|
126
|
+
api: model.api,
|
|
127
|
+
provider: model.provider,
|
|
128
|
+
model: model.id,
|
|
129
|
+
usage: emptyUsage(),
|
|
130
|
+
stopReason: 'stop',
|
|
131
|
+
timestamp: Date.now(),
|
|
132
|
+
};
|
|
133
|
+
// Pi persists provider messages as JSON without a provider-specific
|
|
134
|
+
// metadata bag. Keep this small enumerable provenance field on the
|
|
135
|
+
// assistant object so a later full-history replay can distinguish a
|
|
136
|
+
// disabled-thinking empty channel from an enabled-thinking turn that
|
|
137
|
+
// simply emitted no reasoning. Unknown fields survive Pi's session
|
|
138
|
+
// JSONL round-trip and are ignored by other providers.
|
|
139
|
+
if (thinkingEnabled !== undefined) {
|
|
140
|
+
(this.partial as AssistantMessage & { mlxThinkingEnabled?: boolean }).mlxThinkingEnabled = thinkingEnabled;
|
|
141
|
+
}
|
|
142
|
+
// Same custom-field mechanism: stamp the trace id so a dashboard can join
|
|
143
|
+
// the durable MetricsTrace record back to this persisted pi message.
|
|
144
|
+
(this.partial as AssistantMessage & { mlxTraceId?: string }).mlxTraceId = traceId;
|
|
145
|
+
stream.push({ type: 'start', partial: this.partial });
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
onDelta(delta: ChatStreamDelta): void {
|
|
149
|
+
if (this.finished) return;
|
|
150
|
+
try {
|
|
151
|
+
if (delta.isReasoning === true) {
|
|
152
|
+
this.appendThinking(this.thinkingBuffer.push(delta.text));
|
|
153
|
+
} else {
|
|
154
|
+
// A reasoning suffix that only resembled a partial protocol tag is
|
|
155
|
+
// ordinary text. Release it before opening the visible-text block.
|
|
156
|
+
this.appendThinking(this.thinkingBuffer.flush());
|
|
157
|
+
// Text routes through the tag buffer: partial structural markup
|
|
158
|
+
// (`<tool_call>` etc.) must never leak into pi-visible text.
|
|
159
|
+
const { safeText, tagFound, cleanPrefix } = this.textBuffer.push(delta.text);
|
|
160
|
+
if (tagFound) {
|
|
161
|
+
this.appendVisibleText(cleanPrefix);
|
|
162
|
+
} else if (safeText) {
|
|
163
|
+
this.appendVisibleText(safeText);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
} catch (err) {
|
|
167
|
+
this.onError(err);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
onFinal(final: ChatStreamFinal): void {
|
|
172
|
+
if (this.finished) return;
|
|
173
|
+
try {
|
|
174
|
+
this.partial.usage = usageFromFinal(final);
|
|
175
|
+
if (final.finishReason === 'error') {
|
|
176
|
+
this.finishWithError('error', 'model reported finishReason=error');
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// Release held-back suffixes from both structural-tag buffers, then
|
|
181
|
+
// close the open block.
|
|
182
|
+
this.appendThinking(this.thinkingBuffer.flush());
|
|
183
|
+
this.appendVisibleText(this.textBuffer.flush());
|
|
184
|
+
this.closeOpenBlock();
|
|
185
|
+
|
|
186
|
+
let sawOkToolCall = false;
|
|
187
|
+
for (const call of final.toolCalls) {
|
|
188
|
+
if (call.status === 'ok') sawOkToolCall = true;
|
|
189
|
+
const toolCall = toPiToolCall(call);
|
|
190
|
+
this.partial.content.push(toolCall);
|
|
191
|
+
const contentIndex = this.partial.content.length - 1;
|
|
192
|
+
this.stream.push({ type: 'toolcall_start', contentIndex, partial: this.partial });
|
|
193
|
+
this.stream.push({
|
|
194
|
+
type: 'toolcall_delta',
|
|
195
|
+
contentIndex,
|
|
196
|
+
delta: JSON.stringify(toolCall.arguments),
|
|
197
|
+
partial: this.partial,
|
|
198
|
+
});
|
|
199
|
+
this.stream.push({ type: 'toolcall_end', contentIndex, toolCall, partial: this.partial });
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const reason = sawOkToolCall ? 'toolUse' : final.finishReason === 'length' ? 'length' : 'stop';
|
|
203
|
+
this.partial.stopReason = reason;
|
|
204
|
+
if (final.performance !== undefined && this.onPerformance !== undefined) {
|
|
205
|
+
try {
|
|
206
|
+
this.onPerformance(this.partial, final.performance);
|
|
207
|
+
} catch {
|
|
208
|
+
// Footer telemetry is best-effort and must never break inference.
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
this.finished = true;
|
|
212
|
+
this.stream.push({ type: 'done', reason, message: this.partial });
|
|
213
|
+
this.stream.end();
|
|
214
|
+
} catch (err) {
|
|
215
|
+
this.onError(err);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Synthesize the terminal message for an aborted native stream (which
|
|
221
|
+
* ends with no final event). Mirrors pi's provider abort pattern:
|
|
222
|
+
* `{type:'error', reason:'aborted', error: <partial message>}` with all
|
|
223
|
+
* accumulated text/thinking preserved on the message.
|
|
224
|
+
*/
|
|
225
|
+
onAborted(): void {
|
|
226
|
+
if (this.finished) return;
|
|
227
|
+
this.finishWithError('aborted', 'Request was aborted');
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Terminal for internal/adapter failures. `err` is untrusted: coercion
|
|
232
|
+
* is fully guarded (shared `coerceErrorMessage`), so even a revoked
|
|
233
|
+
* Proxy or a poisoned `message` getter cannot throw out of here.
|
|
234
|
+
*/
|
|
235
|
+
onError(err: unknown): void {
|
|
236
|
+
if (this.finished) return;
|
|
237
|
+
this.finishWithError('error', coerceErrorMessage(err));
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Shared terminal path for every non-`done` ending (abort, native
|
|
242
|
+
* finishReason=error, internal failure). Recovers any held-back buffer
|
|
243
|
+
* residue and closes the open text/thinking block BEFORE emitting the
|
|
244
|
+
* terminal event — a stream must never end `text_start/text_delta/error`
|
|
245
|
+
* with no `text_end` (pi's reference providers balance all blocks
|
|
246
|
+
* before terminals).
|
|
247
|
+
*/
|
|
248
|
+
private finishWithError(reason: 'aborted' | 'error', message: string): void {
|
|
249
|
+
this.finished = true;
|
|
250
|
+
try {
|
|
251
|
+
try {
|
|
252
|
+
this.appendThinking(this.thinkingBuffer.flush());
|
|
253
|
+
this.appendVisibleText(this.textBuffer.flush());
|
|
254
|
+
} catch {
|
|
255
|
+
// Preserving buffered residue is best-effort; the block close and
|
|
256
|
+
// terminal event below must still go out.
|
|
257
|
+
}
|
|
258
|
+
this.closeOpenBlock();
|
|
259
|
+
this.partial.stopReason = reason;
|
|
260
|
+
this.partial.errorMessage = message;
|
|
261
|
+
this.stream.push({ type: 'error', reason, error: this.partial });
|
|
262
|
+
this.stream.end();
|
|
263
|
+
} catch {
|
|
264
|
+
// The StreamFn contract forbids throwing; a push/end failure here
|
|
265
|
+
// has no further recovery surface.
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
private appendThinking(text: string): void {
|
|
270
|
+
if (!text) return;
|
|
271
|
+
let block = this.openBlock;
|
|
272
|
+
if (block?.type !== 'thinking') {
|
|
273
|
+
this.closeOpenBlock();
|
|
274
|
+
block = { type: 'thinking', thinking: '' };
|
|
275
|
+
this.partial.content.push(block);
|
|
276
|
+
this.openBlock = block;
|
|
277
|
+
this.stream.push({ type: 'thinking_start', contentIndex: this.blockIndex(), partial: this.partial });
|
|
278
|
+
}
|
|
279
|
+
block.thinking += text;
|
|
280
|
+
this.stream.push({
|
|
281
|
+
type: 'thinking_delta',
|
|
282
|
+
contentIndex: this.blockIndex(),
|
|
283
|
+
delta: text,
|
|
284
|
+
partial: this.partial,
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
private appendVisibleText(text: string): void {
|
|
289
|
+
if (this.openBlock?.type === 'text') {
|
|
290
|
+
if (!text) return;
|
|
291
|
+
this.openBlock.text += text;
|
|
292
|
+
this.stream.push({ type: 'text_delta', contentIndex: this.blockIndex(), delta: text, partial: this.partial });
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
// No text block open yet: whitespace-only text stays parked so it can
|
|
296
|
+
// front real text later or be dropped silently at tag/terminal time.
|
|
297
|
+
const combined = this.pendingLeadingWhitespace + text;
|
|
298
|
+
if (combined.trim().length === 0) {
|
|
299
|
+
this.pendingLeadingWhitespace = combined;
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
this.pendingLeadingWhitespace = '';
|
|
303
|
+
this.closeOpenBlock();
|
|
304
|
+
const block: TextContent = { type: 'text', text: '' };
|
|
305
|
+
this.partial.content.push(block);
|
|
306
|
+
this.openBlock = block;
|
|
307
|
+
this.stream.push({ type: 'text_start', contentIndex: this.blockIndex(), partial: this.partial });
|
|
308
|
+
block.text += combined;
|
|
309
|
+
this.stream.push({
|
|
310
|
+
type: 'text_delta',
|
|
311
|
+
contentIndex: this.blockIndex(),
|
|
312
|
+
delta: combined,
|
|
313
|
+
partial: this.partial,
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
private closeOpenBlock(): void {
|
|
318
|
+
const block = this.openBlock;
|
|
319
|
+
if (!block) return;
|
|
320
|
+
this.openBlock = null;
|
|
321
|
+
if (block.type === 'text') {
|
|
322
|
+
this.stream.push({
|
|
323
|
+
type: 'text_end',
|
|
324
|
+
contentIndex: this.partial.content.indexOf(block),
|
|
325
|
+
content: block.text,
|
|
326
|
+
partial: this.partial,
|
|
327
|
+
});
|
|
328
|
+
} else {
|
|
329
|
+
this.stream.push({
|
|
330
|
+
type: 'thinking_end',
|
|
331
|
+
contentIndex: this.partial.content.indexOf(block),
|
|
332
|
+
content: block.thinking,
|
|
333
|
+
partial: this.partial,
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
private blockIndex(): number {
|
|
339
|
+
return this.partial.content.length - 1;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `createMlxProviderExtension` — the pi inline extension that registers
|
|
3
|
+
* the in-process `mlx` provider.
|
|
4
|
+
*
|
|
5
|
+
* Task 8's `runAgent` passes the returned extension into pi's `main()`
|
|
6
|
+
* via `extensionFactories`; pi calls the factory during extension load,
|
|
7
|
+
* and `registerProvider` makes every discovered local model resolvable
|
|
8
|
+
* as `mlx/<dir-name>` with no /login (the literal apiKey marks the
|
|
9
|
+
* models available).
|
|
10
|
+
*
|
|
11
|
+
* Import discipline (load-bearing): pi is import-order sensitive to its
|
|
12
|
+
* config env vars, so this module — which the CLI imports BEFORE those
|
|
13
|
+
* env vars are set — must not runtime-import `@earendil-works/pi-coding-agent`
|
|
14
|
+
* at module top level. Only type-only pi imports appear here; the
|
|
15
|
+
* `ExtensionAPI` value arrives as the factory argument.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { ExtensionAPI, InlineExtension } from '@earendil-works/pi-coding-agent';
|
|
19
|
+
import { coldCacheStats, coldSidecarStats, type ColdCacheStats, type ColdSidecarStats } from '@mlx-node/core';
|
|
20
|
+
|
|
21
|
+
import { canonicalCacheRoot } from '../cold-tier.js';
|
|
22
|
+
import { parseThinkingBudget } from './chat-config.js';
|
|
23
|
+
import { MetricsTrace, type MetricsTraceRecord } from './metrics-trace.js';
|
|
24
|
+
import { MLX_API, MLX_API_KEY, MLX_BASE_URL, MLX_PROVIDER_ID } from './mlx-identity.js';
|
|
25
|
+
import { MlxModelHost } from './model-host.js';
|
|
26
|
+
import type { MlxModelInfo } from './models.js';
|
|
27
|
+
import { PerformanceStatus } from './performance-status.js';
|
|
28
|
+
import { makeMlxStreamSimple, type TurnRecorder } from './stream-adapter.js';
|
|
29
|
+
|
|
30
|
+
/** Read the process-wide cold-tier snapshot; the native addon may be absent (unit tests). */
|
|
31
|
+
function safeColdStats(): ColdCacheStats | undefined {
|
|
32
|
+
try {
|
|
33
|
+
return coldCacheStats();
|
|
34
|
+
} catch {
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Read the process-wide sidecar counters. Separate from {@link safeColdStats}
|
|
41
|
+
* because it reads a different native struct that has no `enabled` gate: the
|
|
42
|
+
* counters live in a plain static, not in the tier, so they are valid even
|
|
43
|
+
* when the tier never opened — which is exactly the case a run with zero
|
|
44
|
+
* sidecars needs distinguished.
|
|
45
|
+
*/
|
|
46
|
+
function safeSidecarStats(): ColdSidecarStats | undefined {
|
|
47
|
+
try {
|
|
48
|
+
return coldSidecarStats();
|
|
49
|
+
} catch {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Injectable seams for {@link createMlxProviderExtension} (unit tests). */
|
|
55
|
+
export interface MlxProviderExtensionDeps {
|
|
56
|
+
/** Process-wide cold-tier reader; defaults to the native addon (absent in unit tests). */
|
|
57
|
+
coldStats?: () => ColdCacheStats | undefined;
|
|
58
|
+
/** Process-wide sidecar-counter reader; defaults to the native addon. */
|
|
59
|
+
sidecarStats?: () => ColdSidecarStats | undefined;
|
|
60
|
+
/** Durable per-turn telemetry sink; defaults to a fresh {@link MetricsTrace}. */
|
|
61
|
+
metricsTrace?: MetricsTrace;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Build the `mlx-provider` inline extension serving `models`. The host
|
|
66
|
+
* (one per process — it owns the single GPU-resident model) is created
|
|
67
|
+
* eagerly so repeated factory invocations can never spawn a second
|
|
68
|
+
* host, but stays lazy about weights: nothing loads until the first
|
|
69
|
+
* `streamSimple` call.
|
|
70
|
+
*/
|
|
71
|
+
export function createMlxProviderExtension(
|
|
72
|
+
models: MlxModelInfo[],
|
|
73
|
+
host?: MlxModelHost,
|
|
74
|
+
deps: MlxProviderExtensionDeps = {},
|
|
75
|
+
): InlineExtension {
|
|
76
|
+
const resolvedHost = host ?? new MlxModelHost(models.map((m) => m.discovered));
|
|
77
|
+
const performanceStatus = new PerformanceStatus();
|
|
78
|
+
const metricsTrace = deps.metricsTrace ?? new MetricsTrace();
|
|
79
|
+
const readColdStats = deps.coldStats ?? safeColdStats;
|
|
80
|
+
const readSidecarStats = deps.sidecarStats ?? safeSidecarStats;
|
|
81
|
+
// This closure outlives Pi runtime replacement. Pi creates a replacement
|
|
82
|
+
// runtime for /new and /resume and reruns inline extension factories; each
|
|
83
|
+
// new factory's session_start updates the root while child sessions keep
|
|
84
|
+
// using this registered stream. The root id doubles as the cache owner and
|
|
85
|
+
// the metrics-trace root; the JSONL path travels alongside it for metrics.
|
|
86
|
+
let rootCacheOwnerId: string | undefined;
|
|
87
|
+
let rootSessionFile: string | undefined;
|
|
88
|
+
|
|
89
|
+
// Cold-tier counters are cumulative since the tier opened. Snapshot them at
|
|
90
|
+
// each turn's native start (`onTurnStart`, fired inside the serialized host
|
|
91
|
+
// closure) and diff at the success terminal, so a turn that aborted or
|
|
92
|
+
// errored — which never reaches `onTurnRecord` — can't leak its restores into
|
|
93
|
+
// the next successful turn's delta. Inference is serialized per process (host
|
|
94
|
+
// promise chain), so by the time a turn snapshots, any prior turn has fully
|
|
95
|
+
// drained. The SYNCHRONOUS counters (hits/misses/bytesRestored/enqueued/
|
|
96
|
+
// corruptions/queueDrops/restoreDeclines) are exact per-turn; bytesWritten,
|
|
97
|
+
// evictions and writeErrors advance on the async writer thread, so those
|
|
98
|
+
// deltas are approximate (documented per field on the record).
|
|
99
|
+
//
|
|
100
|
+
// The SIDECAR counters are snapshotted the same way but are exact without
|
|
101
|
+
// exception: every one of them is recorded inside the native turn finalize,
|
|
102
|
+
// on the calling thread, before the turn returns. They also have no `enabled`
|
|
103
|
+
// gate — they live in a plain process static rather than in the tier — so
|
|
104
|
+
// they read honestly on a run where the tier never opened, which is the run
|
|
105
|
+
// that most needs them.
|
|
106
|
+
let turnStartCold = readColdStats();
|
|
107
|
+
let turnStartSidecar = readSidecarStats();
|
|
108
|
+
const onTurnStart = (): void => {
|
|
109
|
+
turnStartCold = readColdStats();
|
|
110
|
+
turnStartSidecar = readSidecarStats();
|
|
111
|
+
};
|
|
112
|
+
const onTurnRecord: TurnRecorder = ({
|
|
113
|
+
traceId,
|
|
114
|
+
sessionId,
|
|
115
|
+
rootSessionId,
|
|
116
|
+
rootSessionFile,
|
|
117
|
+
model,
|
|
118
|
+
final,
|
|
119
|
+
durationMs,
|
|
120
|
+
queueMs,
|
|
121
|
+
resident,
|
|
122
|
+
}) => {
|
|
123
|
+
const rec: Omit<MetricsTraceRecord, 'v'> = {
|
|
124
|
+
traceId,
|
|
125
|
+
ts: Date.now(),
|
|
126
|
+
sessionId,
|
|
127
|
+
rootSessionId,
|
|
128
|
+
rootSessionFile,
|
|
129
|
+
model,
|
|
130
|
+
durationMs,
|
|
131
|
+
queueMs,
|
|
132
|
+
resident,
|
|
133
|
+
finishReason: final.finishReason,
|
|
134
|
+
promptTokens: final.promptTokens,
|
|
135
|
+
cachedTokens: final.cachedTokens ?? 0,
|
|
136
|
+
outputTokens: final.numTokens,
|
|
137
|
+
reasoningTokens: final.reasoningTokens,
|
|
138
|
+
};
|
|
139
|
+
const perf = final.performance;
|
|
140
|
+
if (perf) {
|
|
141
|
+
rec.ttftMs = perf.ttftMs;
|
|
142
|
+
rec.prefillTps = perf.prefillTokensPerSecond;
|
|
143
|
+
rec.decodeTps = perf.decodeTokensPerSecond;
|
|
144
|
+
rec.mtpCycles = perf.mtpCycles;
|
|
145
|
+
// mlx-vlm-comparable headline accept rate (committed tokens per cycle).
|
|
146
|
+
rec.mtpMeanAccepted = perf.mtpMeanAcceptedTokensTotal;
|
|
147
|
+
}
|
|
148
|
+
const cold = readColdStats();
|
|
149
|
+
if (cold && turnStartCold) {
|
|
150
|
+
rec.coldHits = cold.hits - turnStartCold.hits;
|
|
151
|
+
rec.coldMisses = cold.misses - turnStartCold.misses;
|
|
152
|
+
rec.coldBytesWritten = cold.bytesWritten - turnStartCold.bytesWritten;
|
|
153
|
+
rec.coldBytesRestored = cold.bytesRestored - turnStartCold.bytesRestored;
|
|
154
|
+
rec.coldEnqueued = cold.enqueued - turnStartCold.enqueued;
|
|
155
|
+
rec.coldQueueDrops = cold.queueDrops - turnStartCold.queueDrops;
|
|
156
|
+
rec.coldEvictions = cold.evictions - turnStartCold.evictions;
|
|
157
|
+
rec.coldCorruptions = cold.corruptions - turnStartCold.corruptions;
|
|
158
|
+
rec.coldWriteErrors = cold.writeErrors - turnStartCold.writeErrors;
|
|
159
|
+
rec.coldRestoreDeclines = cold.restoreDeclines - turnStartCold.restoreDeclines;
|
|
160
|
+
// Absolutes, not deltas: an aborted/errored turn never reaches this
|
|
161
|
+
// recorder, so a corruption or a dropped write during one lands in NO
|
|
162
|
+
// delta. The cumulative counter observed by the next successful turn
|
|
163
|
+
// still carries it, which is what makes "corruptions must be 0"
|
|
164
|
+
// checkable at all (`MAX(total) > 0` over any window).
|
|
165
|
+
rec.coldCorruptionsTotal = cold.corruptions;
|
|
166
|
+
rec.coldQueueDropsTotal = cold.queueDrops;
|
|
167
|
+
// Same latch, and the counter that needs it most: a write error is
|
|
168
|
+
// raised on the background writer, so the one covering the LAST turn
|
|
169
|
+
// before a crash lands in no delta at all — and "is my cache root
|
|
170
|
+
// broken?" is a question about ever, not about this turn.
|
|
171
|
+
rec.coldWriteErrorsTotal = cold.writeErrors;
|
|
172
|
+
// Cache IDENTITY comes from the END-of-turn snapshot, never the baseline:
|
|
173
|
+
// the tier opens LAZILY on first use, so on a process's first turn
|
|
174
|
+
// `turnStartCold` is the all-zero default with `enabled: false` and an
|
|
175
|
+
// empty root. Canonicalized here — in the writer — so the dashboard
|
|
176
|
+
// never has to match whatever spelling Rust happened to construct.
|
|
177
|
+
rec.coldEnabled = cold.enabled;
|
|
178
|
+
// ONE emptiness test, applied to the CANONICAL value. Gating on the raw
|
|
179
|
+
// native string used a DIFFERENT test from the one `canonicalCacheRoot`
|
|
180
|
+
// applies (it trims), so a whitespace-only root passed `length > 0` here
|
|
181
|
+
// and canonicalized to `''` — which `MetricsTrace.record` then dropped,
|
|
182
|
+
// leaving a row that says the tier was ON while carrying no root at all.
|
|
183
|
+
if (cold.enabled) {
|
|
184
|
+
const canonicalRoot = canonicalCacheRoot(cold.root);
|
|
185
|
+
if (canonicalRoot.length > 0) rec.coldRoot = canonicalRoot;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
// Its own guard, deliberately not folded into the block above: the sidecar
|
|
189
|
+
// reader never consults the tier, so it keeps reporting when `cold` is
|
|
190
|
+
// undefined or the tier failed to open. A run with `coldHits: 0` and no
|
|
191
|
+
// tier is exactly when "did the capture even run?" is the question, and
|
|
192
|
+
// gating these on `cold` would blank them precisely then.
|
|
193
|
+
const sidecar = readSidecarStats();
|
|
194
|
+
if (sidecar && turnStartSidecar) {
|
|
195
|
+
rec.coldSidecarCaptureReached = sidecar.captureReached - turnStartSidecar.captureReached;
|
|
196
|
+
rec.coldSidecarChainEmpty = sidecar.chainEmpty - turnStartSidecar.chainEmpty;
|
|
197
|
+
rec.coldSidecarBoundarySkips = sidecar.boundarySkips - turnStartSidecar.boundarySkips;
|
|
198
|
+
rec.coldSidecarAlreadyPersisted = sidecar.alreadyPersisted - turnStartSidecar.alreadyPersisted;
|
|
199
|
+
rec.coldSidecarEnqueued = sidecar.enqueued - turnStartSidecar.enqueued;
|
|
200
|
+
rec.coldSidecarQueueDrops = sidecar.queueDrops - turnStartSidecar.queueDrops;
|
|
201
|
+
rec.coldSidecarInstalled = sidecar.installed - turnStartSidecar.installed;
|
|
202
|
+
rec.coldSidecarRestoreSuppressed = sidecar.restoreSuppressed - turnStartSidecar.restoreSuppressed;
|
|
203
|
+
}
|
|
204
|
+
metricsTrace.record(rec);
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
let getThinkingBudget = (): number | undefined => undefined;
|
|
208
|
+
const streamSimple = makeMlxStreamSimple(
|
|
209
|
+
resolvedHost,
|
|
210
|
+
performanceStatus.record,
|
|
211
|
+
() => rootCacheOwnerId,
|
|
212
|
+
onTurnRecord,
|
|
213
|
+
onTurnStart,
|
|
214
|
+
() => rootSessionFile,
|
|
215
|
+
() => getThinkingBudget(),
|
|
216
|
+
);
|
|
217
|
+
return {
|
|
218
|
+
name: 'mlx-provider',
|
|
219
|
+
factory: (pi: ExtensionAPI) => {
|
|
220
|
+
pi.registerFlag('thinking-budget', {
|
|
221
|
+
type: 'string',
|
|
222
|
+
description: 'Maximum reasoning tokens per model turn; 0 closes thinking immediately.',
|
|
223
|
+
});
|
|
224
|
+
getThinkingBudget = () => parseThinkingBudget(pi.getFlag('thinking-budget'));
|
|
225
|
+
pi.registerProvider(MLX_PROVIDER_ID, {
|
|
226
|
+
api: MLX_API,
|
|
227
|
+
baseUrl: MLX_BASE_URL,
|
|
228
|
+
apiKey: MLX_API_KEY,
|
|
229
|
+
streamSimple,
|
|
230
|
+
models: models.map((m) => m.piModel),
|
|
231
|
+
});
|
|
232
|
+
pi.on('session_start', (_event, ctx) => {
|
|
233
|
+
rootCacheOwnerId = ctx.sessionManager.getSessionId();
|
|
234
|
+
// Snapshot the root JSONL path so a turn submitted under this root is
|
|
235
|
+
// correlated to it at completion — even for subagent turns, which have
|
|
236
|
+
// no session file of their own. `getSessionFile` is optional-chained so
|
|
237
|
+
// a minimal test/mock session manager without it still works.
|
|
238
|
+
rootSessionFile = ctx.sessionManager.getSessionFile?.();
|
|
239
|
+
});
|
|
240
|
+
pi.on('message_end', (event, ctx) => {
|
|
241
|
+
performanceStatus.showMessage(event, ctx);
|
|
242
|
+
});
|
|
243
|
+
// Do not clear on turn_start. Pi emits a fresh turn after every tool
|
|
244
|
+
// result, before the next inference has terminal metrics to replace the
|
|
245
|
+
// completed sample; clearing here makes the footer disappear precisely
|
|
246
|
+
// while a long tool-follow-up prefill is running.
|
|
247
|
+
pi.on('model_select', (_event, ctx) => {
|
|
248
|
+
performanceStatus.clear(ctx);
|
|
249
|
+
});
|
|
250
|
+
pi.on('session_shutdown', (_event, ctx) => {
|
|
251
|
+
performanceStatus.clear(ctx);
|
|
252
|
+
});
|
|
253
|
+
},
|
|
254
|
+
};
|
|
255
|
+
}
|