@fugood/buttress-server 2.26.0-beta.21 → 2.26.0-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/mlx-bridge.py
CHANGED
|
@@ -454,6 +454,7 @@ def handle_generate(req_id, params):
|
|
|
454
454
|
**kwargs,
|
|
455
455
|
)
|
|
456
456
|
|
|
457
|
+
should_stop = False
|
|
457
458
|
for response in gen_iter:
|
|
458
459
|
if is_cancelled(req_id):
|
|
459
460
|
clear_cancel(req_id)
|
|
@@ -504,13 +505,22 @@ def handle_generate(req_id, params):
|
|
|
504
505
|
"interrupted": False,
|
|
505
506
|
}
|
|
506
507
|
if last_response:
|
|
508
|
+
generation_tokens = getattr(last_response, "generation_tokens", 0)
|
|
509
|
+
finish_reason = getattr(last_response, "finish_reason", None)
|
|
507
510
|
result_data.update({
|
|
508
511
|
"prompt_tokens": getattr(last_response, "prompt_tokens", 0),
|
|
509
512
|
"prompt_tps": getattr(last_response, "prompt_tps", 0),
|
|
510
|
-
"generation_tokens":
|
|
513
|
+
"generation_tokens": generation_tokens,
|
|
511
514
|
"generation_tps": getattr(last_response, "generation_tps", 0),
|
|
512
515
|
"peak_memory": getattr(last_response, "peak_memory", 0),
|
|
513
|
-
"finish_reason":
|
|
516
|
+
"finish_reason": finish_reason,
|
|
517
|
+
# Same name as llama.node's parallel result, which the completion routes read.
|
|
518
|
+
# mlx_lm reports "length" at max_tokens; mlx_vlm reports no reason, so a run
|
|
519
|
+
# that produced max_tokens tokens without a stop sequence hit the limit.
|
|
520
|
+
"stopped_limit": not should_stop and (
|
|
521
|
+
finish_reason == "length"
|
|
522
|
+
or (finish_reason is None and 0 < max_tokens <= generation_tokens)
|
|
523
|
+
),
|
|
514
524
|
})
|
|
515
525
|
|
|
516
526
|
send({"id": req_id, "event": "result", "data": result_data})
|