lex-llm 0.6.11 → 0.6.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/lib/legion/extensions/llm/canonical/chunk.rb +9 -6
- data/lib/legion/extensions/llm/message.rb +2 -1
- data/lib/legion/extensions/llm/responses/thinking_extractor.rb +25 -2
- data/lib/legion/extensions/llm/stream_accumulator.rb +61 -1
- data/lib/legion/extensions/llm/streaming.rb +8 -2
- data/lib/legion/extensions/llm/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '0307598766a4f750febc8f0e72b2ebfa344291eb7feb3f134b2f7f1fffcb7620'
|
|
4
|
+
data.tar.gz: 713d7789cc022b01300ab4cc75811c43ea297ec817bb2780ce046426328155e9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ef02b0c8ac4f2850a2823982d564ea6ca0a9f6e9c5fb23180231080ff2291f22b242848ad700b889fe7a721d555c2a8d10f55b9f057e11ea0f4c67f3348df6a4
|
|
7
|
+
data.tar.gz: cb4a22c7c6c8e4f151f09c7cb4f6b6b445a8faa085936344c747d7a90b648620f79673435731f9fe3f311e95b804e16c2af439b577aaa180ad2ae1d20b052721
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.6.13 - 2026-07-24
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- **StreamAccumulator now captures `stop_reason` from provider done chunks.** Previously, the real `finish_reason` from the provider was silently discarded — the accumulator had no field for it, and the legacy `Message` class couldn't carry it. Downstream code always synthesized `:end_turn` regardless of what the provider actually said. Now `stop_reason` propagates through accumulator → Message → `normalize_response`.
|
|
7
|
+
- **Canonical::Chunk factory methods accept `stop_reason:` and `usage:` kwargs.** `text_delta`, `thinking_delta`, and `tool_call_delta` factories now pass through stop_reason and usage when present on the SSE event, enabling translators to propagate finish_reason on non-empty chunks without buffering.
|
|
8
|
+
- **Streaming handler iterates array results from `build_chunk`.** When a translator returns multiple chunks from a single SSE event (e.g. both thinking and content on the same delta), the handler now yields each one to the accumulator instead of expecting a single chunk. Prevents content loss at thinking/content boundaries.
|
|
9
|
+
|
|
10
|
+
## 0.6.12 - 2026-07-15
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
- **Strip leaked Gemma4 special tokens from response content.** Gemma4 models emit `<turn|>`, `<|turn>`, `<channel|>` as literal text when the serving engine fails to intercept them. `<turn|>` and `<channel|>` are end-of-turn signals — content is truncated at the first occurrence (everything after it is garbage). Other leaked tokens (`<|channel>`, `<|turn|>`) are stripped entirely. Prevents users seeing raw control tokens in responses and fixes cases where the leaked token caused downstream "no response returned" errors.
|
|
14
|
+
- **Remove `<|channel>` / `<channel|>` from `THINK_TAG_PAIRS`.** The channel delimiter is NOT always a thinking block — `<|channel>` is a generic channel marker and only becomes thinking when followed by `thought\n`. Having `<channel|>` as a close tag in `THINK_TAG_PAIRS` caused the extractor to consume ALL content before an orphaned `<channel|>` stop token as thinking, producing empty content (dead stop). Now handled only as a leaked stop token to truncate at.
|
|
15
|
+
|
|
3
16
|
## 0.6.11 - 2026-07-15
|
|
4
17
|
|
|
5
18
|
### Added
|
|
@@ -16,39 +16,42 @@ module Legion
|
|
|
16
16
|
) do
|
|
17
17
|
# Build a text delta chunk.
|
|
18
18
|
def self.text_delta(delta:, request_id:, conversation_id: nil, exchange_id: nil,
|
|
19
|
-
index: 0, block_index: nil, item_id: nil
|
|
19
|
+
index: 0, block_index: nil, item_id: nil,
|
|
20
|
+
stop_reason: nil, usage: nil)
|
|
20
21
|
new(
|
|
21
22
|
type: :text_delta, delta: delta, index: index,
|
|
22
23
|
request_id: request_id, conversation_id: conversation_id,
|
|
23
24
|
exchange_id: exchange_id, block_index: block_index,
|
|
24
25
|
item_id: item_id, tool_call: nil, signature: nil,
|
|
25
|
-
usage:
|
|
26
|
+
usage: usage, stop_reason: stop_reason, metadata: {},
|
|
26
27
|
timestamp: ::Time.now
|
|
27
28
|
)
|
|
28
29
|
end
|
|
29
30
|
|
|
30
31
|
# Build a thinking delta chunk.
|
|
31
32
|
def self.thinking_delta(delta:, request_id:, conversation_id: nil, exchange_id: nil,
|
|
32
|
-
index: 0, block_index: nil, item_id: nil, signature: nil
|
|
33
|
+
index: 0, block_index: nil, item_id: nil, signature: nil,
|
|
34
|
+
stop_reason: nil, usage: nil)
|
|
33
35
|
new(
|
|
34
36
|
type: :thinking_delta, delta: delta, index: index,
|
|
35
37
|
request_id: request_id, conversation_id: conversation_id,
|
|
36
38
|
exchange_id: exchange_id, block_index: block_index,
|
|
37
39
|
item_id: item_id, tool_call: nil, signature: signature,
|
|
38
|
-
usage:
|
|
40
|
+
usage: usage, stop_reason: stop_reason, metadata: {},
|
|
39
41
|
timestamp: ::Time.now
|
|
40
42
|
)
|
|
41
43
|
end
|
|
42
44
|
|
|
43
45
|
# Build a tool_call_delta chunk (supports multiple in-flight tool calls via tool_call.id).
|
|
44
46
|
def self.tool_call_delta(tool_call:, request_id:, conversation_id: nil, exchange_id: nil,
|
|
45
|
-
index: 0, block_index: nil, item_id: nil
|
|
47
|
+
index: 0, block_index: nil, item_id: nil,
|
|
48
|
+
stop_reason: nil, usage: nil)
|
|
46
49
|
new(
|
|
47
50
|
type: :tool_call_delta, index: index,
|
|
48
51
|
request_id: request_id, conversation_id: conversation_id,
|
|
49
52
|
exchange_id: exchange_id, block_index: block_index,
|
|
50
53
|
item_id: item_id, delta: nil, tool_call: tool_call, signature: nil,
|
|
51
|
-
usage:
|
|
54
|
+
usage: usage, stop_reason: stop_reason, metadata: {},
|
|
52
55
|
timestamp: ::Time.now
|
|
53
56
|
)
|
|
54
57
|
end
|
|
@@ -7,7 +7,7 @@ module Legion
|
|
|
7
7
|
class Message
|
|
8
8
|
ROLES = %i[system user assistant tool].freeze
|
|
9
9
|
|
|
10
|
-
attr_reader :role, :model_id, :tool_calls, :tool_call_id, :raw, :thinking, :tokens
|
|
10
|
+
attr_reader :role, :model_id, :tool_calls, :tool_call_id, :raw, :thinking, :tokens, :stop_reason
|
|
11
11
|
attr_writer :content
|
|
12
12
|
|
|
13
13
|
def initialize(options = {})
|
|
@@ -26,6 +26,7 @@ module Legion
|
|
|
26
26
|
)
|
|
27
27
|
@raw = options[:raw]
|
|
28
28
|
@thinking = options[:thinking]
|
|
29
|
+
@stop_reason = options[:stop_reason]
|
|
29
30
|
|
|
30
31
|
ensure_valid_role
|
|
31
32
|
end
|
|
@@ -10,9 +10,14 @@ module Legion
|
|
|
10
10
|
|
|
11
11
|
THINK_TAG_PAIRS = [
|
|
12
12
|
['<thinking>', '</thinking>'],
|
|
13
|
-
['<think>', '</think>']
|
|
14
|
-
['<|channel>', '<channel|>']
|
|
13
|
+
['<think>', '</think>']
|
|
15
14
|
].freeze
|
|
15
|
+
# Gemma4 special tokens that leak into content text when the serving
|
|
16
|
+
# engine fails to intercept them. <turn|> and <channel|> are stop
|
|
17
|
+
# signals — content is truncated at the first occurrence.
|
|
18
|
+
# Others are stripped entirely.
|
|
19
|
+
LEAKED_STOP_TOKENS = ['<turn|>', '<|turn>', '<channel|>'].freeze
|
|
20
|
+
LEAKED_STRIP_TOKENS = ['<|channel>', '<|turn|>'].freeze
|
|
16
21
|
UNTAGGED_PREAMBLE_MAX_LENGTH = 4_000
|
|
17
22
|
UNTAGGED_PREAMBLE_STARTS = [
|
|
18
23
|
'the user',
|
|
@@ -76,10 +81,28 @@ module Legion
|
|
|
76
81
|
clean, untagged_thinking = extract_untagged_preamble(clean.strip)
|
|
77
82
|
thinking_parts << untagged_thinking
|
|
78
83
|
|
|
84
|
+
clean = truncate_at_leaked_stop_token(clean)
|
|
85
|
+
clean = strip_leaked_tokens(clean)
|
|
86
|
+
|
|
79
87
|
[clean, compact_thinking(thinking_parts)]
|
|
80
88
|
end
|
|
81
89
|
private_class_method :extract_from_content
|
|
82
90
|
|
|
91
|
+
def truncate_at_leaked_stop_token(text)
|
|
92
|
+
earliest = nil
|
|
93
|
+
LEAKED_STOP_TOKENS.each do |token|
|
|
94
|
+
idx = text.index(token)
|
|
95
|
+
earliest = idx if idx && (earliest.nil? || idx < earliest)
|
|
96
|
+
end
|
|
97
|
+
earliest ? text[0, earliest].rstrip : text
|
|
98
|
+
end
|
|
99
|
+
private_class_method :truncate_at_leaked_stop_token
|
|
100
|
+
|
|
101
|
+
def strip_leaked_tokens(text)
|
|
102
|
+
LEAKED_STRIP_TOKENS.reduce(text) { |t, token| t.gsub(token, '') }
|
|
103
|
+
end
|
|
104
|
+
private_class_method :strip_leaked_tokens
|
|
105
|
+
|
|
83
106
|
def extract_untagged_preamble(content)
|
|
84
107
|
return [content, nil] unless content.is_a?(String)
|
|
85
108
|
|
|
@@ -7,13 +7,14 @@ module Legion
|
|
|
7
7
|
class StreamAccumulator
|
|
8
8
|
include Legion::Logging::Helper
|
|
9
9
|
|
|
10
|
-
attr_reader :content, :model_id, :tool_calls
|
|
10
|
+
attr_reader :content, :model_id, :tool_calls, :stop_reason
|
|
11
11
|
|
|
12
12
|
def initialize
|
|
13
13
|
@content = +''
|
|
14
14
|
@thinking_text = +''
|
|
15
15
|
@thinking_signature = nil
|
|
16
16
|
@tool_calls = {}
|
|
17
|
+
@stop_reason = nil
|
|
17
18
|
@input_tokens = nil
|
|
18
19
|
@output_tokens = nil
|
|
19
20
|
@cached_tokens = nil
|
|
@@ -36,6 +37,7 @@ module Legion
|
|
|
36
37
|
handle_chunk_content(chunk)
|
|
37
38
|
append_thinking_from_chunk(chunk)
|
|
38
39
|
count_tokens chunk
|
|
40
|
+
@stop_reason = chunk.stop_reason if chunk.respond_to?(:stop_reason) && chunk.stop_reason
|
|
39
41
|
log.debug { inspect } if Legion::Extensions::Llm.config.log_stream_debug
|
|
40
42
|
end
|
|
41
43
|
|
|
@@ -81,6 +83,21 @@ module Legion
|
|
|
81
83
|
def to_message(response)
|
|
82
84
|
flush_pending_untagged_preamble
|
|
83
85
|
|
|
86
|
+
if content.length < 50
|
|
87
|
+
log.debug '[llm][stream_accumulator] action=short_content_debug ' \
|
|
88
|
+
"content=#{content.inspect} thinking_chars=#{@thinking_text.length} " \
|
|
89
|
+
"tool_calls=#{tool_calls.size} " \
|
|
90
|
+
"inside_think_tag=#{@inside_think_tag} " \
|
|
91
|
+
"pending_think_tag=#{@pending_think_tag.inspect} " \
|
|
92
|
+
"untagged_preamble_pending=#{@untagged_preamble_pending} " \
|
|
93
|
+
"thinking_start=#{@thinking_text[0, 200].inspect} " \
|
|
94
|
+
"thinking_end=#{@thinking_text[-200..].inspect}"
|
|
95
|
+
if content.length < 5 && @thinking_text.length > 100 && tool_calls.empty?
|
|
96
|
+
log.debug '[llm][stream_accumulator] action=POSSIBLE_CONTENT_EATEN ' \
|
|
97
|
+
"full_thinking=#{@thinking_text.inspect}"
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
84
101
|
Message.new(
|
|
85
102
|
role: :assistant,
|
|
86
103
|
content: content.empty? ? nil : content,
|
|
@@ -97,6 +114,7 @@ module Legion
|
|
|
97
114
|
),
|
|
98
115
|
model_id: model_id,
|
|
99
116
|
tool_calls: tool_calls_from_stream,
|
|
117
|
+
stop_reason: @stop_reason,
|
|
100
118
|
raw: response
|
|
101
119
|
)
|
|
102
120
|
end
|
|
@@ -197,10 +215,21 @@ module Legion
|
|
|
197
215
|
|
|
198
216
|
def append_text_with_thinking(text)
|
|
199
217
|
content_chunk, thinking_chunk = extract_think_tags(text)
|
|
218
|
+
if @content.empty? && !content_chunk.empty?
|
|
219
|
+
log.debug '[llm][stream_accumulator] action=first_content_after_extract ' \
|
|
220
|
+
"content_chunk_start=#{content_chunk[0, 50].inspect} " \
|
|
221
|
+
"thinking_chunk_present=#{!thinking_chunk.nil?} " \
|
|
222
|
+
"thinking_text_so_far=#{@thinking_text.length} " \
|
|
223
|
+
"raw_text_start=#{text[0, 50].inspect}"
|
|
224
|
+
end
|
|
200
225
|
content_chunk, untagged_thinking = extract_untagged_preamble(content_chunk)
|
|
201
226
|
@content << content_chunk
|
|
202
227
|
@last_content_delta << content_chunk
|
|
203
228
|
if untagged_thinking
|
|
229
|
+
log.debug '[llm][stream_accumulator] action=untagged_thinking_from_chunk ' \
|
|
230
|
+
"content_kept=#{content_chunk[0, 50].inspect} " \
|
|
231
|
+
"untagged_thinking=#{untagged_thinking[0, 100].inspect} " \
|
|
232
|
+
"inside_think_tag=#{@inside_think_tag}"
|
|
204
233
|
@thinking_text << untagged_thinking
|
|
205
234
|
@last_thinking_delta << untagged_thinking
|
|
206
235
|
end
|
|
@@ -245,6 +274,9 @@ module Legion
|
|
|
245
274
|
|
|
246
275
|
content, thinking = Responses::ThinkingExtractor.extract_untagged_preamble(@untagged_preamble_buffer)
|
|
247
276
|
if thinking
|
|
277
|
+
log.debug '[llm][stream_accumulator] action=untagged_preamble_classified_as_thinking ' \
|
|
278
|
+
"buffer=#{@untagged_preamble_buffer[0, 100].inspect} " \
|
|
279
|
+
"content_kept=#{content[0, 50].inspect} thinking_extracted=#{thinking[0, 100].inspect}"
|
|
248
280
|
@content << content
|
|
249
281
|
@thinking_text << thinking
|
|
250
282
|
else
|
|
@@ -290,6 +322,13 @@ module Legion
|
|
|
290
322
|
output = +''
|
|
291
323
|
thinking = +''
|
|
292
324
|
|
|
325
|
+
if @inside_think_tag && text.length > 10
|
|
326
|
+
log.debug '[llm][stream_accumulator] action=chunk_arrives_inside_think ' \
|
|
327
|
+
"text_length=#{text.length} text=#{text[0, 200].inspect} " \
|
|
328
|
+
"active_close_tag=#{@active_think_close_tag.inspect} " \
|
|
329
|
+
"content_so_far=#{@content.length} thinking_so_far=#{@thinking_text.length}"
|
|
330
|
+
end
|
|
331
|
+
|
|
293
332
|
until remaining.empty?
|
|
294
333
|
remaining = if @inside_think_tag
|
|
295
334
|
consume_think_content(remaining, @active_think_close_tag, thinking)
|
|
@@ -309,6 +348,13 @@ module Legion
|
|
|
309
348
|
@active_think_close_tag = nil
|
|
310
349
|
remaining.slice((end_index + end_tag.length)..) || +''
|
|
311
350
|
else
|
|
351
|
+
consumed = remaining.slice(0, remaining.length - longest_suffix_prefix(remaining, [end_tag]))
|
|
352
|
+
if @content.length.positive? && consumed.length > 20
|
|
353
|
+
log.debug '[llm][stream_accumulator] action=think_consuming_without_close ' \
|
|
354
|
+
"end_tag=#{end_tag.inspect} consumed_chars=#{consumed.length} " \
|
|
355
|
+
"consumed_start=#{consumed[0, 80].inspect} " \
|
|
356
|
+
"total_thinking=#{thinking.length + consumed.length}"
|
|
357
|
+
end
|
|
312
358
|
suffix_len = longest_suffix_prefix(remaining, [end_tag])
|
|
313
359
|
thinking << remaining.slice(0, remaining.length - suffix_len)
|
|
314
360
|
@pending_think_tag = remaining.slice(-suffix_len, suffix_len)
|
|
@@ -322,6 +368,12 @@ module Legion
|
|
|
322
368
|
if unmatched_close && (start_match.nil? || unmatched_close[:index] < start_match[:index])
|
|
323
369
|
consume_unmatched_think_close(remaining, unmatched_close)
|
|
324
370
|
elsif start_match
|
|
371
|
+
if @content.length > 10 || output.length > 10
|
|
372
|
+
log.debug '[llm][stream_accumulator] action=think_tag_opened_mid_content ' \
|
|
373
|
+
"tag=#{start_match[:tag].inspect} " \
|
|
374
|
+
"content_before_tag=#{remaining.slice(0, start_match[:index])[0, 50].inspect} " \
|
|
375
|
+
"content_accumulated=#{@content.length} output_accumulated=#{output.length}"
|
|
376
|
+
end
|
|
325
377
|
output << remaining.slice(0, start_match[:index])
|
|
326
378
|
@inside_think_tag = true
|
|
327
379
|
@active_think_close_tag = start_match[:close_tag]
|
|
@@ -336,6 +388,14 @@ module Legion
|
|
|
336
388
|
|
|
337
389
|
def consume_unmatched_think_close(remaining, close_match)
|
|
338
390
|
thinking = remaining.slice(0, close_match[:index])
|
|
391
|
+
if thinking.length > 5
|
|
392
|
+
log.debug '[llm][stream_accumulator] action=unmatched_close_eating_content ' \
|
|
393
|
+
"close_tag=#{close_match[:tag].inspect} " \
|
|
394
|
+
"eaten_chars=#{thinking.length} " \
|
|
395
|
+
"eaten_start=#{thinking[0, 80].inspect} " \
|
|
396
|
+
"inside_think_tag=#{@inside_think_tag} " \
|
|
397
|
+
"content_so_far=#{@content.length}"
|
|
398
|
+
end
|
|
339
399
|
@thinking_text << thinking
|
|
340
400
|
@last_thinking_delta << thinking
|
|
341
401
|
remaining.slice((close_match[:index] + close_match[:tag].length)..).to_s.sub(/\A[[:space:]]+/, '')
|
|
@@ -48,8 +48,14 @@ module Legion
|
|
|
48
48
|
build_on_data_handler do |data|
|
|
49
49
|
next unless data.is_a?(Hash)
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
result = build_chunk(data)
|
|
52
|
+
next unless result
|
|
53
|
+
|
|
54
|
+
if result.is_a?(Array)
|
|
55
|
+
result.each { |chunk| block.call(chunk) if chunk }
|
|
56
|
+
else
|
|
57
|
+
block.call(result)
|
|
58
|
+
end
|
|
53
59
|
end
|
|
54
60
|
end
|
|
55
61
|
|