lex-llm 0.5.0 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -0
- data/lib/legion/extensions/llm/canonical/message.rb +16 -3
- data/lib/legion/extensions/llm/canonical/tool_definition.rb +26 -1
- data/lib/legion/extensions/llm/canonical/tool_schema.rb +46 -0
- data/lib/legion/extensions/llm/canonical/usage.rb +13 -0
- data/lib/legion/extensions/llm/canonical.rb +1 -0
- data/lib/legion/extensions/llm/capability_policy.rb +107 -0
- data/lib/legion/extensions/llm/configuration.rb +4 -0
- data/lib/legion/extensions/llm/error.rb +2 -0
- data/lib/legion/extensions/llm/provider/open_ai_compatible.rb +9 -8
- data/lib/legion/extensions/llm/provider.rb +33 -5
- data/lib/legion/extensions/llm/provider_contract.rb +10 -1
- data/lib/legion/extensions/llm/routing/model_offering.rb +14 -2
- data/lib/legion/extensions/llm/stream_accumulator.rb +39 -0
- data/lib/legion/extensions/llm/streaming.rb +36 -3
- data/lib/legion/extensions/llm/version.rb +1 -1
- data/lib/legion/extensions/llm.rb +3 -0
- data/spec/legion/extensions/llm/canonical/tool_definition_spec.rb +49 -2
- data/spec/legion/extensions/llm/canonical/tool_schema_spec.rb +83 -0
- data/spec/legion/extensions/llm/canonical/usage_spec.rb +40 -0
- data/spec/legion/extensions/llm/capability_policy_spec.rb +192 -0
- data/spec/legion/extensions/llm/configuration_spec.rb +40 -0
- data/spec/legion/extensions/llm/conformance/client_translator_examples.rb +163 -0
- data/spec/legion/extensions/llm/conformance/fixtures/canonical_server_tool_continuation_request.json +43 -0
- data/spec/legion/extensions/llm/conformance/fixtures/canonical_server_tool_use_response.json +29 -0
- data/spec/legion/extensions/llm/conformance/fixtures/canonical_streaming_server_tool_chunks.json +52 -0
- data/spec/legion/extensions/llm/conformance/provider_tool_rendering_examples.rb +77 -0
- data/spec/legion/extensions/llm/provider/open_ai_compatible_tool_calls_array_spec.rb +68 -0
- data/spec/legion/extensions/llm/provider_spec.rb +55 -3
- data/spec/legion/extensions/llm/routing/model_offering_spec.rb +58 -0
- data/spec/legion/extensions/llm/stream_accumulator_spec.rb +52 -0
- data/spec/legion/extensions/llm/streaming_spec.rb +9 -0
- metadata +10 -1
|
@@ -9,7 +9,7 @@ RSpec.describe Legion::Extensions::Llm::Canonical::ToolDefinition do
|
|
|
9
9
|
|
|
10
10
|
expect(tool.name).to eq('search')
|
|
11
11
|
expect(tool.description).to eq('Search the web')
|
|
12
|
-
expect(tool.parameters).to eq({})
|
|
12
|
+
expect(tool.parameters).to eq(type: 'object', properties: {})
|
|
13
13
|
expect(tool.source).to eq({ type: :builtin })
|
|
14
14
|
end
|
|
15
15
|
|
|
@@ -113,7 +113,7 @@ RSpec.describe Legion::Extensions::Llm::Canonical::ToolDefinition do
|
|
|
113
113
|
entry = {
|
|
114
114
|
name: 'custom',
|
|
115
115
|
description: 'Custom tool',
|
|
116
|
-
parameters: {},
|
|
116
|
+
parameters: { type: 'object', properties: { name: { type: 'string' } } },
|
|
117
117
|
extension: 'custom-ext'
|
|
118
118
|
}
|
|
119
119
|
tool = described_class.from_registry_entry(entry)
|
|
@@ -160,6 +160,53 @@ RSpec.describe Legion::Extensions::Llm::Canonical::ToolDefinition do
|
|
|
160
160
|
end
|
|
161
161
|
end
|
|
162
162
|
|
|
163
|
+
describe '.normalize_parameters' do
|
|
164
|
+
it 'injects type: object when a schema has properties but no type' do
|
|
165
|
+
schema = { properties: { task: { type: 'string' } }, required: ['task'] }
|
|
166
|
+
result = described_class.normalize_parameters(schema)
|
|
167
|
+
expect(result[:type]).to eq('object')
|
|
168
|
+
expect(result[:properties]).to eq(task: { type: 'string' })
|
|
169
|
+
expect(result[:required]).to eq(['task'])
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
it 'passes schemas with an explicit type through unchanged' do
|
|
173
|
+
schema = { type: 'object', properties: { a: { type: 'string' } } }
|
|
174
|
+
expect(described_class.normalize_parameters(schema)).to eq(schema)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
it 'wraps a bare property map under type:object/properties' do
|
|
178
|
+
expect(described_class.normalize_parameters(location: { type: 'string' }))
|
|
179
|
+
.to eq(type: 'object', properties: { location: { type: 'string' } })
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
it 'returns an empty object schema for nil/empty' do
|
|
183
|
+
expect(described_class.normalize_parameters(nil)).to eq(type: 'object', properties: {})
|
|
184
|
+
expect(described_class.normalize_parameters({})).to eq(type: 'object', properties: {})
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
it 'symbolizes top-level string keys' do
|
|
188
|
+
result = described_class.normalize_parameters('properties' => { 'a' => { 'type' => 'string' } })
|
|
189
|
+
expect(result[:type]).to eq('object')
|
|
190
|
+
expect(result).to have_key(:properties)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
it 'leaves composite schemas (oneOf etc.) without forcing type' do
|
|
194
|
+
schema = { oneOf: [{ type: 'string' }, { type: 'integer' }] }
|
|
195
|
+
expect(described_class.normalize_parameters(schema)).to eq(schema)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
it 'normalizes parameters at construction via .build' do
|
|
199
|
+
td = described_class.build(name: 'multi_agent_v1',
|
|
200
|
+
parameters: { properties: { task: { type: 'string' } } })
|
|
201
|
+
expect(td.parameters[:type]).to eq('object')
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
it 'normalizes nil parameters to empty object schema via .build' do
|
|
205
|
+
td = described_class.build(name: 'bare_tool')
|
|
206
|
+
expect(td.parameters).to eq(type: 'object', properties: {})
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
163
210
|
describe 'round-trip' do
|
|
164
211
|
it 'preserves values through from_hash/to_h' do
|
|
165
212
|
original = { name: 'search', description: 'Search', parameters: { type: 'object' } }
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
|
|
5
|
+
RSpec.describe Legion::Extensions::Llm::Canonical::ToolSchema do
|
|
6
|
+
let(:full_schema) { { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] } }
|
|
7
|
+
let(:canonical_tool) do
|
|
8
|
+
Legion::Extensions::Llm::Canonical::ToolDefinition.build(
|
|
9
|
+
name: 'get_weather', description: 'Weather lookup', parameters: full_schema
|
|
10
|
+
)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
describe '.extract' do
|
|
14
|
+
it 'extracts from a Canonical::ToolDefinition' do
|
|
15
|
+
result = described_class.extract(canonical_tool)
|
|
16
|
+
expect(result[:type]).to eq('object')
|
|
17
|
+
expect(result[:properties]).to eq(city: { type: 'string' })
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
it 'extracts from a Hash with :parameters' do
|
|
21
|
+
result = described_class.extract({ parameters: full_schema })
|
|
22
|
+
expect(result[:type]).to eq('object')
|
|
23
|
+
expect(result[:properties]).to eq(city: { type: 'string' })
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
it 'extracts from a Hash with :input_schema' do
|
|
27
|
+
result = described_class.extract({ input_schema: full_schema })
|
|
28
|
+
expect(result[:type]).to eq('object')
|
|
29
|
+
expect(result[:properties]).to eq(city: { type: 'string' })
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
it 'extracts from a Hash with :params_schema' do
|
|
33
|
+
result = described_class.extract({ params_schema: full_schema })
|
|
34
|
+
expect(result[:type]).to eq('object')
|
|
35
|
+
expect(result[:properties]).to eq(city: { type: 'string' })
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
it 'extracts from an object responding to params_schema' do
|
|
39
|
+
tool = Struct.new(:params_schema).new(full_schema)
|
|
40
|
+
result = described_class.extract(tool)
|
|
41
|
+
expect(result[:type]).to eq('object')
|
|
42
|
+
expect(result[:properties]).to eq(city: { type: 'string' })
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
it 'returns empty object schema for nil' do
|
|
46
|
+
expect(described_class.extract(nil)).to eq(type: 'object', properties: {})
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
it 'returns empty object schema for empty hash' do
|
|
50
|
+
expect(described_class.extract({})).to eq(type: 'object', properties: {})
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
describe '.tool_name' do
|
|
55
|
+
it 'gets name from Canonical::ToolDefinition' do
|
|
56
|
+
expect(described_class.tool_name(canonical_tool)).to eq('get_weather')
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
it 'gets name from Hash' do
|
|
60
|
+
expect(described_class.tool_name({ name: 'foo' })).to eq('foo')
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
describe '.tool_description' do
|
|
65
|
+
it 'gets description from Canonical::ToolDefinition' do
|
|
66
|
+
expect(described_class.tool_description(canonical_tool)).to eq('Weather lookup')
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
it 'gets description from Hash' do
|
|
70
|
+
expect(described_class.tool_description({ description: 'bar' })).to eq('bar')
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
describe 'ToolDefinition compatibility readers' do
|
|
75
|
+
it 'params_schema returns normalized parameters' do
|
|
76
|
+
expect(canonical_tool.params_schema).to eq(full_schema)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
it 'input_schema aliases params_schema' do
|
|
80
|
+
expect(canonical_tool.input_schema).to eq(canonical_tool.params_schema)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -114,6 +114,46 @@ RSpec.describe Legion::Extensions::Llm::Canonical::Usage do
|
|
|
114
114
|
end
|
|
115
115
|
end
|
|
116
116
|
|
|
117
|
+
describe 'OpenAI nested details extraction' do
|
|
118
|
+
it 'extracts cached_tokens from prompt_tokens_details (Chat API)' do
|
|
119
|
+
usage = described_class.from_hash(
|
|
120
|
+
prompt_tokens: 1000,
|
|
121
|
+
completion_tokens: 200,
|
|
122
|
+
prompt_tokens_details: { cached_tokens: 800 },
|
|
123
|
+
completion_tokens_details: { reasoning_tokens: 50 }
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
expect(usage.input_tokens).to eq(1000)
|
|
127
|
+
expect(usage.output_tokens).to eq(200)
|
|
128
|
+
expect(usage.cache_read_tokens).to eq(800)
|
|
129
|
+
expect(usage.thinking_tokens).to eq(50)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
it 'extracts cached_tokens from input_tokens_details (Responses API)' do
|
|
133
|
+
usage = described_class.from_hash(
|
|
134
|
+
input_tokens: 500,
|
|
135
|
+
output_tokens: 100,
|
|
136
|
+
input_tokens_details: { cached_tokens: 400 },
|
|
137
|
+
output_tokens_details: { reasoning_tokens: 30 }
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
expect(usage.input_tokens).to eq(500)
|
|
141
|
+
expect(usage.output_tokens).to eq(100)
|
|
142
|
+
expect(usage.cache_read_tokens).to eq(400)
|
|
143
|
+
expect(usage.thinking_tokens).to eq(30)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
it 'prefers top-level cache_read_tokens over nested details' do
|
|
147
|
+
usage = described_class.from_hash(
|
|
148
|
+
input_tokens: 500,
|
|
149
|
+
cache_read_tokens: 300,
|
|
150
|
+
prompt_tokens_details: { cached_tokens: 400 }
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
expect(usage.cache_read_tokens).to eq(300)
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
117
157
|
describe 'round-trip' do
|
|
118
158
|
it 'preserves values through from_hash/to_h' do
|
|
119
159
|
original = { input_tokens: 100, output_tokens: 50, cache_read_tokens: 10 }
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
|
|
5
|
+
RSpec.describe Legion::Extensions::Llm::CapabilityPolicy do
|
|
6
|
+
let(:empty_sources) do
|
|
7
|
+
{ real: {}, provider_catalog: {}, probe: {}, provider_envelope: {}, provider_config: {}, instance_config: {},
|
|
8
|
+
model_config: {} }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
describe '.resolve' do
|
|
12
|
+
context 'with no data at all' do
|
|
13
|
+
it 'defaults all optional capabilities to false' do
|
|
14
|
+
policy = described_class.resolve(**empty_sources)
|
|
15
|
+
|
|
16
|
+
expect(policy[:capabilities]).to eq([])
|
|
17
|
+
expect(policy[:sources][:embeddings]).to eq(value: false, source: :default_false)
|
|
18
|
+
expect(policy[:sources][:thinking]).to eq(value: false, source: :default_false)
|
|
19
|
+
expect(policy[:sources][:streaming]).to eq(value: false, source: :default_false)
|
|
20
|
+
expect(policy[:sources][:tools]).to eq(value: false, source: :default_false)
|
|
21
|
+
expect(policy[:sources][:vision]).to eq(value: false, source: :default_false)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
context 'with instance override' do
|
|
26
|
+
it 'resolves capabilities from instance config' do
|
|
27
|
+
policy = described_class.resolve(
|
|
28
|
+
real: {},
|
|
29
|
+
provider_catalog: {},
|
|
30
|
+
probe: {},
|
|
31
|
+
provider_envelope: {},
|
|
32
|
+
provider_config: {
|
|
33
|
+
capabilities: { embeddings: false },
|
|
34
|
+
tools_flag: false
|
|
35
|
+
},
|
|
36
|
+
instance_config: {
|
|
37
|
+
capabilities: { streaming: true, tools: true },
|
|
38
|
+
enable_thinking: true
|
|
39
|
+
},
|
|
40
|
+
model_config: {}
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
expect(policy[:capabilities]).to contain_exactly(:streaming, :tools, :thinking)
|
|
44
|
+
expect(policy[:sources][:thinking]).to eq(value: true, source: :instance_override)
|
|
45
|
+
expect(policy[:sources][:embeddings]).to eq(value: false, source: :provider_override)
|
|
46
|
+
expect(policy[:sources][:tools]).to eq(value: true, source: :instance_override)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
context 'with provider-level override' do
|
|
51
|
+
it 'resolves capabilities from provider config' do
|
|
52
|
+
policy = described_class.resolve(
|
|
53
|
+
real: {},
|
|
54
|
+
provider_catalog: {},
|
|
55
|
+
probe: {},
|
|
56
|
+
provider_envelope: {},
|
|
57
|
+
provider_config: {
|
|
58
|
+
capabilities: { streaming: true },
|
|
59
|
+
embedding_flag: false,
|
|
60
|
+
thinking_flag: true
|
|
61
|
+
},
|
|
62
|
+
instance_config: {},
|
|
63
|
+
model_config: {}
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
expect(policy[:capabilities]).to contain_exactly(:streaming, :thinking)
|
|
67
|
+
expect(policy[:sources][:streaming]).to eq(value: true, source: :provider_override)
|
|
68
|
+
expect(policy[:sources][:embeddings]).to eq(value: false, source: :provider_override)
|
|
69
|
+
expect(policy[:sources][:thinking]).to eq(value: true, source: :provider_override)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
context 'with full precedence chain' do
|
|
74
|
+
it 'resolves each capability from the highest-priority source' do
|
|
75
|
+
policy = described_class.resolve(
|
|
76
|
+
real: { tools: false, vision: true },
|
|
77
|
+
provider_catalog: { structured_output: true },
|
|
78
|
+
probe: { embeddings: true },
|
|
79
|
+
provider_envelope: { streaming: true, tools: true },
|
|
80
|
+
provider_config: { capabilities: { tools: true, vision: false } },
|
|
81
|
+
instance_config: { capabilities: { tools: false } },
|
|
82
|
+
model_config: { capabilities: { tools: true } }
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
expect(policy[:capabilities]).to include(:tools, :embeddings, :streaming, :structured_output)
|
|
86
|
+
expect(policy[:capabilities]).not_to include(:vision)
|
|
87
|
+
expect(policy[:sources][:tools]).to eq(value: true, source: :model_override)
|
|
88
|
+
expect(policy[:sources][:vision]).to eq(value: false, source: :provider_override)
|
|
89
|
+
expect(policy[:sources][:embeddings]).to eq(value: true, source: :probe)
|
|
90
|
+
expect(policy[:sources][:structured_output]).to eq(value: true, source: :provider_catalog)
|
|
91
|
+
expect(policy[:sources][:streaming]).to eq(value: true, source: :provider_envelope)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
context 'with boolean aliases' do
|
|
96
|
+
it 'resolves enable_* and *_flag aliases' do
|
|
97
|
+
policy = described_class.resolve(
|
|
98
|
+
real: {},
|
|
99
|
+
provider_catalog: {},
|
|
100
|
+
probe: {},
|
|
101
|
+
provider_envelope: {},
|
|
102
|
+
provider_config: {},
|
|
103
|
+
instance_config: { enable_thinking: true, streaming_flag: true, tools_flag: false },
|
|
104
|
+
model_config: {}
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
expect(policy[:capabilities]).to contain_exactly(:streaming, :thinking)
|
|
108
|
+
expect(policy[:sources][:thinking]).to eq(value: true, source: :instance_override)
|
|
109
|
+
expect(policy[:sources][:streaming]).to eq(value: true, source: :instance_override)
|
|
110
|
+
expect(policy[:sources][:tools]).to eq(value: false, source: :instance_override)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
context 'when capabilities hash wins over alias at same level' do
|
|
115
|
+
it 'prefers capabilities nested key over boolean alias' do
|
|
116
|
+
policy = described_class.resolve(
|
|
117
|
+
real: {},
|
|
118
|
+
provider_catalog: {},
|
|
119
|
+
probe: {},
|
|
120
|
+
provider_envelope: {},
|
|
121
|
+
provider_config: {},
|
|
122
|
+
instance_config: { capabilities: { tools: true }, tools_flag: false },
|
|
123
|
+
model_config: {}
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
expect(policy[:capabilities]).to include(:tools)
|
|
127
|
+
expect(policy[:sources][:tools]).to eq(value: true, source: :instance_override)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
context 'with model override' do
|
|
132
|
+
it 'model override beats instance and provider' do
|
|
133
|
+
policy = described_class.resolve(
|
|
134
|
+
real: {},
|
|
135
|
+
provider_catalog: {},
|
|
136
|
+
probe: {},
|
|
137
|
+
provider_envelope: {},
|
|
138
|
+
provider_config: { capabilities: { thinking: false } },
|
|
139
|
+
instance_config: { capabilities: { thinking: false } },
|
|
140
|
+
model_config: { thinking_flag: true }
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
expect(policy[:capabilities]).to include(:thinking)
|
|
144
|
+
expect(policy[:sources][:thinking]).to eq(value: true, source: :model_override)
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
context 'with provider envelope' do
|
|
149
|
+
it 'uses provider envelope when no overrides exist' do
|
|
150
|
+
policy = described_class.resolve(
|
|
151
|
+
real: {},
|
|
152
|
+
provider_catalog: {},
|
|
153
|
+
probe: {},
|
|
154
|
+
provider_envelope: { streaming: true },
|
|
155
|
+
provider_config: {},
|
|
156
|
+
instance_config: {},
|
|
157
|
+
model_config: {}
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
expect(policy[:capabilities]).to contain_exactly(:streaming)
|
|
161
|
+
expect(policy[:sources][:streaming]).to eq(value: true, source: :provider_envelope)
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
describe '.normalized_overrides' do
|
|
167
|
+
it 'handles string keys in capabilities hash' do
|
|
168
|
+
result = described_class.normalized_overrides({ 'capabilities' => { 'streaming' => true } })
|
|
169
|
+
expect(result[:streaming]).to be(true)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
it 'handles symbol keys in capabilities hash' do
|
|
173
|
+
result = described_class.normalized_overrides({ capabilities: { streaming: true } })
|
|
174
|
+
expect(result[:streaming]).to be(true)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
it 'ignores non-boolean values' do
|
|
178
|
+
result = described_class.normalized_overrides({ capabilities: { streaming: 'yes' } })
|
|
179
|
+
expect(result).not_to have_key(:streaming)
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
describe '.normalize_hash' do
|
|
184
|
+
it 'returns empty hash for nil' do
|
|
185
|
+
expect(described_class.normalize_hash(nil)).to eq({})
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
it 'symbolizes keys' do
|
|
189
|
+
expect(described_class.normalize_hash({ 'foo' => 1 })).to eq({ foo: 1 })
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
@@ -35,4 +35,44 @@ RSpec.describe Legion::Extensions::Llm::Configuration do
|
|
|
35
35
|
expect(config.cache_control_prefix_tokens).to eq(4)
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
|
+
|
|
39
|
+
describe '.register_provider_options' do
|
|
40
|
+
after do
|
|
41
|
+
# Clean up test options to avoid polluting other specs
|
|
42
|
+
%i[test_api_key test_api_base].each do |key|
|
|
43
|
+
described_class.send(:option_keys).delete(key)
|
|
44
|
+
described_class.send(:defaults).delete(key)
|
|
45
|
+
described_class.send(:remove_method, key) if described_class.method_defined?(key)
|
|
46
|
+
described_class.send(:remove_method, :"#{key}=") if described_class.method_defined?(:"#{key}=")
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
it 'registers new options that become accessible on instances' do
|
|
51
|
+
described_class.register_provider_options(%i[test_api_key test_api_base])
|
|
52
|
+
|
|
53
|
+
config = described_class.new
|
|
54
|
+
expect(config).to respond_to(:test_api_key)
|
|
55
|
+
expect(config).to respond_to(:test_api_base)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
it 'adds registered options to the options list' do
|
|
59
|
+
described_class.register_provider_options(%i[test_api_key test_api_base])
|
|
60
|
+
|
|
61
|
+
expect(described_class.options).to include(:test_api_key, :test_api_base)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
it 'is idempotent — duplicate registrations do not add duplicates' do
|
|
65
|
+
described_class.register_provider_options(%i[test_api_key])
|
|
66
|
+
described_class.register_provider_options(%i[test_api_key])
|
|
67
|
+
|
|
68
|
+
count = described_class.options.count(:test_api_key)
|
|
69
|
+
expect(count).to eq(1)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
it 'accepts string keys and normalizes them to symbols' do
|
|
73
|
+
described_class.register_provider_options(%w[test_api_key])
|
|
74
|
+
|
|
75
|
+
expect(described_class.options).to include(:test_api_key)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
38
78
|
end
|
|
@@ -265,5 +265,168 @@ RSpec.shared_examples 'a canonical client translator' do |translator_class|
|
|
|
265
265
|
end
|
|
266
266
|
end
|
|
267
267
|
end
|
|
268
|
+
|
|
269
|
+
# G24 — execution-proxy response contract.
|
|
270
|
+
#
|
|
271
|
+
# When a server-executed LegionIO tool resolves before the canonical response
|
|
272
|
+
# is returned, the tool_call carries `:result` and a server-tool source
|
|
273
|
+
# (registry/special/extension/mcp). Client translators MUST surface that
|
|
274
|
+
# exchange as a completed, NON-actionable item — the client must not try to
|
|
275
|
+
# re-execute it. Per format:
|
|
276
|
+
#
|
|
277
|
+
# * Claude /v1/messages — server_tool_use + server_tool_result content
|
|
278
|
+
# blocks (NOT plain tool_use). stop_reason end_turn once all server
|
|
279
|
+
# results are present.
|
|
280
|
+
# * Codex /v1/responses — completed function_call items (or message items)
|
|
281
|
+
# showing name+arguments+result, status 'completed' (NOT 'in_progress'
|
|
282
|
+
# or 'requires_action'). The response status is 'completed', not
|
|
283
|
+
# 'requires_action'.
|
|
284
|
+
# * Codex /v1/chat/completions — finish_reason 'stop' (not 'tool_calls')
|
|
285
|
+
# when only server tools were called and they all have results; the
|
|
286
|
+
# server tool exchange does not appear as actionable tool_calls.
|
|
287
|
+
#
|
|
288
|
+
# Translators declare their family via `g24_format` (one of :claude_messages,
|
|
289
|
+
# :openai_responses, :openai_chat) so the shared examples can pick the right
|
|
290
|
+
# shape assertions. Translators that don't implement g24_format are skipped.
|
|
291
|
+
describe 'G24 execution-proxy contract' do
|
|
292
|
+
let(:canonical_resp) do
|
|
293
|
+
canonical::Response.from_hash(conformance.fixture_symbolized('canonical_server_tool_use_response'))
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
let(:format) do
|
|
297
|
+
next nil unless translator.respond_to?(:g24_format)
|
|
298
|
+
|
|
299
|
+
translator.g24_format
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
context 'with a server-executed tool result in the canonical response' do
|
|
303
|
+
it 'surfaces the server tool name in the formatted response' do
|
|
304
|
+
next if format.nil?
|
|
305
|
+
|
|
306
|
+
formatted = translator.format_response(canonical_resp)
|
|
307
|
+
formatted_str = formatted.to_s
|
|
308
|
+
expect(formatted_str).to include('legion_list_all_tools')
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
it 'surfaces the server tool result text in the formatted response' do
|
|
312
|
+
next if format.nil?
|
|
313
|
+
|
|
314
|
+
formatted = translator.format_response(canonical_resp)
|
|
315
|
+
formatted_str = formatted.to_s
|
|
316
|
+
expect(formatted_str).to include('legion_list_all_tools, legion_apollo_search')
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
it 'never surfaces server-executed tools as actionable items', :aggregate_failures do
|
|
320
|
+
next if format.nil?
|
|
321
|
+
|
|
322
|
+
formatted = translator.format_response(canonical_resp)
|
|
323
|
+
|
|
324
|
+
case format
|
|
325
|
+
when :claude_messages
|
|
326
|
+
# Server-side tools must appear as server_tool_use, never tool_use.
|
|
327
|
+
# The G24 contract says the model must know the call happened AND
|
|
328
|
+
# the client must not re-execute, so server_tool_use+server_tool_result
|
|
329
|
+
# are the only shape — plain tool_use would put the client in a
|
|
330
|
+
# tool-loop trying to fulfill an already-resolved exchange.
|
|
331
|
+
types = (formatted[:content] || formatted['content']).map { |b| b[:type] || b['type'] }
|
|
332
|
+
expect(types).to include('server_tool_use')
|
|
333
|
+
expect(types).to include('server_tool_result')
|
|
334
|
+
expect(types).not_to include('tool_use')
|
|
335
|
+
|
|
336
|
+
stop_reason = formatted[:stop_reason] || formatted['stop_reason']
|
|
337
|
+
expect(stop_reason).to eq('end_turn')
|
|
338
|
+
|
|
339
|
+
server_use = (formatted[:content] || formatted['content']).find do |b|
|
|
340
|
+
(b[:type] || b['type']).to_s == 'server_tool_use'
|
|
341
|
+
end
|
|
342
|
+
expect(server_use[:name] || server_use['name']).to eq('legion_list_all_tools')
|
|
343
|
+
|
|
344
|
+
server_result = (formatted[:content] || formatted['content']).find do |b|
|
|
345
|
+
(b[:type] || b['type']).to_s == 'server_tool_result'
|
|
346
|
+
end
|
|
347
|
+
result_text = (server_result[:content] || server_result['content']).first
|
|
348
|
+
expect(result_text[:text] || result_text['text']).to include('legion_list_all_tools')
|
|
349
|
+
when :openai_responses
|
|
350
|
+
status = formatted[:status] || formatted['status']
|
|
351
|
+
expect(status).to eq('completed')
|
|
352
|
+
|
|
353
|
+
output = formatted[:output] || formatted['output']
|
|
354
|
+
actionable = output.select do |item|
|
|
355
|
+
type = (item[:type] || item['type']).to_s
|
|
356
|
+
status_str = (item[:status] || item['status']).to_s
|
|
357
|
+
type == 'function_call' && status_str != 'completed'
|
|
358
|
+
end
|
|
359
|
+
expect(actionable).to be_empty,
|
|
360
|
+
"found actionable function_call items for server tools: #{actionable.inspect}"
|
|
361
|
+
|
|
362
|
+
# action_required is the legacy requires-action surface — server
|
|
363
|
+
# tools must never end up there.
|
|
364
|
+
action_required = formatted[:action_required] || formatted['action_required']
|
|
365
|
+
expect(action_required).to be_nil
|
|
366
|
+
when :openai_chat
|
|
367
|
+
choice = (formatted[:choices] || formatted['choices']).first
|
|
368
|
+
finish_reason = choice[:finish_reason] || choice['finish_reason']
|
|
369
|
+
expect(finish_reason).to eq('stop')
|
|
370
|
+
|
|
371
|
+
message = choice[:message] || choice['message']
|
|
372
|
+
actionable = (message[:tool_calls] || message['tool_calls'] || []).reject do |tc|
|
|
373
|
+
(tc[:status] || tc['status']).to_s == 'completed'
|
|
374
|
+
end
|
|
375
|
+
expect(actionable).to be_empty,
|
|
376
|
+
"found actionable tool_calls for server tools: #{actionable.inspect}"
|
|
377
|
+
else
|
|
378
|
+
raise "unknown G24 format: #{format.inspect}"
|
|
379
|
+
end
|
|
380
|
+
end
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
it 'formats the streaming tool_call_delta with the registry source' do
|
|
384
|
+
next if format.nil?
|
|
385
|
+
|
|
386
|
+
# The streaming tool_call_delta carries the resolved server_tool result.
|
|
387
|
+
# We don't assert the per-chunk SSE encoding here (that's the route's
|
|
388
|
+
# event emitter contract); we assert format_chunk doesn't drop the call
|
|
389
|
+
# and the source flows through.
|
|
390
|
+
stream_fixture = conformance.fixture('canonical_streaming_server_tool_chunks')
|
|
391
|
+
tool_chunk = stream_fixture['chunks'].find do |c|
|
|
392
|
+
c['type'] == 'tool_call_delta' && c.dig('tool_call', 'result')
|
|
393
|
+
end
|
|
394
|
+
expect(tool_chunk).not_to be_nil
|
|
395
|
+
|
|
396
|
+
chunk = canonical::Chunk.from_hash(tool_chunk)
|
|
397
|
+
formatted = translator.format_chunk(chunk)
|
|
398
|
+
|
|
399
|
+
next if formatted.nil?
|
|
400
|
+
|
|
401
|
+
# Format-specific minimum: the tool name is reachable. Streaming
|
|
402
|
+
# shape per format is asserted in the matrix harness; here we only
|
|
403
|
+
# require that the server-tool name survives chunk formatting.
|
|
404
|
+
expect(formatted.to_s).to include('legion_list_all_tools')
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
it 'parses a continuation request with a prior server-executed exchange losslessly' do
|
|
408
|
+
next if format.nil?
|
|
409
|
+
next unless translator.respond_to?(:format_request)
|
|
410
|
+
|
|
411
|
+
# Each translator round-trips its own format. We render the canonical
|
|
412
|
+
# continuation with format_request (when available) then re-parse —
|
|
413
|
+
# the prior server tool exchange must survive intact.
|
|
414
|
+
continuation_body = conformance.fixture_symbolized('canonical_server_tool_continuation_request')
|
|
415
|
+
canonical_req = canonical::Request.from_hash(continuation_body)
|
|
416
|
+
formatted_body = translator.format_request(canonical_req)
|
|
417
|
+
next if formatted_body.nil?
|
|
418
|
+
|
|
419
|
+
parsed = translator.parse_request(formatted_body, {})
|
|
420
|
+
expect(parsed).to be_a(canonical::Request)
|
|
421
|
+
|
|
422
|
+
# The assistant tool_call and the tool result both survive the cycle.
|
|
423
|
+
roles = parsed.messages.map { |m| m.role.to_sym }
|
|
424
|
+
expect(roles).to include(:assistant)
|
|
425
|
+
expect(roles).to include(:tool)
|
|
426
|
+
|
|
427
|
+
tool_msg = parsed.messages.find { |m| m.role.to_sym == :tool }
|
|
428
|
+
expect(tool_msg.content.to_s).to include('legion_list_all_tools')
|
|
429
|
+
end
|
|
430
|
+
end
|
|
268
431
|
end
|
|
269
432
|
# rubocop:enable Lint/NonLocalExitFromIterator
|
data/spec/legion/extensions/llm/conformance/fixtures/canonical_server_tool_continuation_request.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
{
|
|
2
|
+
"description": "G24 round-trip — when a client sends back a history that contains a completed server-side exchange (Claude: server_tool_use+server_tool_result blocks; Codex: completed function_call+function_call_output items), the client translator must parse them into canonical messages losslessly so the next turn rendering still attributes the call to the assistant and its result to the tool role. Tools array is empty because the server-side exchange is already closed; this request is just continuing the conversation.",
|
|
3
|
+
"id": "req_g24_continuation_001",
|
|
4
|
+
"messages": [
|
|
5
|
+
{
|
|
6
|
+
"role": "user",
|
|
7
|
+
"content": "what legionio tools do you have available?"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"role": "assistant",
|
|
11
|
+
"content": "I called the legion_list_all_tools tool.",
|
|
12
|
+
"tool_calls": [
|
|
13
|
+
{
|
|
14
|
+
"id": "call_legion_001",
|
|
15
|
+
"name": "legion_list_all_tools",
|
|
16
|
+
"arguments": {"filter": "all"},
|
|
17
|
+
"source": "registry",
|
|
18
|
+
"status": "success",
|
|
19
|
+
"result": "tools: legion_list_all_tools, legion_apollo_search"
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"role": "tool",
|
|
25
|
+
"tool_call_id": "call_legion_001",
|
|
26
|
+
"content": "tools: legion_list_all_tools, legion_apollo_search"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"role": "user",
|
|
30
|
+
"content": "thanks — anything else?"
|
|
31
|
+
}
|
|
32
|
+
],
|
|
33
|
+
"system": null,
|
|
34
|
+
"tools": null,
|
|
35
|
+
"params": null,
|
|
36
|
+
"thinking": null,
|
|
37
|
+
"stream": false,
|
|
38
|
+
"conversation_id": "conv_g24_001",
|
|
39
|
+
"routing": {},
|
|
40
|
+
"metadata": {
|
|
41
|
+
"g24": "continuation"
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
{
|
|
2
|
+
"description": "G24 — server-executed LegionIO tool. The provider returned tool_use, the executor ran the tool registry-side, and the canonical response carries both the call AND its result so client translators can surface it as a completed exchange (Claude: server_tool_use+server_tool_result, Codex: completed function_call+function_call_output non-actionable). stop_reason is end_turn because the server-side exchange is closed by the time the canonical response is built.",
|
|
3
|
+
"text": "I called the legion_list_all_tools tool and here is the list.",
|
|
4
|
+
"thinking": null,
|
|
5
|
+
"tool_calls": [
|
|
6
|
+
{
|
|
7
|
+
"id": "call_legion_001",
|
|
8
|
+
"exchange_id": "exch_legion_001",
|
|
9
|
+
"name": "legion_list_all_tools",
|
|
10
|
+
"arguments": {
|
|
11
|
+
"filter": "all"
|
|
12
|
+
},
|
|
13
|
+
"source": "registry",
|
|
14
|
+
"status": "success",
|
|
15
|
+
"result": "tools: legion_list_all_tools, legion_apollo_search, legion_runner_dispatch",
|
|
16
|
+
"duration_ms": 42
|
|
17
|
+
}
|
|
18
|
+
],
|
|
19
|
+
"usage": {
|
|
20
|
+
"input_tokens": 50,
|
|
21
|
+
"output_tokens": 22
|
|
22
|
+
},
|
|
23
|
+
"stop_reason": "end_turn",
|
|
24
|
+
"model": "test-model-1",
|
|
25
|
+
"routing": {},
|
|
26
|
+
"metadata": {
|
|
27
|
+
"g24": true
|
|
28
|
+
}
|
|
29
|
+
}
|