llm_logs 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +25 -0
- data/app/controllers/llm_logs/prompts_controller.rb +21 -14
- data/app/models/llm_logs/batch/adapters/bedrock.rb +7 -0
- data/app/models/llm_logs/batch/adapters/openai_responses.rb +9 -0
- data/app/models/llm_logs/batch.rb +3 -2
- data/app/models/llm_logs/prompt_version.rb +20 -0
- data/app/views/llm_logs/prompts/_form.html.erb +19 -0
- data/app/views/llm_logs/prompts/index.html.erb +13 -3
- data/app/views/llm_logs/prompts/show.html.erb +16 -16
- data/lib/llm_logs/configuration.rb +5 -1
- data/lib/llm_logs/version.rb +1 -1
- data/lib/llm_logs.rb +4 -0
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8982c29ec3f53e2271e24f78065472f3f71b15828449dd51cda1154122cb8d6f
|
|
4
|
+
data.tar.gz: 2a2026e3441b16b61c36497689a7d468f09987bfb0c85c95aebb641ba3ee9d94
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d3d62b8c8d71bb12639ef74895d53e2974aba1eca3e4d219bce6387f790cb7712ac2ca46ea24e72c7940bb2970ecc04af976802d7a7d1c2f758bc507a392e3aa
|
|
7
|
+
data.tar.gz: 9dfdbe34061fe8b748fda1ca4ad75f8646e57030c264549b91da0cd0b0314577b42b4b0fef8bfa87b336cbac9c2b95bf5d10bf92686c5548780539751b14402b
|
data/README.md
CHANGED
|
@@ -105,6 +105,31 @@ params = prompt.build(
|
|
|
105
105
|
# => { model: "claude-sonnet-4", messages: [...], temperature: 0.3, max_tokens: 2048 }
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
+
### Reasoning Effort
|
|
109
|
+
|
|
110
|
+
Reasoning models take an effort tier rather than a temperature. Set it like any
|
|
111
|
+
other model param, from the admin form or a prompt `.md`:
|
|
112
|
+
|
|
113
|
+
```yaml
|
|
114
|
+
model: gpt-5.6-terra
|
|
115
|
+
model_params:
|
|
116
|
+
reasoning_effort: low
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Values are validated against `LlmLogs.reasoning_effort_options`, which defaults to
|
|
120
|
+
`%w[none low medium high xhigh max]` — the union of what current providers accept.
|
|
121
|
+
Narrow it if your app targets a single provider (Anthropic has no `none` tier):
|
|
122
|
+
|
|
123
|
+
```ruby
|
|
124
|
+
LlmLogs.setup do |config|
|
|
125
|
+
config.reasoning_effort_options = %w[low medium high]
|
|
126
|
+
end
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Leaving the field blank stores nothing, so the model's own default applies.
|
|
130
|
+
Applying the value to a request is the caller's job — the gem stores and validates
|
|
131
|
+
it, it does not build your chat object.
|
|
132
|
+
|
|
108
133
|
### Versioning
|
|
109
134
|
|
|
110
135
|
Every save creates a new version. Previous versions are never modified.
|
|
@@ -26,14 +26,14 @@ module LlmLogs
|
|
|
26
26
|
def create
|
|
27
27
|
@prompt = Prompt.new(prompt_params)
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
end
|
|
33
|
-
redirect_to prompt_path(@prompt), notice: "Prompt created."
|
|
34
|
-
else
|
|
35
|
-
render :new, status: :unprocessable_entity
|
|
29
|
+
ActiveRecord::Base.transaction do
|
|
30
|
+
@prompt.save!
|
|
31
|
+
@prompt.update_content!(**version_params) if version_params[:messages].present?
|
|
36
32
|
end
|
|
33
|
+
redirect_to prompt_path(@prompt), notice: "Prompt created."
|
|
34
|
+
rescue ActiveRecord::RecordInvalid => e
|
|
35
|
+
surface_version_errors(e)
|
|
36
|
+
render :new, status: :unprocessable_entity
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
def edit
|
|
@@ -44,14 +44,15 @@ module LlmLogs
|
|
|
44
44
|
def update
|
|
45
45
|
@prompt = Prompt.find(params[:id])
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
end
|
|
51
|
-
redirect_to prompt_path(@prompt), notice: "Prompt updated."
|
|
52
|
-
else
|
|
53
|
-
render :edit, status: :unprocessable_entity
|
|
47
|
+
ActiveRecord::Base.transaction do
|
|
48
|
+
@prompt.update!(prompt_params)
|
|
49
|
+
@prompt.update_content!(**version_params) if version_params[:messages].present?
|
|
54
50
|
end
|
|
51
|
+
redirect_to prompt_path(@prompt), notice: "Prompt updated."
|
|
52
|
+
rescue ActiveRecord::RecordInvalid => e
|
|
53
|
+
@current_version = @prompt.current_version
|
|
54
|
+
surface_version_errors(e)
|
|
55
|
+
render :edit, status: :unprocessable_entity
|
|
55
56
|
end
|
|
56
57
|
|
|
57
58
|
def destroy
|
|
@@ -70,6 +71,12 @@ module LlmLogs
|
|
|
70
71
|
raw.except(:tags_input)
|
|
71
72
|
end
|
|
72
73
|
|
|
74
|
+
def surface_version_errors(error)
|
|
75
|
+
return if error.record == @prompt
|
|
76
|
+
|
|
77
|
+
error.record.errors.full_messages.each { |message| @prompt.errors.add(:base, message) }
|
|
78
|
+
end
|
|
79
|
+
|
|
73
80
|
def version_params
|
|
74
81
|
raw = params.require(:prompt).permit(:model, :changelog, model_params: {})
|
|
75
82
|
messages = parse_messages
|
|
@@ -96,10 +96,17 @@ module LlmLogs
|
|
|
96
96
|
"messages" => [{"role" => "user", "content" => payload["input"]}]
|
|
97
97
|
}
|
|
98
98
|
body["system"] = payload["instructions"] if payload["instructions"]
|
|
99
|
+
body["temperature"] = payload["temperature"] if payload["temperature"]
|
|
100
|
+
body.merge!(adaptive_thinking(payload["reasoning_effort"])) if payload["reasoning_effort"].present?
|
|
99
101
|
body.merge!(structured_output(payload["schema"])) if payload["schema"]
|
|
100
102
|
body
|
|
101
103
|
end
|
|
102
104
|
|
|
105
|
+
# Anthropic takes an effort tier as adaptive thinking rather than a bare param.
|
|
106
|
+
def adaptive_thinking(effort)
|
|
107
|
+
{"thinking" => {"type" => "adaptive"}, "output_config" => {"effort" => effort}}
|
|
108
|
+
end
|
|
109
|
+
|
|
103
110
|
# Anthropic structured output via a single forced tool (verify encoding against QA).
|
|
104
111
|
def structured_output(schema)
|
|
105
112
|
spec = schema["schema"] || schema
|
|
@@ -15,6 +15,7 @@ module LlmLogs
|
|
|
15
15
|
id: request.custom_id,
|
|
16
16
|
instructions: payload["instructions"],
|
|
17
17
|
temperature: payload["temperature"],
|
|
18
|
+
**reasoning_extra(payload["reasoning_effort"]),
|
|
18
19
|
**schema_extra(payload["schema"])
|
|
19
20
|
)
|
|
20
21
|
end
|
|
@@ -36,6 +37,14 @@ module LlmLogs
|
|
|
36
37
|
|
|
37
38
|
private
|
|
38
39
|
|
|
40
|
+
# The Responses API nests effort under `reasoning:`; `reasoning_effort` is the
|
|
41
|
+
# Chat Completions spelling and is silently ignored here.
|
|
42
|
+
def reasoning_extra(effort)
|
|
43
|
+
return {} if effort.blank?
|
|
44
|
+
|
|
45
|
+
{reasoning: {effort: effort}}
|
|
46
|
+
end
|
|
47
|
+
|
|
39
48
|
# No memoization: LlmLogs.batch_adapters holds one shared instance of this adapter
|
|
40
49
|
# for the life of the process, so caching the resumed handle here would go stale
|
|
41
50
|
# for a long-running PollJob worker (in-progress batches would never re-fetch a
|
|
@@ -18,7 +18,7 @@ module LlmLogs
|
|
|
18
18
|
scope :recent, -> { order(created_at: :desc) }
|
|
19
19
|
scope :unreconciled, -> { where.not(status: %i[reconciled failed expired]) }
|
|
20
20
|
|
|
21
|
-
def self.enqueue(purpose:, model:, input:, instructions:, schema:, routing:, temperature: nil)
|
|
21
|
+
def self.enqueue(purpose:, model:, input:, instructions:, schema:, routing:, temperature: nil, reasoning_effort: nil)
|
|
22
22
|
BatchRequest.create!(
|
|
23
23
|
purpose: purpose,
|
|
24
24
|
model: model,
|
|
@@ -29,7 +29,8 @@ module LlmLogs
|
|
|
29
29
|
"input" => input,
|
|
30
30
|
"instructions" => instructions,
|
|
31
31
|
"schema" => schema,
|
|
32
|
-
"temperature" => temperature
|
|
32
|
+
"temperature" => temperature,
|
|
33
|
+
"reasoning_effort" => reasoning_effort
|
|
33
34
|
}.compact
|
|
34
35
|
)
|
|
35
36
|
end
|
|
@@ -7,6 +7,13 @@ module LlmLogs
|
|
|
7
7
|
|
|
8
8
|
validates :version_number, presence: true, uniqueness: { scope: :prompt_id }
|
|
9
9
|
validates :messages, presence: true
|
|
10
|
+
validate :reasoning_effort_is_supported
|
|
11
|
+
|
|
12
|
+
def reasoning_effort
|
|
13
|
+
return nil unless model_params.is_a?(Hash)
|
|
14
|
+
|
|
15
|
+
model_params["reasoning_effort"] || model_params[:reasoning_effort]
|
|
16
|
+
end
|
|
10
17
|
|
|
11
18
|
def variables
|
|
12
19
|
messages.flat_map { |msg| msg["content"].to_s.scan(/\{\{[#^]?([^\/}]+)\}\}/) }.flatten.uniq.sort
|
|
@@ -27,5 +34,18 @@ module LlmLogs
|
|
|
27
34
|
params.merge!(model_params.symbolize_keys) if model_params.present?
|
|
28
35
|
params
|
|
29
36
|
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
# Providers reject an unknown effort tier at request time; catching it here means
|
|
41
|
+
# a typo in the admin form or a prompt .md fails loudly at write time instead.
|
|
42
|
+
def reasoning_effort_is_supported
|
|
43
|
+
return if reasoning_effort.blank?
|
|
44
|
+
|
|
45
|
+
allowed = LlmLogs.reasoning_effort_options
|
|
46
|
+
return if allowed.include?(reasoning_effort.to_s)
|
|
47
|
+
|
|
48
|
+
errors.add(:model_params, "reasoning_effort #{reasoning_effort.inspect} is not supported (expected one of: #{allowed.join(", ")})")
|
|
49
|
+
end
|
|
30
50
|
end
|
|
31
51
|
end
|
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
<%= form_with(model: prompt, url: prompt.persisted? ? prompt_path(prompt) : prompts_path, class: "space-y-6") do |f| %>
|
|
2
|
+
<% if prompt.errors[:base].any? %>
|
|
3
|
+
<div class="rounded-md bg-red-50 p-3 ring-1 ring-red-200">
|
|
4
|
+
<% prompt.errors[:base].each do |error| %>
|
|
5
|
+
<p class="text-sm text-red-700"><%= error %></p>
|
|
6
|
+
<% end %>
|
|
7
|
+
</div>
|
|
8
|
+
<% end %>
|
|
9
|
+
|
|
2
10
|
<div class="bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5 p-6 space-y-4">
|
|
3
11
|
<h2 class="text-lg font-medium text-gray-900">Prompt Details</h2>
|
|
4
12
|
|
|
@@ -83,6 +91,17 @@
|
|
|
83
91
|
value="<%= current_version&.model_params&.dig('max_tokens') %>"
|
|
84
92
|
class="w-full rounded-md border-gray-300 shadow-sm text-sm px-3 py-2 border">
|
|
85
93
|
</div>
|
|
94
|
+
<div>
|
|
95
|
+
<label class="block text-sm font-medium text-gray-700 mb-1">Reasoning Effort</label>
|
|
96
|
+
<select name="prompt[model_params][reasoning_effort]"
|
|
97
|
+
class="w-full rounded-md border-gray-300 shadow-sm text-sm px-3 py-2 border">
|
|
98
|
+
<%= options_for_select(
|
|
99
|
+
[["Model default", ""]] + LlmLogs.reasoning_effort_options.map { |o| [o, o] },
|
|
100
|
+
current_version&.model_params&.dig("reasoning_effort").to_s
|
|
101
|
+
) %>
|
|
102
|
+
</select>
|
|
103
|
+
<p class="mt-1 text-xs text-gray-500">Only applies to reasoning models. Leave as default to send nothing.</p>
|
|
104
|
+
</div>
|
|
86
105
|
</div>
|
|
87
106
|
</div>
|
|
88
107
|
|
|
@@ -20,12 +20,14 @@
|
|
|
20
20
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase"><%= sort_link "Name", "name" %></th>
|
|
21
21
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase"><%= sort_link "Slug", "slug" %></th>
|
|
22
22
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Tags</th>
|
|
23
|
+
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Model</th>
|
|
23
24
|
<th class="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase">Version</th>
|
|
24
25
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase"><%= sort_link "Updated", "updated" %></th>
|
|
25
26
|
</tr>
|
|
26
27
|
</thead>
|
|
27
28
|
<tbody class="divide-y divide-gray-200">
|
|
28
29
|
<% @prompts.each do |prompt| %>
|
|
30
|
+
<% latest = prompt.versions.max_by(&:version_number) %>
|
|
29
31
|
<tr class="hover:bg-gray-50">
|
|
30
32
|
<td class="px-4 py-3 text-sm">
|
|
31
33
|
<%= link_to prompt.name, prompt_path(prompt), class: "text-indigo-600 hover:text-indigo-900 font-medium" %>
|
|
@@ -36,9 +38,17 @@
|
|
|
36
38
|
<span class="inline-block bg-gray-100 text-gray-700 text-xs px-1.5 py-0.5 rounded mr-1"><%= tag %></span>
|
|
37
39
|
<% end %>
|
|
38
40
|
</td>
|
|
41
|
+
<td class="px-4 py-3 text-sm text-gray-500 font-mono">
|
|
42
|
+
<% if latest&.model.present? %>
|
|
43
|
+
<%= latest.model %>
|
|
44
|
+
<% if latest.reasoning_effort.present? %>
|
|
45
|
+
<span class="inline-block bg-gray-100 text-gray-700 text-xs px-1.5 py-0.5 rounded ml-1 font-sans"><%= latest.reasoning_effort %></span>
|
|
46
|
+
<% end %>
|
|
47
|
+
<% else %>—<% end %>
|
|
48
|
+
</td>
|
|
39
49
|
<td class="px-4 py-3 text-sm text-right">
|
|
40
|
-
<% if
|
|
41
|
-
<%= link_to "v#{
|
|
50
|
+
<% if latest %>
|
|
51
|
+
<%= link_to "v#{latest.version_number}", prompt_versions_path(prompt), class: "text-indigo-600 hover:text-indigo-900" %>
|
|
42
52
|
<% else %>—<% end %>
|
|
43
53
|
</td>
|
|
44
54
|
<td class="px-4 py-3 text-sm text-gray-500"><%= prompt.updated_at.strftime('%b %d %H:%M') %></td>
|
|
@@ -47,7 +57,7 @@
|
|
|
47
57
|
|
|
48
58
|
<% if @prompts.empty? %>
|
|
49
59
|
<tr>
|
|
50
|
-
<td colspan="
|
|
60
|
+
<td colspan="6" class="px-4 py-8 text-center text-sm text-gray-500">
|
|
51
61
|
No prompts yet. <%= link_to "Create one", new_prompt_path, class: "text-indigo-600 hover:text-indigo-900" %>.
|
|
52
62
|
</td>
|
|
53
63
|
</tr>
|
|
@@ -44,22 +44,6 @@
|
|
|
44
44
|
</div>
|
|
45
45
|
</div>
|
|
46
46
|
|
|
47
|
-
<% if @current_version.model_params.present? && @current_version.model_params.any? %>
|
|
48
|
-
<div class="bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5">
|
|
49
|
-
<div class="px-4 py-3 border-b border-gray-200">
|
|
50
|
-
<h2 class="text-sm font-medium text-gray-900">Model Parameters</h2>
|
|
51
|
-
</div>
|
|
52
|
-
<div class="p-4">
|
|
53
|
-
<dl class="grid grid-cols-2 gap-2 text-sm">
|
|
54
|
-
<% @current_version.model_params.each do |key, value| %>
|
|
55
|
-
<dt class="text-gray-500"><%= key %></dt>
|
|
56
|
-
<dd class="text-gray-900 font-mono"><%= value %></dd>
|
|
57
|
-
<% end %>
|
|
58
|
-
</dl>
|
|
59
|
-
</div>
|
|
60
|
-
</div>
|
|
61
|
-
<% end %>
|
|
62
|
-
|
|
63
47
|
<% if @current_version.default_variables.present? && @current_version.default_variables.any? %>
|
|
64
48
|
<div class="bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5">
|
|
65
49
|
<div class="px-4 py-3 border-b border-gray-200">
|
|
@@ -101,6 +85,22 @@
|
|
|
101
85
|
</div>
|
|
102
86
|
</div>
|
|
103
87
|
|
|
88
|
+
<% if @current_version&.model_params.present? %>
|
|
89
|
+
<div class="mt-4 bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5">
|
|
90
|
+
<div class="px-4 py-3 border-b border-gray-200">
|
|
91
|
+
<h2 class="text-sm font-medium text-gray-900">Model Parameters</h2>
|
|
92
|
+
</div>
|
|
93
|
+
<div class="p-4">
|
|
94
|
+
<dl class="grid grid-cols-2 gap-2 text-sm">
|
|
95
|
+
<% @current_version.model_params.each do |key, value| %>
|
|
96
|
+
<dt class="text-gray-500"><%= key %></dt>
|
|
97
|
+
<dd class="text-gray-900 font-mono text-right"><%= value %></dd>
|
|
98
|
+
<% end %>
|
|
99
|
+
</dl>
|
|
100
|
+
</div>
|
|
101
|
+
</div>
|
|
102
|
+
<% end %>
|
|
103
|
+
|
|
104
104
|
<details class="mt-4 bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5">
|
|
105
105
|
<summary class="cursor-pointer px-4 py-3 text-sm font-medium text-gray-900">SDK Usage</summary>
|
|
106
106
|
<div class="px-4 pb-4">
|
|
@@ -3,7 +3,7 @@ module LlmLogs
|
|
|
3
3
|
BedrockBatch = Struct.new(:role_arn, :s3_bucket, :s3_prefix, :min_records, :model_matcher, :region, keyword_init: true)
|
|
4
4
|
|
|
5
5
|
attr_accessor :enabled, :auto_instrument, :retention_days, :prompts_source_path, :prompt_subfolders,
|
|
6
|
-
:batch_enabled, :batch_provider, :page_size, :bedrock_batch
|
|
6
|
+
:batch_enabled, :batch_provider, :page_size, :bedrock_batch, :reasoning_effort_options
|
|
7
7
|
|
|
8
8
|
def initialize
|
|
9
9
|
@enabled = true
|
|
@@ -15,6 +15,10 @@ module LlmLogs
|
|
|
15
15
|
@batch_provider = :openai_responses
|
|
16
16
|
@page_size = 50
|
|
17
17
|
@bedrock_batch = nil
|
|
18
|
+
# Effort tiers offered in the prompt form. Union of what current providers
|
|
19
|
+
# accept -- OpenAI takes all six, Anthropic has no "none". Narrow this in an
|
|
20
|
+
# initializer if your app targets one provider.
|
|
21
|
+
@reasoning_effort_options = %w[none low medium high xhigh max]
|
|
18
22
|
end
|
|
19
23
|
end
|
|
20
24
|
|
data/lib/llm_logs/version.rb
CHANGED
data/lib/llm_logs.rb
CHANGED