solid_loop 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +445 -0
- data/README.md +305 -4
- data/Rakefile +5 -4
- data/app/controllers/solid_loop/application_controller.rb +6 -0
- data/app/controllers/solid_loop/dashboard_controller.rb +167 -22
- data/app/controllers/solid_loop/events_controller.rb +9 -1
- data/app/controllers/solid_loop/mcp_sessions_controller.rb +12 -0
- data/app/controllers/solid_loop/messages_controller.rb +11 -1
- data/app/controllers/solid_loop/tool_calls_controller.rb +32 -0
- data/app/helpers/solid_loop/application_helper.rb +4 -1
- data/app/helpers/solid_loop/metrics_helper.rb +232 -0
- data/app/jobs/solid_loop/janitor_job.rb +24 -0
- data/app/jobs/solid_loop/llm_completion_job.rb +2 -2
- data/app/models/solid_loop/base.rb +89 -6
- data/app/models/solid_loop/loop.rb +22 -0
- data/app/models/solid_loop/message.rb +33 -7
- data/app/models/solid_loop/tool_call.rb +7 -1
- data/app/services/solid_loop/adapters/native.rb +156 -23
- data/app/services/solid_loop/dialects/anthropic.rb +43 -10
- data/app/services/solid_loop/dialects/gemini.rb +55 -15
- data/app/services/solid_loop/dialects/open_ai.rb +17 -5
- data/app/services/solid_loop/dialects/reasoning_packer.rb +72 -7
- data/app/services/solid_loop/llm_usage_parser/llama.rb +18 -3
- data/app/services/solid_loop/mcp_session_initializer.rb +1 -0
- data/app/services/solid_loop/middlewares/agent_initialization.rb +1 -1
- data/app/services/solid_loop/middlewares/error_handling.rb +5 -1
- data/app/services/solid_loop/middlewares/event_logging.rb +3 -3
- data/app/services/solid_loop/middlewares/message_building.rb +38 -4
- data/app/services/solid_loop/middlewares/response_parsing.rb +19 -3
- data/app/views/layouts/solid_loop/admin.html.erb +134 -23
- data/app/views/solid_loop/dashboard/index.html.erb +330 -41
- data/app/views/solid_loop/events/index.html.erb +17 -1
- data/app/views/solid_loop/loops/index.html.erb +40 -3
- data/app/views/solid_loop/loops/show.html.erb +1 -1
- data/app/views/solid_loop/mcp_sessions/index.html.erb +40 -2
- data/app/views/solid_loop/messages/_message.html.erb +26 -35
- data/app/views/solid_loop/messages/index.html.erb +16 -2
- data/app/views/solid_loop/tool_calls/index.html.erb +18 -3
- data/db/migrate/20260819000100_solid_loop_add_retention_indexes.rb +22 -0
- data/docs/contributing/coverage.md +8 -8
- data/docs/decisions/mcp-server.md +4 -2
- data/docs/decisions/reasoning_persistence.md +120 -4
- data/docs/guides/dialects.md +1 -1
- data/docs/guides/mcp_transports.md +72 -0
- data/docs/validation.md +85 -0
- data/lib/solid_loop/configuration.rb +93 -0
- data/lib/solid_loop/engine.rb +8 -0
- data/lib/solid_loop/janitor.rb +110 -0
- data/lib/solid_loop/mcp/toolset.rb +119 -31
- data/lib/solid_loop/pipeline/builder.rb +31 -10
- data/lib/solid_loop/redaction.rb +26 -0
- data/lib/solid_loop/version.rb +1 -1
- data/lib/solid_loop.rb +47 -0
- metadata +7 -2
- data/lib/tasks/coverage.rake +0 -206
|
@@ -5,6 +5,18 @@ module SolidLoop
|
|
|
5
5
|
def index
|
|
6
6
|
@pagination = paginate(SolidLoop::Admin::McpSessionsQuery.new(params).call)
|
|
7
7
|
@mcp_sessions = @pagination[:records]
|
|
8
|
+
|
|
9
|
+
# A session is only ever reused while its loop can still run, so "is this
|
|
10
|
+
# session still usable" is really "is its loop still alive" — plus whether
|
|
11
|
+
# tool discovery ever produced anything. Both are gathered in one grouped
|
|
12
|
+
# query each instead of touching `session.loop` / `session.mcp_tools` per
|
|
13
|
+
# row (an N+1 over two tables on every page view).
|
|
14
|
+
session_ids = @mcp_sessions.map(&:id)
|
|
15
|
+
@loop_statuses = SolidLoop::Loop
|
|
16
|
+
.where(id: @mcp_sessions.map(&:loop_id).compact.uniq)
|
|
17
|
+
.pluck(:id, :status)
|
|
18
|
+
.to_h
|
|
19
|
+
@tool_counts = SolidLoop::McpTool.where(mcp_session_id: session_ids).group(:mcp_session_id).count
|
|
8
20
|
end
|
|
9
21
|
|
|
10
22
|
def show
|
|
@@ -1,8 +1,18 @@
|
|
|
1
1
|
module SolidLoop
|
|
2
2
|
class MessagesController < ApplicationController
|
|
3
|
+
# Same reasoning as EventsController::LIST_COLUMNS: the list renders a role,
|
|
4
|
+
# a model name and token counts, while `content` / `reasoning_content` /
|
|
5
|
+
# `tool_calls_raw` are the biggest columns on the table (a single reasoning
|
|
6
|
+
# blob routinely runs to thousands of characters). Narrowed after pagination
|
|
7
|
+
# so the count query is unaffected.
|
|
8
|
+
LIST_COLUMNS = %i[
|
|
9
|
+
id loop_id role metadata cost created_at
|
|
10
|
+
tokens_total tokens_prompt tokens_completion tokens_prompt_cached
|
|
11
|
+
].freeze
|
|
12
|
+
|
|
3
13
|
def index
|
|
4
14
|
@pagination = paginate(SolidLoop::Admin::MessagesQuery.new(params).call)
|
|
5
|
-
@messages = @pagination[:records]
|
|
15
|
+
@messages = @pagination[:records].select(*LIST_COLUMNS)
|
|
6
16
|
end
|
|
7
17
|
|
|
8
18
|
def show
|
|
@@ -3,10 +3,42 @@ module SolidLoop
|
|
|
3
3
|
def index
|
|
4
4
|
@pagination = paginate(SolidLoop::Admin::ToolCallsQuery.new(params).call)
|
|
5
5
|
@tool_calls = @pagination[:records]
|
|
6
|
+
@loop_ids_by_message = loop_ids_by_message(@tool_calls)
|
|
7
|
+
@function_names = function_filter_options
|
|
6
8
|
end
|
|
7
9
|
|
|
8
10
|
def show
|
|
9
11
|
@tool_call = SolidLoop::ToolCall.find(params[:id])
|
|
10
12
|
end
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
# The loop a tool call belongs to lives one hop away (tool_call -> message ->
|
|
17
|
+
# loop_id). Reading it as `tool_call.message.loop_id` in the view is an N+1
|
|
18
|
+
# over a table that grows by several rows per turn, and `includes(:message)`
|
|
19
|
+
# would drag every message's content / reasoning_content / tool_calls_raw
|
|
20
|
+
# (routinely tens of KB each) into memory to read a single bigint. One narrow
|
|
21
|
+
# pluck keyed by message_id instead: a single extra query, two columns.
|
|
22
|
+
def loop_ids_by_message(tool_calls)
|
|
23
|
+
message_ids = tool_calls.map(&:message_id).compact.uniq
|
|
24
|
+
return {} if message_ids.empty?
|
|
25
|
+
|
|
26
|
+
SolidLoop::Message.where(id: message_ids).pluck(:id, :loop_id).to_h
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# The callable functions are a small, known set — the tools discovered from
|
|
30
|
+
# the MCP servers — so the filter is a select, not a free-text box the user
|
|
31
|
+
# has to spell from memory. `solid_loop_mcp_tools` is bounded by
|
|
32
|
+
# (sessions x tools) and is the cheap side of this lookup; DISTINCTing the
|
|
33
|
+
# ever-growing tool_calls table would not be.
|
|
34
|
+
def function_filter_options
|
|
35
|
+
names = SolidLoop::McpTool.distinct.order(:name).pluck(:name)
|
|
36
|
+
|
|
37
|
+
# Keep a hand-entered / bookmarked filter selectable even when its tool is
|
|
38
|
+
# no longer advertised by any session, so the URL is never silently
|
|
39
|
+
# rewritten to "All".
|
|
40
|
+
current = params[:function_name].presence
|
|
41
|
+
current ? (names + [ current ]).uniq : names
|
|
42
|
+
end
|
|
11
43
|
end
|
|
12
44
|
end
|
|
@@ -93,7 +93,10 @@ module SolidLoop
|
|
|
93
93
|
label = resolved[:label].presence || "#{loop.subject_type} ##{loop.subject_id}"
|
|
94
94
|
url = sl_safe_subject_url(resolved[:url])
|
|
95
95
|
|
|
96
|
-
|
|
96
|
+
# Without an explicit class the anchor takes the browser's default link
|
|
97
|
+
# blue, which is nearly unreadable on the dark admin background — and the
|
|
98
|
+
# muted colour set on the wrapping element does not cascade into an <a>.
|
|
99
|
+
url ? link_to(label, url, class: "sl-subject-link") : label
|
|
97
100
|
end
|
|
98
101
|
|
|
99
102
|
private
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
module SolidLoop
|
|
2
|
+
# Presentation of the numbers the admin list pages are actually read FOR:
|
|
3
|
+
# how long something took, when it last did anything, and whether the wire
|
|
4
|
+
# call succeeded. Kept out of ApplicationHelper (money / failure classification
|
|
5
|
+
# / subject linking) because these are list-table formatters with their own
|
|
6
|
+
# thresholds, and the thresholds are the interesting part.
|
|
7
|
+
module MetricsHelper
|
|
8
|
+
# ---- Durations ---------------------------------------------------------
|
|
9
|
+
# One duration formatter for every list page, so 0.24s and 40s are visibly
|
|
10
|
+
# different at a glance and a 10-minute call never reads as "600.0".
|
|
11
|
+
#
|
|
12
|
+
# 0.004 -> "0.004s" 12.42 -> "12.4s"
|
|
13
|
+
# 0.24 -> "0.240s" 608.55 -> "10m 9s"
|
|
14
|
+
# 2.98 -> "2.98s" 5400 -> "1h 30m"
|
|
15
|
+
#
|
|
16
|
+
# `0` (and nil) render as "—": every duration column in the schema defaults
|
|
17
|
+
# to 0.0, so a zero is far more likely to mean "never recorded" than "took
|
|
18
|
+
# no time at all", and printing "0.00s" for it invents a measurement.
|
|
19
|
+
def sl_duration(seconds, blank: "—")
|
|
20
|
+
return blank if seconds.nil?
|
|
21
|
+
|
|
22
|
+
value = seconds.to_f
|
|
23
|
+
return blank if value <= 0
|
|
24
|
+
|
|
25
|
+
case value
|
|
26
|
+
when 0...1 then format("%.3fs", value)
|
|
27
|
+
when 1...10 then format("%.2fs", value)
|
|
28
|
+
when 10...60 then format("%.1fs", value)
|
|
29
|
+
when 60...3600 then "#{(value / 60).floor}m #{(value % 60).round}s"
|
|
30
|
+
else "#{(value / 3600).floor}h #{((value % 3600) / 60).round}m"
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Slowness is relative to what the call IS. A local LLM completion routinely
|
|
35
|
+
# runs for a minute and that is normal; an MCP tool call that takes a minute
|
|
36
|
+
# is pathological. Colouring both against one absolute threshold would paint
|
|
37
|
+
# the whole Events page red and make the outliers — the 608s read timeout
|
|
38
|
+
# that had to be found by hand in psql — invisible again.
|
|
39
|
+
SL_DURATION_SCALES = {
|
|
40
|
+
llm: { warn: 60.0, slow: 300.0 },
|
|
41
|
+
tool: { warn: 5.0, slow: 30.0 }
|
|
42
|
+
}.freeze
|
|
43
|
+
|
|
44
|
+
def sl_duration_severity(seconds, scale: :tool)
|
|
45
|
+
value = seconds.to_f
|
|
46
|
+
thresholds = SL_DURATION_SCALES.fetch(scale, SL_DURATION_SCALES[:tool])
|
|
47
|
+
|
|
48
|
+
return :slow if value >= thresholds[:slow]
|
|
49
|
+
return :warn if value >= thresholds[:warn]
|
|
50
|
+
|
|
51
|
+
nil
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# A formatted, severity-coloured duration cell.
|
|
55
|
+
def sl_duration_cell(seconds, scale: :tool, blank: "—")
|
|
56
|
+
text = sl_duration(seconds, blank: blank)
|
|
57
|
+
return tag.span(text, class: "sl-metric sl-metric--blank") if text == blank
|
|
58
|
+
|
|
59
|
+
severity = sl_duration_severity(seconds, scale: scale)
|
|
60
|
+
classes = [ "sl-metric" ]
|
|
61
|
+
classes << "sl-metric--#{severity}" if severity
|
|
62
|
+
|
|
63
|
+
tag.span(text, class: classes.join(" "), title: "#{format('%.3f', seconds.to_f)}s")
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# ---- Event families ----------------------------------------------------
|
|
67
|
+
# Every event used to wear the same blue badge, so an LLM completion and an
|
|
68
|
+
# MCP handshake were indistinguishable while scanning. Group by protocol
|
|
69
|
+
# family instead: LLM traffic, MCP tool invocations, and the MCP lifecycle
|
|
70
|
+
# chatter (initialize / list_tools / list_prompts / list_resources) that is
|
|
71
|
+
# noise most of the time.
|
|
72
|
+
def sl_event_family(name)
|
|
73
|
+
case name.to_s
|
|
74
|
+
when /\Allm/ then :llm
|
|
75
|
+
when "mcp_tool_call" then :mcp
|
|
76
|
+
when /\Amcp/ then :mcp_meta
|
|
77
|
+
else :other
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def sl_event_badge_class(name)
|
|
82
|
+
case sl_event_family(name)
|
|
83
|
+
when :llm then "sl-badge--llm"
|
|
84
|
+
when :mcp then "sl-badge--mcp"
|
|
85
|
+
when :mcp_meta then "sl-badge--mcp-meta"
|
|
86
|
+
else "sl-badge--processing"
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# The duration scale an event should be judged on (see SL_DURATION_SCALES).
|
|
91
|
+
def sl_event_duration_scale(name)
|
|
92
|
+
sl_event_family(name) == :llm ? :llm : :tool
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# ---- Event status ------------------------------------------------------
|
|
96
|
+
# There is no `status` column: an event's outcome is buried in
|
|
97
|
+
# `response_data`, whose shape depends on which middleware wrote it.
|
|
98
|
+
#
|
|
99
|
+
# * a failing HTTP response -> { http_status:, response_headers:, response_body: }
|
|
100
|
+
# * a raised/transport error -> { error: "Faraday::TimeoutError: ..." }
|
|
101
|
+
# * a JSON-RPC envelope -> { jsonrpc:, result: } or { error: { code:, message: } }
|
|
102
|
+
# * an MCP tool-level failure -> { result: { isError: true, ... } }
|
|
103
|
+
# * a plain OK completion -> the parsed provider body (no status at all)
|
|
104
|
+
#
|
|
105
|
+
# Returns { label:, kind:, title: } where `kind` is :ok, :warn, :error or
|
|
106
|
+
# :unknown. A 2xx envelope carrying an `isError` tool result is :warn, not
|
|
107
|
+
# :error — the transport worked, the tool did not.
|
|
108
|
+
def sl_event_status(event)
|
|
109
|
+
data = event.response_data.presence || {}
|
|
110
|
+
data = {} unless data.is_a?(Hash)
|
|
111
|
+
|
|
112
|
+
http_status = data["http_status"] || data[:http_status]
|
|
113
|
+
error = data["error"] || data[:error]
|
|
114
|
+
|
|
115
|
+
if http_status.present?
|
|
116
|
+
code = http_status.to_i
|
|
117
|
+
{ label: code.to_s, kind: code.between?(200, 299) ? :ok : :error, title: error.presence || "HTTP #{code}" }
|
|
118
|
+
elsif error.present?
|
|
119
|
+
{ label: "ERR", kind: :error, title: sl_event_error_text(error) }
|
|
120
|
+
elsif sl_event_tool_error?(data)
|
|
121
|
+
{ label: "TOOL ERR", kind: :warn, title: "200 — the call succeeded but the tool reported isError" }
|
|
122
|
+
elsif data.any?
|
|
123
|
+
{ label: "200", kind: :ok, title: nil }
|
|
124
|
+
else
|
|
125
|
+
{ label: "—", kind: :unknown, title: "No response recorded" }
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def sl_event_status_badge(event)
|
|
130
|
+
status = sl_event_status(event)
|
|
131
|
+
if status[:kind] == :unknown
|
|
132
|
+
# No response was ever written back onto the event — either it is still
|
|
133
|
+
# in flight, or the process died before finalizing it. Both are worth
|
|
134
|
+
# seeing, and neither is a status code.
|
|
135
|
+
return tag.span(status[:label], class: "sl-metric sl-metric--blank", title: status[:title])
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
tag.span(status[:label],
|
|
139
|
+
class: "sl-badge sl-badge--http-#{status[:kind]}",
|
|
140
|
+
title: status[:title])
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# ---- Recency -----------------------------------------------------------
|
|
144
|
+
# Compact "how long ago", because on a list page the question is never the
|
|
145
|
+
# wall-clock timestamp, it is "is this thing still moving".
|
|
146
|
+
def sl_time_ago(time)
|
|
147
|
+
return "—" if time.blank?
|
|
148
|
+
|
|
149
|
+
seconds = (Time.current - time).to_f
|
|
150
|
+
return "just now" if seconds < 10
|
|
151
|
+
|
|
152
|
+
case seconds
|
|
153
|
+
when 10...60 then "#{seconds.round}s ago"
|
|
154
|
+
when 60...3600 then "#{(seconds / 60).floor}m ago"
|
|
155
|
+
when 3600...86400 then "#{(seconds / 3600).floor}h ago"
|
|
156
|
+
else "#{(seconds / 86400).floor}d ago"
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# An idle ACTIVE loop is the whole point of the Last Activity column: a
|
|
161
|
+
# `running` loop whose row has not been written for minutes is not working,
|
|
162
|
+
# it is hung. Terminal loops are supposed to be idle, so they are never
|
|
163
|
+
# flagged.
|
|
164
|
+
SL_IDLE_WARN_SECONDS = 5 * 60
|
|
165
|
+
SL_IDLE_STALLED_SECONDS = 15 * 60
|
|
166
|
+
|
|
167
|
+
def sl_idle_severity(loop, at: nil)
|
|
168
|
+
return nil unless SolidLoop::Loop::ACTIVE_STATUSES.include?(loop.status)
|
|
169
|
+
|
|
170
|
+
timestamp = at || loop.updated_at
|
|
171
|
+
return nil if timestamp.blank?
|
|
172
|
+
|
|
173
|
+
idle = (Time.current - timestamp).to_f
|
|
174
|
+
return :slow if idle >= SL_IDLE_STALLED_SECONDS
|
|
175
|
+
return :warn if idle >= SL_IDLE_WARN_SECONDS
|
|
176
|
+
|
|
177
|
+
nil
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# ---- Tokens ------------------------------------------------------------
|
|
181
|
+
def sl_tokens(count, blank: "—")
|
|
182
|
+
value = count.to_i
|
|
183
|
+
return blank if value.zero?
|
|
184
|
+
|
|
185
|
+
number_with_delimiter(value)
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# ---- Identifiers -------------------------------------------------------
|
|
189
|
+
# A 48-character MCP session id used to eat half the table. Show enough of
|
|
190
|
+
# it to recognise and to match against a server log, keep the whole thing
|
|
191
|
+
# one click away.
|
|
192
|
+
def sl_truncated_id(value, head: 20, tail: 6)
|
|
193
|
+
value = value.to_s
|
|
194
|
+
return "—" if value.empty?
|
|
195
|
+
return value if value.length <= head + tail + 1
|
|
196
|
+
|
|
197
|
+
"#{value[0, head]}…#{value[-tail, tail]}"
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Truncated id + a copy button. `navigator.clipboard` needs a secure
|
|
201
|
+
# context; the fallback keeps the full value selectable via the title
|
|
202
|
+
# attribute and a text selection on click.
|
|
203
|
+
def sl_copyable_id(value)
|
|
204
|
+
value = value.to_s
|
|
205
|
+
return tag.span("—", class: "sl-metric sl-metric--blank") if value.empty?
|
|
206
|
+
|
|
207
|
+
tag.span(class: "sl-copyable") do
|
|
208
|
+
concat tag.code(sl_truncated_id(value), class: "sl-copyable__text", title: value)
|
|
209
|
+
concat tag.button("Copy",
|
|
210
|
+
type: "button",
|
|
211
|
+
class: "sl-btn sl-btn--small sl-copyable__btn",
|
|
212
|
+
data: { clipboard: value },
|
|
213
|
+
onclick: "navigator.clipboard && navigator.clipboard.writeText(this.dataset.clipboard);" \
|
|
214
|
+
"var b=this;b.textContent='Copied';setTimeout(function(){b.textContent='Copy'},1200);")
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
private
|
|
219
|
+
|
|
220
|
+
def sl_event_error_text(error)
|
|
221
|
+
text = error.is_a?(Hash) ? (error["message"] || error[:message] || error.to_json) : error.to_s
|
|
222
|
+
text.length > 300 ? "#{text[0, 299]}…" : text
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def sl_event_tool_error?(data)
|
|
226
|
+
result = data["result"] || data[:result]
|
|
227
|
+
return false unless result.is_a?(Hash)
|
|
228
|
+
|
|
229
|
+
result["isError"] == true || result[:isError] == true
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
module SolidLoop
|
|
2
|
+
# A plain entry point for `SolidLoop.prune!`, installed by the host with
|
|
3
|
+
# whatever scheduler it already runs — the gem does NOT own a scheduler, for
|
|
4
|
+
# the same reason ReaperJob does not.
|
|
5
|
+
#
|
|
6
|
+
# Unlike the reaper this is NOT a per-minute concern: retention is measured in
|
|
7
|
+
# days, so daily is the natural cadence.
|
|
8
|
+
#
|
|
9
|
+
# # good_job:
|
|
10
|
+
# config.good_job.cron = {
|
|
11
|
+
# solid_loop_reaper: { cron: "* * * * *", class: "SolidLoop::ReaperJob" },
|
|
12
|
+
# solid_loop_janitor: { cron: "17 4 * * *", class: "SolidLoop::JanitorJob" }
|
|
13
|
+
# }
|
|
14
|
+
#
|
|
15
|
+
# A no-op unless at least one retention window is configured. Idempotent and
|
|
16
|
+
# safe to run concurrently.
|
|
17
|
+
class JanitorJob < ApplicationJob
|
|
18
|
+
queue_as :solid_loop_maintenance
|
|
19
|
+
|
|
20
|
+
def perform
|
|
21
|
+
SolidLoop.prune!
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -27,7 +27,7 @@ module SolidLoop
|
|
|
27
27
|
loop_for_lease.transition_status(
|
|
28
28
|
from: :queued,
|
|
29
29
|
to: :failed,
|
|
30
|
-
error_message: "#{e.class}: #{e.message}",
|
|
30
|
+
error_message: SolidLoop::Redaction.redact_credentials("#{e.class}: #{e.message}"),
|
|
31
31
|
execution_token: nil,
|
|
32
32
|
lease_expires_at: nil
|
|
33
33
|
)
|
|
@@ -119,7 +119,7 @@ module SolidLoop
|
|
|
119
119
|
# loop row is not a stale-worker no-op).
|
|
120
120
|
return false unless loop_record
|
|
121
121
|
|
|
122
|
-
error_msg = "#{error.class}: #{error.message}"
|
|
122
|
+
error_msg = SolidLoop::Redaction.redact_credentials("#{error.class}: #{error.message}")
|
|
123
123
|
Rails.logger.error "LlmCompletionJob reconciling uncaught error: #{error_msg}\n#{error.backtrace&.take(10)&.join("\n")}"
|
|
124
124
|
|
|
125
125
|
loop_record.transition_status(
|
|
@@ -85,17 +85,65 @@ module SolidLoop
|
|
|
85
85
|
:open_ai
|
|
86
86
|
end
|
|
87
87
|
|
|
88
|
+
# How previous turns' reasoning is sent back to the model. `nil` (the
|
|
89
|
+
# default) defers to the dialect, which picks the shape its provider
|
|
90
|
+
# actually has a slot for — `"reasoning_content"` on `open_ai`, `:xml`
|
|
91
|
+
# elsewhere. Return an explicit array to override, including `[]` to send no
|
|
92
|
+
# reasoning back at all.
|
|
93
|
+
#
|
|
94
|
+
# This matters more than it looks. `:xml` welds every past thought into the
|
|
95
|
+
# assistant message body, where it is billed as prompt tokens on every
|
|
96
|
+
# subsequent turn and compounds: measured agent loops were spending 40% of
|
|
97
|
+
# their context window on repacked reasoning that the chat template then
|
|
98
|
+
# discarded. See docs/decisions/reasoning_persistence.md.
|
|
88
99
|
def reasoning_strategies
|
|
89
|
-
|
|
90
|
-
[ :xml ]
|
|
100
|
+
nil
|
|
91
101
|
end
|
|
92
102
|
|
|
93
103
|
def mcp_principal
|
|
94
104
|
subject
|
|
95
105
|
end
|
|
96
106
|
|
|
107
|
+
# How hard the model should think, when the provider exposes that as a
|
|
108
|
+
# discrete level. `nil` (the default) sends nothing and leaves the provider's
|
|
109
|
+
# own default in place. Each dialect renders it into its native shape:
|
|
110
|
+
#
|
|
111
|
+
# open_ai reasoning_effort: "low"
|
|
112
|
+
# anthropic output_config: { effort: "low" }
|
|
113
|
+
# gemini generationConfig.thinkingConfig.thinkingLevel: "LOW"
|
|
114
|
+
#
|
|
115
|
+
# The accepted levels differ by provider (OpenAI-compatible endpoints take
|
|
116
|
+
# minimal/low/medium/high; Anthropic adds xhigh and max; Gemini takes
|
|
117
|
+
# LOW/MEDIUM/HIGH), so the value is passed through rather than validated
|
|
118
|
+
# here — the provider is the authority on its own vocabulary.
|
|
119
|
+
def reasoning_effort
|
|
120
|
+
nil
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Cap on generated tokens, sent as `max_tokens` when set. `nil` (the default)
|
|
124
|
+
# sends NOTHING — the provider applies its own default, which is what most
|
|
125
|
+
# OpenAI-compatible endpoints want; a gem-chosen number would silently
|
|
126
|
+
# truncate responses that were previously uncapped. Override to opt in.
|
|
97
127
|
def max_tokens
|
|
98
|
-
|
|
128
|
+
nil
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Extra provider payload keys, deep-merged OVER the built payload right
|
|
132
|
+
# before the dialect renders it. The built payload is otherwise closed
|
|
133
|
+
# (model, messages, stream, tools), so this is the seam for anything the
|
|
134
|
+
# gem does not model itself: `reasoning_effort`, `temperature`, `top_p`,
|
|
135
|
+
# `max_tokens`, provider-specific extensions.
|
|
136
|
+
#
|
|
137
|
+
# def llm_params
|
|
138
|
+
# { reasoning_effort: "low", temperature: 0.2 }
|
|
139
|
+
# end
|
|
140
|
+
#
|
|
141
|
+
# Merged last, so a host can also override a key the gem set (e.g. force
|
|
142
|
+
# `stream: false` for one agent). Keys reach the wire as written for the
|
|
143
|
+
# OpenAI dialect; the Anthropic and Gemini dialects re-render the payload,
|
|
144
|
+
# so only keys those dialects read survive there.
|
|
145
|
+
def llm_params
|
|
146
|
+
{}
|
|
99
147
|
end
|
|
100
148
|
|
|
101
149
|
def max_steps
|
|
@@ -110,6 +158,9 @@ module SolidLoop
|
|
|
110
158
|
10.0
|
|
111
159
|
end
|
|
112
160
|
|
|
161
|
+
# Budget of ACTUAL WORK — LLM generation plus tool execution — not wall clock
|
|
162
|
+
# since the loop was created. Time spent paused, failed, or queued does not
|
|
163
|
+
# count against it (see `Loop#work_duration`).
|
|
113
164
|
def max_duration
|
|
114
165
|
2.hours
|
|
115
166
|
end
|
|
@@ -143,13 +194,45 @@ module SolidLoop
|
|
|
143
194
|
end
|
|
144
195
|
end
|
|
145
196
|
|
|
146
|
-
|
|
197
|
+
# `from:` narrows which statuses may be resumed, defaulting to all of
|
|
198
|
+
# SolidLoop::Loop::RESUMABLE_STATUSES. A caller that means "restart what
|
|
199
|
+
# stalled" should pass `from: %i[paused failed]`: the default also re-enters
|
|
200
|
+
# a `completed` loop, which is right for continuing a conversation and wrong
|
|
201
|
+
# for anything that has already reported a final result — a benchmark run,
|
|
202
|
+
# a settled ticket — where the resumed turn silently appends to it.
|
|
203
|
+
#
|
|
204
|
+
# Returns false when the loop is not in an accepted status, exactly as
|
|
205
|
+
# before; passing a status that can NEVER be resumed (`running`, `queued`) is
|
|
206
|
+
# a caller error and raises, because it would otherwise no-op forever.
|
|
207
|
+
def resume!(from: SolidLoop::Loop::RESUMABLE_STATUSES)
|
|
208
|
+
accepted = Array(from).map(&:to_s)
|
|
209
|
+
unknown = accepted - SolidLoop::Loop::RESUMABLE_STATUSES
|
|
210
|
+
if unknown.any?
|
|
211
|
+
raise ArgumentError,
|
|
212
|
+
"resume! cannot accept #{unknown.join(', ')}; " \
|
|
213
|
+
"resumable statuses are #{SolidLoop::Loop::RESUMABLE_STATUSES.join(', ')}"
|
|
214
|
+
end
|
|
215
|
+
|
|
147
216
|
# State advancement and dispatch commit atomically inside the lock's
|
|
148
217
|
# transaction (same-database transactional job backend): a crash leaves
|
|
149
218
|
# neither a running loop without jobs nor jobs for a rolled-back resume.
|
|
150
219
|
loop_record.with_lock do
|
|
151
|
-
raise
|
|
152
|
-
return false unless
|
|
220
|
+
raise SolidLoop::FrozenLoopError, "Unfreeze in admin UI before resuming this loop" if loop_record.admin_frozen?
|
|
221
|
+
return false unless accepted.include?(loop_record.status)
|
|
222
|
+
|
|
223
|
+
# Mirror of `pause!`'s cleanup. Any assistant shell still `processing`
|
|
224
|
+
# here belongs to a DEAD generation by construction — none of the
|
|
225
|
+
# resumable statuses has a live turn — and nothing else will ever finish
|
|
226
|
+
# it: the graceful-error path in Middlewares::ErrorHandling only
|
|
227
|
+
# finalizes the shell its own attempt created, and its token-fenced CAS
|
|
228
|
+
# no-ops entirely once a resume has rotated the generation, which leaves
|
|
229
|
+
# that shell orphaned. It stays hidden (born-hidden protocol) so a
|
|
230
|
+
# partial never re-enters a retried turn as history; marking it failed
|
|
231
|
+
# keeps `processing` meaning "a turn is actually in flight" instead of
|
|
232
|
+
# accumulating phantoms in the admin UI.
|
|
233
|
+
loop_record.messages.where(role: "assistant", status: "processing").find_each do |shell|
|
|
234
|
+
shell.update!(status: "failed", is_hidden: true)
|
|
235
|
+
end
|
|
153
236
|
|
|
154
237
|
pending_tool_calls = loop_record.unresolved_tool_calls.to_a
|
|
155
238
|
if pending_tool_calls.any?
|
|
@@ -22,8 +22,30 @@ module SolidLoop
|
|
|
22
22
|
}, default: :init
|
|
23
23
|
|
|
24
24
|
ACTIVE_STATUSES = %w[init queued running].freeze
|
|
25
|
+
|
|
26
|
+
# Seconds this loop has actually WORKED: LLM generation plus tool execution.
|
|
27
|
+
# Derived rather than stored so it can never drift from the counters that
|
|
28
|
+
# feed it (`duration_generation` is incremented on both the success and the
|
|
29
|
+
# failure path; `duration_tools` on each tool invocation).
|
|
30
|
+
#
|
|
31
|
+
# This is what `max_duration` budgets against. Wall clock since `created_at`
|
|
32
|
+
# would also count time the loop spent paused, failed, or simply waiting in a
|
|
33
|
+
# queue — so a loop resumed after an outage could exhaust its whole budget
|
|
34
|
+
# having done none of the work, and a loop paused overnight would be dead on
|
|
35
|
+
# arrival. Work that a hard kill lost is not counted either, which is the
|
|
36
|
+
# right direction to err: you do not spend budget on work you did not get.
|
|
37
|
+
def work_duration
|
|
38
|
+
duration_generation.to_f + duration_tools.to_f
|
|
39
|
+
end
|
|
25
40
|
STOPPABLE_STATUSES = (ACTIVE_STATUSES + [ "paused" ]).freeze
|
|
26
41
|
|
|
42
|
+
# Statuses `Base#resume!` will re-enter. `completed` is included on purpose:
|
|
43
|
+
# continuing a finished conversation is a legitimate operation for a chat.
|
|
44
|
+
# It is also the one that surprises callers who mean "restart what stalled" —
|
|
45
|
+
# a resumed `completed` loop appends to a run that had already reported its
|
|
46
|
+
# result — so `resume!` takes a `from:` to narrow this set.
|
|
47
|
+
RESUMABLE_STATUSES = %w[init paused failed completed].freeze
|
|
48
|
+
|
|
27
49
|
def admin_frozen?
|
|
28
50
|
frozen_at.present?
|
|
29
51
|
end
|
|
@@ -17,16 +17,42 @@ module SolidLoop
|
|
|
17
17
|
tool: "tool"
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
# Shortest decode window (seconds) from which a throughput figure is believed.
|
|
21
|
+
# Below it the ratio is dominated by measurement noise: when the whole
|
|
22
|
+
# response arrives in a single chunk, `duration_generation` and `ttft` nearly
|
|
23
|
+
# coincide, the denominator collapses toward zero and the quotient explodes
|
|
24
|
+
# into the thousands. Measured against real traffic, one turn in ~200 lands
|
|
25
|
+
# there and reports ~9200 tok/s; every other turn sits under 80.
|
|
26
|
+
MIN_DECODE_WINDOW = 0.25
|
|
27
|
+
|
|
28
|
+
# Tokens per second over the DECODE window only — prefill (ttft) is excluded,
|
|
29
|
+
# so this is generation speed rather than end-to-end latency. Returns 0.0
|
|
30
|
+
# when the window is too short to measure, which callers render as "no data"
|
|
31
|
+
# rather than as a zero throughput.
|
|
32
|
+
def self.derive_tps(tokens_completion:, duration_generation:, ttft:)
|
|
33
|
+
tokens = tokens_completion.to_i
|
|
34
|
+
return 0.0 unless tokens.positive?
|
|
22
35
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
actual_gen_time = duration_generation.to_f - ttft.to_f
|
|
36
|
+
window = duration_generation.to_f - ttft.to_f
|
|
37
|
+
return 0.0 if window < MIN_DECODE_WINDOW
|
|
26
38
|
|
|
27
|
-
|
|
39
|
+
(tokens / window).round(2)
|
|
40
|
+
end
|
|
28
41
|
|
|
29
|
-
|
|
42
|
+
# Prefers the PERSISTED column so Ruby and SQL agree: aggregating `tps` in a
|
|
43
|
+
# query and reading `message.tps` must not disagree. Falls back to deriving
|
|
44
|
+
# it for rows written before the column was populated for every provider
|
|
45
|
+
# (only the llama.cpp parser reported a rate; the OpenAI/Anthropic/Gemini
|
|
46
|
+
# parsers stored 0.0, so historical rows on those providers hold nothing).
|
|
47
|
+
def tps
|
|
48
|
+
persisted = self[:tps].to_f
|
|
49
|
+
return persisted if persisted.positive?
|
|
50
|
+
|
|
51
|
+
self.class.derive_tps(
|
|
52
|
+
tokens_completion: tokens_completion,
|
|
53
|
+
duration_generation: duration_generation,
|
|
54
|
+
ttft: ttft
|
|
55
|
+
)
|
|
30
56
|
end
|
|
31
57
|
|
|
32
58
|
def cumulative_duration
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
module SolidLoop
|
|
2
2
|
class ToolCall < ApplicationRecord
|
|
3
|
-
|
|
3
|
+
# `touch: true`: a tool_call's OUTCOME (result / is_success / executed_at)
|
|
4
|
+
# fills in via a nested write LONG after its parent assistant message was
|
|
5
|
+
# created. Touching the message on every tool_call write bumps the message's
|
|
6
|
+
# `updated_at`, so a host polling messages by `updated_at > cursor` re-syncs
|
|
7
|
+
# the enriched tool_call even once that message is no longer the newest —
|
|
8
|
+
# otherwise the outcome (e.g. a rejection) can stay stuck client-side.
|
|
9
|
+
belongs_to :message, inverse_of: :tool_calls, touch: true
|
|
4
10
|
belongs_to :mcp_session, class_name: "SolidLoop::McpSession", optional: true, inverse_of: :tool_calls
|
|
5
11
|
|
|
6
12
|
scope :ordered, -> { order(:id) }
|