rubyllm-observ 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +56 -8
- data/app/assets/stylesheets/observ/_annotations.scss +114 -103
- data/app/assets/stylesheets/observ/_card.scss +58 -49
- data/app/assets/stylesheets/observ/_chat.scss +247 -155
- data/app/assets/stylesheets/observ/_components.scss +622 -340
- data/app/assets/stylesheets/observ/_dashboard.scss +31 -28
- data/app/assets/stylesheets/observ/_datasets.scss +494 -547
- data/app/assets/stylesheets/observ/_drawer.scss +250 -228
- data/app/assets/stylesheets/observ/_filters.scss +139 -0
- data/app/assets/stylesheets/observ/_json_viewer.scss +103 -97
- data/app/assets/stylesheets/observ/_layout.scss +443 -178
- data/app/assets/stylesheets/observ/_metrics.scss +79 -76
- data/app/assets/stylesheets/observ/_namespace.scss +18 -0
- data/app/assets/stylesheets/observ/_observations.scss +122 -119
- data/app/assets/stylesheets/observ/_pagination.scss +129 -112
- data/app/assets/stylesheets/observ/_prompts.scss +485 -269
- data/app/assets/stylesheets/observ/_reset.scss +249 -0
- data/app/assets/stylesheets/observ/_table.scss +46 -38
- data/app/assets/stylesheets/observ/_variables.scss +54 -0
- data/app/assets/stylesheets/observ/application.scss +3 -0
- data/app/controllers/observ/dataset_run_items_controller.rb +0 -1
- data/app/controllers/observ/review_queue_controller.rb +154 -0
- data/app/controllers/observ/scores_controller.rb +64 -0
- data/app/controllers/observ/sessions_controller.rb +23 -0
- data/app/helpers/observ/application_helper.rb +1 -0
- data/app/helpers/observ/reviews_helper.rb +33 -0
- data/app/models/concerns/observ/json_queryable.rb +138 -0
- data/app/models/concerns/observ/reviewable.rb +41 -0
- data/app/models/concerns/observ/scoreable.rb +34 -0
- data/app/models/observ/dataset_run_item.rb +3 -13
- data/app/models/observ/review_item.rb +48 -0
- data/app/models/observ/score.rb +38 -6
- data/app/models/observ/session.rb +5 -1
- data/app/models/observ/trace.rb +3 -0
- data/app/services/observ/evaluators/base_evaluator.rb +0 -1
- data/app/services/observ/guardrail_service.rb +128 -0
- data/app/views/kaminari/_first_page.html.erb +1 -1
- data/app/views/kaminari/_gap.html.erb +1 -1
- data/app/views/kaminari/_last_page.html.erb +1 -1
- data/app/views/kaminari/_next_page.html.erb +1 -1
- data/app/views/kaminari/_page.html.erb +1 -1
- data/app/views/kaminari/_paginator.html.erb +1 -1
- data/app/views/kaminari/_prev_page.html.erb +1 -1
- data/app/views/kaminari/observ/_first_page.html.erb +1 -1
- data/app/views/kaminari/observ/_gap.html.erb +1 -1
- data/app/views/kaminari/observ/_last_page.html.erb +1 -1
- data/app/views/kaminari/observ/_next_page.html.erb +1 -1
- data/app/views/kaminari/observ/_page.html.erb +1 -1
- data/app/views/kaminari/observ/_paginator.html.erb +1 -1
- data/app/views/kaminari/observ/_prev_page.html.erb +1 -1
- data/app/views/layouts/observ/application.html.erb +97 -59
- data/app/views/observ/annotations/_form.html.erb +5 -5
- data/app/views/observ/annotations/index.html.erb +4 -4
- data/app/views/observ/annotations/sessions_index.html.erb +9 -9
- data/app/views/observ/annotations/traces_index.html.erb +9 -9
- data/app/views/observ/chats/_form.html.erb +7 -7
- data/app/views/observ/datasets/index.html.erb +6 -6
- data/app/views/observ/messages/_form.html.erb +11 -12
- data/app/views/observ/observations/index.html.erb +3 -4
- data/app/views/observ/prompts/_form.html.erb +37 -38
- data/app/views/observ/prompts/_new_form.html.erb +37 -38
- data/app/views/observ/prompts/compare.html.erb +59 -55
- data/app/views/observ/prompts/edit.html.erb +3 -3
- data/app/views/observ/prompts/index.html.erb +9 -9
- data/app/views/observ/prompts/new.html.erb +3 -3
- data/app/views/observ/prompts/show.html.erb +2 -2
- data/app/views/observ/prompts/versions.html.erb +22 -22
- data/app/views/observ/review_queue/_item.html.erb +39 -0
- data/app/views/observ/review_queue/_stats.html.erb +18 -0
- data/app/views/observ/review_queue/index.html.erb +49 -0
- data/app/views/observ/review_queue/show.html.erb +76 -0
- data/app/views/observ/review_queue/stats.html.erb +100 -0
- data/app/views/observ/scores/_form.html.erb +39 -0
- data/app/views/observ/scores/create.turbo_stream.erb +10 -0
- data/app/views/observ/sessions/_chat.html.erb +59 -0
- data/app/views/observ/sessions/_metadata.html.erb +17 -0
- data/app/views/observ/sessions/_metrics.html.erb +81 -0
- data/app/views/observ/sessions/_traces.html.erb +92 -0
- data/app/views/observ/sessions/annotations_drawer.turbo_stream.erb +8 -1
- data/app/views/observ/sessions/index.html.erb +60 -4
- data/app/views/observ/sessions/show.html.erb +4 -217
- data/app/views/observ/traces/_details.html.erb +47 -0
- data/app/views/observ/traces/_input.html.erb +10 -0
- data/app/views/observ/traces/_metadata.html.erb +10 -0
- data/app/views/observ/traces/_observations.html.erb +172 -0
- data/app/views/observ/traces/_output.html.erb +10 -0
- data/app/views/observ/traces/annotations_drawer.turbo_stream.erb +8 -1
- data/app/views/observ/traces/index.html.erb +3 -4
- data/app/views/observ/traces/show.html.erb +5 -232
- data/config/routes.rb +14 -0
- data/db/migrate/015_refactor_scores_to_polymorphic.rb +27 -0
- data/db/migrate/016_create_observ_review_items.rb +25 -0
- data/lib/generators/observ/install/install_generator.rb +61 -9
- data/lib/generators/observ/install/templates/observ.js +19 -0
- data/lib/observ/version.rb +1 -1
- metadata +31 -2
- data/app/assets/javascripts/observ/controllers/index.js +0 -52
|
@@ -46,240 +46,13 @@
|
|
|
46
46
|
<% end %>
|
|
47
47
|
|
|
48
48
|
<div class="observ-container">
|
|
49
|
-
|
|
50
|
-
<header class="observ-card__header">
|
|
51
|
-
<h2 class="observ-card__title">Details</h2>
|
|
52
|
-
</header>
|
|
53
|
-
<div class="observ-card__body">
|
|
54
|
-
<dl class="observ-definition-list observ-definition-list--compact">
|
|
55
|
-
<div class="observ-definition-list__item">
|
|
56
|
-
<dt class="observ-definition-list__term">Session</dt>
|
|
57
|
-
<dd class="observ-definition-list__definition">
|
|
58
|
-
<%= link_to truncate_id(@trace.observ_session.session_id, 12), session_path(@trace.observ_session), class: "observ-link" %>
|
|
59
|
-
</dd>
|
|
60
|
-
</div>
|
|
61
|
-
<div class="observ-definition-list__item">
|
|
62
|
-
<dt class="observ-definition-list__term">Name</dt>
|
|
63
|
-
<dd class="observ-definition-list__definition"><%= @trace.name || "—" %></dd>
|
|
64
|
-
</div>
|
|
65
|
-
<div class="observ-definition-list__item">
|
|
66
|
-
<dt class="observ-definition-list__term">Start Time</dt>
|
|
67
|
-
<dd class="observ-definition-list__definition"><%= observ_timestamp(@trace.start_time) %></dd>
|
|
68
|
-
</div>
|
|
69
|
-
<% if @trace.end_time %>
|
|
70
|
-
<div class="observ-definition-list__item">
|
|
71
|
-
<dt class="observ-definition-list__term">End Time</dt>
|
|
72
|
-
<dd class="observ-definition-list__definition"><%= observ_timestamp(@trace.end_time) %></dd>
|
|
73
|
-
</div>
|
|
74
|
-
<% end %>
|
|
75
|
-
<div class="observ-definition-list__item">
|
|
76
|
-
<dt class="observ-definition-list__term">Duration</dt>
|
|
77
|
-
<dd class="observ-definition-list__definition">
|
|
78
|
-
<% if @trace.duration_ms %>
|
|
79
|
-
<%= format_duration_ms(@trace.duration_ms) %>
|
|
80
|
-
<% else %>
|
|
81
|
-
<span class="observ-text--muted">In Progress</span>
|
|
82
|
-
<% end %>
|
|
83
|
-
</dd>
|
|
84
|
-
</div>
|
|
85
|
-
<div class="observ-definition-list__item">
|
|
86
|
-
<dt class="observ-definition-list__term">Total Tokens</dt>
|
|
87
|
-
<dd class="observ-definition-list__definition"><%= format_tokens(@trace.total_tokens) %></dd>
|
|
88
|
-
</div>
|
|
89
|
-
<div class="observ-definition-list__item">
|
|
90
|
-
<dt class="observ-definition-list__term">Total Cost</dt>
|
|
91
|
-
<dd class="observ-definition-list__definition"><%= format_currency(@trace.total_cost) %></dd>
|
|
92
|
-
</div>
|
|
93
|
-
</dl>
|
|
94
|
-
</div>
|
|
95
|
-
</section>
|
|
96
|
-
|
|
97
|
-
<% if @trace.input.present? %>
|
|
98
|
-
<section class="observ-card observ-card--compact">
|
|
99
|
-
<header class="observ-card__header">
|
|
100
|
-
<h2 class="observ-card__title">Input</h2>
|
|
101
|
-
</header>
|
|
102
|
-
<div class="observ-card__body">
|
|
103
|
-
<%= render_input_output(@trace.input, compact: true) %>
|
|
104
|
-
</div>
|
|
105
|
-
</section>
|
|
106
|
-
<% end %>
|
|
107
|
-
|
|
108
|
-
<% if @trace.output.present? %>
|
|
109
|
-
<section class="observ-card observ-card--compact">
|
|
110
|
-
<header class="observ-card__header">
|
|
111
|
-
<h2 class="observ-card__title">Output</h2>
|
|
112
|
-
</header>
|
|
113
|
-
<div class="observ-card__body">
|
|
114
|
-
<%= render_input_output(@trace.output, compact: true) %>
|
|
115
|
-
</div>
|
|
116
|
-
</section>
|
|
117
|
-
<% end %>
|
|
118
|
-
|
|
119
|
-
<% if @trace.metadata.present? && @trace.metadata.any? %>
|
|
120
|
-
<section class="observ-card observ-card--compact">
|
|
121
|
-
<header class="observ-card__header">
|
|
122
|
-
<h2 class="observ-card__title">Metadata</h2>
|
|
123
|
-
</header>
|
|
124
|
-
<div class="observ-card__body">
|
|
125
|
-
<%= render_json_viewer(@trace.metadata, compact: true) %>
|
|
126
|
-
</div>
|
|
127
|
-
</section>
|
|
128
|
-
<% end %>
|
|
129
|
-
|
|
130
|
-
<section class="observ-card">
|
|
131
|
-
<header class="observ-card__header">
|
|
132
|
-
<h2 class="observ-card__title">Observations (<%= @observations.count %>)</h2>
|
|
133
|
-
</header>
|
|
134
|
-
<div class="observ-card__body">
|
|
135
|
-
<% if @observations.any? %>
|
|
136
|
-
<div class="observ-observations-list observ-observations-list--compact">
|
|
137
|
-
<% @observations.each do |observation| %>
|
|
138
|
-
<div class="observ-observation-card observ-observation-card--compact observ-observation-card--<%= observation.type.demodulize.downcase %>">
|
|
139
|
-
<div class="observ-observation-card__header">
|
|
140
|
-
<span class="observ-observation-card__type-badge">
|
|
141
|
-
<%= observation.type.demodulize %>
|
|
142
|
-
</span>
|
|
143
|
-
<h3 class="observ-observation-card__name"><%= observation.name %></h3>
|
|
144
|
-
<div class="observ-observation-card__meta">
|
|
145
|
-
<span class="observ-observation-card__id">
|
|
146
|
-
<code class="observ-code observ-code--inline"><%= truncate_id(observation.observation_id, 8) %></code>
|
|
147
|
-
</span>
|
|
148
|
-
</div>
|
|
149
|
-
</div>
|
|
150
|
-
|
|
151
|
-
<div class="observ-observation-card__body">
|
|
152
|
-
<dl class="observ-definition-list observ-definition-list--horizontal observ-definition-list--compact">
|
|
153
|
-
<% if observation.is_a?(Observ::Generation) %>
|
|
154
|
-
<div class="observ-definition-list__item">
|
|
155
|
-
<dt class="observ-definition-list__term">Model</dt>
|
|
156
|
-
<dd class="observ-definition-list__definition">
|
|
157
|
-
<%= observ_model_badge(observation.model) %>
|
|
158
|
-
</dd>
|
|
159
|
-
</div>
|
|
160
|
-
<div class="observ-definition-list__item">
|
|
161
|
-
<dt class="observ-definition-list__term">Tokens</dt>
|
|
162
|
-
<dd class="observ-definition-list__definition">
|
|
163
|
-
<%= format_tokens(observation.total_tokens) %>
|
|
164
|
-
</dd>
|
|
165
|
-
</div>
|
|
166
|
-
<div class="observ-definition-list__item">
|
|
167
|
-
<dt class="observ-definition-list__term">Cost</dt>
|
|
168
|
-
<dd class="observ-definition-list__definition">
|
|
169
|
-
<%= format_currency(observation.cost_usd) %>
|
|
170
|
-
</dd>
|
|
171
|
-
</div>
|
|
172
|
-
<% if observation.finish_reason %>
|
|
173
|
-
<div class="observ-definition-list__item">
|
|
174
|
-
<dt class="observ-definition-list__term">Finish</dt>
|
|
175
|
-
<dd class="observ-definition-list__definition">
|
|
176
|
-
<%= observation.finish_reason %>
|
|
177
|
-
</dd>
|
|
178
|
-
</div>
|
|
179
|
-
<% end %>
|
|
180
|
-
<% if observation.time_to_first_token_ms %>
|
|
181
|
-
<div class="observ-definition-list__item">
|
|
182
|
-
<dt class="observ-definition-list__term">TTFT</dt>
|
|
183
|
-
<dd class="observ-definition-list__definition">
|
|
184
|
-
<%= format_duration_ms(observation.time_to_first_token_ms) %>
|
|
185
|
-
</dd>
|
|
186
|
-
</div>
|
|
187
|
-
<% end %>
|
|
188
|
-
<div class="observ-definition-list__item">
|
|
189
|
-
<dt class="observ-definition-list__term">Prompt</dt>
|
|
190
|
-
<dd class="observ-definition-list__definition">
|
|
191
|
-
<% if observation.prompt_name.present? %>
|
|
192
|
-
<%= observation.prompt_name %>
|
|
193
|
-
<% if observation.prompt_version.present? %>
|
|
194
|
-
<span class="observ-text--muted">(v<%= observation.prompt_version %>)</span>
|
|
195
|
-
<% end %>
|
|
196
|
-
<% else %>
|
|
197
|
-
<span class="observ-text--muted">default prompt</span>
|
|
198
|
-
<% end %>
|
|
199
|
-
</dd>
|
|
200
|
-
</div>
|
|
201
|
-
<% end %>
|
|
202
|
-
<div class="observ-definition-list__item">
|
|
203
|
-
<dt class="observ-definition-list__term">Duration</dt>
|
|
204
|
-
<dd class="observ-definition-list__definition">
|
|
205
|
-
<% if observation.duration_ms %>
|
|
206
|
-
<%= format_duration_ms(observation.duration_ms) %>
|
|
207
|
-
<% else %>
|
|
208
|
-
<span class="observ-text--muted">—</span>
|
|
209
|
-
<% end %>
|
|
210
|
-
</dd>
|
|
211
|
-
</div>
|
|
212
|
-
<% if observation.is_a?(Observ::Span) && observation.status_message %>
|
|
213
|
-
<div class="observ-definition-list__item">
|
|
214
|
-
<dt class="observ-definition-list__term">Status</dt>
|
|
215
|
-
<dd class="observ-definition-list__definition">
|
|
216
|
-
<%= observation.status_message %>
|
|
217
|
-
</dd>
|
|
218
|
-
</div>
|
|
219
|
-
<% end %>
|
|
220
|
-
</dl>
|
|
221
|
-
</div>
|
|
49
|
+
<%= render "observ/traces/details", trace: @trace %>
|
|
222
50
|
|
|
223
|
-
|
|
224
|
-
<div class="observ-observation-card__section observ-observation-card__section--compact">
|
|
225
|
-
<h4 class="observ-observation-card__section-title">Token Usage</h4>
|
|
226
|
-
<dl class="observ-definition-list observ-definition-list--horizontal observ-definition-list--compact">
|
|
227
|
-
<% observation.usage.each do |key, value| %>
|
|
228
|
-
<div class="observ-definition-list__item">
|
|
229
|
-
<dt class="observ-definition-list__term"><%= key.humanize %></dt>
|
|
230
|
-
<dd class="observ-definition-list__definition"><%= format_tokens(value) %></dd>
|
|
231
|
-
</div>
|
|
232
|
-
<% end %>
|
|
233
|
-
</dl>
|
|
234
|
-
</div>
|
|
235
|
-
<% end %>
|
|
51
|
+
<%= render "observ/traces/input", trace: @trace %>
|
|
236
52
|
|
|
237
|
-
|
|
238
|
-
<div class="observ-observation-card__section observ-observation-card__section--compact">
|
|
239
|
-
<h4 class="observ-observation-card__section-title">Input</h4>
|
|
240
|
-
<pre class="observ-code-block observ-code-block--compact"><%= observation.input %></pre>
|
|
241
|
-
</div>
|
|
242
|
-
<% end %>
|
|
53
|
+
<%= render "observ/traces/output", trace: @trace %>
|
|
243
54
|
|
|
244
|
-
|
|
245
|
-
<div class="observ-observation-card__section observ-observation-card__section--compact">
|
|
246
|
-
<h4 class="observ-observation-card__section-title">Output</h4>
|
|
247
|
-
<pre class="observ-code-block observ-code-block--compact"><%= observation.output %></pre>
|
|
248
|
-
</div>
|
|
249
|
-
<% end %>
|
|
55
|
+
<%= render "observ/traces/metadata", trace: @trace %>
|
|
250
56
|
|
|
251
|
-
|
|
252
|
-
<div class="observ-observation-card__section observ-observation-card__section--compact">
|
|
253
|
-
<h4 class="observ-observation-card__section-title">Messages</h4>
|
|
254
|
-
<%= render_json_viewer(observation.messages, compact: true) %>
|
|
255
|
-
</div>
|
|
256
|
-
<% end %>
|
|
257
|
-
|
|
258
|
-
<% if observation.is_a?(Observ::Generation) && observation.provider_metadata.present? && observation.provider_metadata.any? %>
|
|
259
|
-
<div class="observ-observation-card__section observ-observation-card__section--compact">
|
|
260
|
-
<h4 class="observ-observation-card__section-title">Provider Metadata</h4>
|
|
261
|
-
<%= render_json_viewer(observation.provider_metadata, compact: true) %>
|
|
262
|
-
</div>
|
|
263
|
-
<% end %>
|
|
264
|
-
|
|
265
|
-
<% if observation.is_a?(Observ::Span) && observation.metadata.present? && observation.metadata.any? %>
|
|
266
|
-
<div class="observ-observation-card__section observ-observation-card__section--compact">
|
|
267
|
-
<h4 class="observ-observation-card__section-title">Metadata</h4>
|
|
268
|
-
<%= render_json_viewer(observation.metadata, compact: true) %>
|
|
269
|
-
</div>
|
|
270
|
-
<% end %>
|
|
271
|
-
|
|
272
|
-
<div class="observ-observation-card__footer">
|
|
273
|
-
<%= link_to "View Details →", observation_path(observation), class: "observ-link" %>
|
|
274
|
-
</div>
|
|
275
|
-
</div>
|
|
276
|
-
<% end %>
|
|
277
|
-
</div>
|
|
278
|
-
<% else %>
|
|
279
|
-
<div class="observ-card__empty">
|
|
280
|
-
<p>No observations found for this trace.</p>
|
|
281
|
-
</div>
|
|
282
|
-
<% end %>
|
|
283
|
-
</div>
|
|
284
|
-
</section>
|
|
57
|
+
<%= render "observ/traces/observations", observations: @observations, trace: @trace %>
|
|
285
58
|
</div>
|
data/config/routes.rb
CHANGED
|
@@ -19,6 +19,7 @@ Observ::Engine.routes.draw do
|
|
|
19
19
|
get :annotations_drawer
|
|
20
20
|
end
|
|
21
21
|
resources :annotations, only: [ :index, :create, :destroy ]
|
|
22
|
+
resources :scores, only: [ :create, :destroy ]
|
|
22
23
|
end
|
|
23
24
|
|
|
24
25
|
resources :traces, only: [ :index, :show ] do
|
|
@@ -32,6 +33,7 @@ Observ::Engine.routes.draw do
|
|
|
32
33
|
post :add_to_dataset
|
|
33
34
|
end
|
|
34
35
|
resources :annotations, only: [ :index, :create, :destroy ]
|
|
36
|
+
resources :scores, only: [ :create, :destroy ]
|
|
35
37
|
end
|
|
36
38
|
|
|
37
39
|
resources :observations, only: [ :index, :show ] do
|
|
@@ -45,6 +47,18 @@ Observ::Engine.routes.draw do
|
|
|
45
47
|
get "annotations/traces", to: "annotations#traces_index", as: :traces_annotations
|
|
46
48
|
get "annotations/export", to: "annotations#export", as: :export_annotations
|
|
47
49
|
|
|
50
|
+
resources :reviews, only: [ :index, :show ], controller: "review_queue" do
|
|
51
|
+
collection do
|
|
52
|
+
get :sessions
|
|
53
|
+
get :traces
|
|
54
|
+
get :stats
|
|
55
|
+
end
|
|
56
|
+
member do
|
|
57
|
+
post :complete
|
|
58
|
+
post :skip
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
48
62
|
resources :prompts do
|
|
49
63
|
member do
|
|
50
64
|
get :versions # Version history view
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class RefactorScoresToPolymorphic < ActiveRecord::Migration[7.0]
|
|
4
|
+
def change
|
|
5
|
+
# Add polymorphic columns
|
|
6
|
+
add_column :observ_scores, :scoreable_type, :string
|
|
7
|
+
add_column :observ_scores, :scoreable_id, :bigint
|
|
8
|
+
|
|
9
|
+
# Remove old foreign keys
|
|
10
|
+
remove_foreign_key :observ_scores, :observ_dataset_run_items, column: :dataset_run_item_id
|
|
11
|
+
remove_foreign_key :observ_scores, :observ_traces, column: :trace_id
|
|
12
|
+
|
|
13
|
+
# Remove old indexes that reference the columns we're dropping
|
|
14
|
+
remove_index :observ_scores, name: "idx_scores_on_run_item_name_source"
|
|
15
|
+
remove_index :observ_scores, :trace_id
|
|
16
|
+
|
|
17
|
+
# Remove old columns
|
|
18
|
+
remove_column :observ_scores, :dataset_run_item_id, :bigint
|
|
19
|
+
remove_column :observ_scores, :trace_id, :bigint
|
|
20
|
+
|
|
21
|
+
# Add new indexes
|
|
22
|
+
add_index :observ_scores, [ :scoreable_type, :scoreable_id ]
|
|
23
|
+
add_index :observ_scores, [ :scoreable_type, :scoreable_id, :name, :source ],
|
|
24
|
+
unique: true,
|
|
25
|
+
name: "idx_scores_unique_on_scoreable_name_source"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreateObservReviewItems < ActiveRecord::Migration[7.0]
|
|
4
|
+
def change
|
|
5
|
+
create_table :observ_review_items do |t|
|
|
6
|
+
t.string :reviewable_type, null: false
|
|
7
|
+
t.bigint :reviewable_id, null: false
|
|
8
|
+
|
|
9
|
+
t.integer :status, default: 0, null: false
|
|
10
|
+
t.integer :priority, default: 0, null: false
|
|
11
|
+
|
|
12
|
+
t.string :reason
|
|
13
|
+
t.json :reason_details
|
|
14
|
+
|
|
15
|
+
t.datetime :completed_at
|
|
16
|
+
t.string :completed_by
|
|
17
|
+
|
|
18
|
+
t.timestamps
|
|
19
|
+
|
|
20
|
+
t.index [ :reviewable_type, :reviewable_id ], unique: true, name: "idx_review_items_on_reviewable"
|
|
21
|
+
t.index [ :status, :priority, :created_at ], name: "idx_review_items_on_status_priority_created"
|
|
22
|
+
t.index :status
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -14,6 +14,7 @@ module Observ
|
|
|
14
14
|
# rails generate observ:install --skip-index
|
|
15
15
|
# rails generate observ:install --skip-routes # Don't auto-mount engine
|
|
16
16
|
# rails generate observ:install --force # Skip confirmation prompt
|
|
17
|
+
# rails generate observ:install --skip-vite-entrypoint # Don't generate Vite entry point
|
|
17
18
|
class InstallGenerator < Rails::Generators::Base
|
|
18
19
|
source_root File.expand_path("templates", __dir__)
|
|
19
20
|
|
|
@@ -29,8 +30,8 @@ module Observ
|
|
|
29
30
|
|
|
30
31
|
class_option :skip_index,
|
|
31
32
|
type: :boolean,
|
|
32
|
-
desc: "Skip generation of index files",
|
|
33
|
-
default:
|
|
33
|
+
desc: "Skip generation of index files (default: true, index.js no longer needed)",
|
|
34
|
+
default: true
|
|
34
35
|
|
|
35
36
|
class_option :force,
|
|
36
37
|
type: :boolean,
|
|
@@ -42,6 +43,16 @@ module Observ
|
|
|
42
43
|
desc: "Skip automatic route mounting",
|
|
43
44
|
default: false
|
|
44
45
|
|
|
46
|
+
class_option :skip_vite_entrypoint,
|
|
47
|
+
type: :boolean,
|
|
48
|
+
desc: "Skip generation of Vite entry point",
|
|
49
|
+
default: false
|
|
50
|
+
|
|
51
|
+
class_option :vite_entrypoint_dest,
|
|
52
|
+
type: :string,
|
|
53
|
+
desc: "Destination path for Vite entry point (default: app/javascript/entrypoints)",
|
|
54
|
+
default: "app/javascript/entrypoints"
|
|
55
|
+
|
|
45
56
|
def confirm_installation
|
|
46
57
|
return if options[:force]
|
|
47
58
|
|
|
@@ -59,10 +70,12 @@ module Observ
|
|
|
59
70
|
js_files_to_copy = collect_files_to_copy("javascripts", "*.js", js_dest)
|
|
60
71
|
index_will_be_generated = !options[:skip_index] && will_generate_index?(js_dest)
|
|
61
72
|
route_will_be_added = !options[:skip_routes] && !route_already_exists?
|
|
73
|
+
vite_entrypoint_will_be_created = !options[:skip_vite_entrypoint] && will_create_vite_entrypoint?
|
|
62
74
|
|
|
63
75
|
# Check if there are any changes to make
|
|
64
76
|
total_changes = stylesheets_to_copy.count + js_files_to_copy.count +
|
|
65
|
-
(index_will_be_generated ? 1 : 0) + (route_will_be_added ? 1 : 0)
|
|
77
|
+
(index_will_be_generated ? 1 : 0) + (route_will_be_added ? 1 : 0) +
|
|
78
|
+
(vite_entrypoint_will_be_created ? 1 : 0)
|
|
66
79
|
|
|
67
80
|
if total_changes == 0
|
|
68
81
|
say "No changes needed - all files are up to date!", :green
|
|
@@ -92,9 +105,13 @@ module Observ
|
|
|
92
105
|
end
|
|
93
106
|
|
|
94
107
|
# Show generated files
|
|
95
|
-
|
|
108
|
+
generated_files = []
|
|
109
|
+
generated_files << "#{js_dest}/index.js (controller index)" if index_will_be_generated
|
|
110
|
+
generated_files << "#{options[:vite_entrypoint_dest]}/observ.js (Vite entry point)" if vite_entrypoint_will_be_created
|
|
111
|
+
|
|
112
|
+
if generated_files.any?
|
|
96
113
|
say "Generated Files:", :yellow
|
|
97
|
-
say " • #{
|
|
114
|
+
generated_files.each { |file| say " • #{file}", :white }
|
|
98
115
|
say "\n"
|
|
99
116
|
end
|
|
100
117
|
|
|
@@ -145,6 +162,26 @@ module Observ
|
|
|
145
162
|
)
|
|
146
163
|
end
|
|
147
164
|
|
|
165
|
+
def install_vite_entrypoint
|
|
166
|
+
return if options[:skip_vite_entrypoint]
|
|
167
|
+
|
|
168
|
+
dest_dir = Rails.root.join(options[:vite_entrypoint_dest])
|
|
169
|
+
dest_file = dest_dir.join("observ.js")
|
|
170
|
+
|
|
171
|
+
if dest_file.exist?
|
|
172
|
+
say " Vite entry point already exists: #{dest_file}", :yellow
|
|
173
|
+
return
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
say "Creating Vite entry point...", :cyan
|
|
177
|
+
say "-" * 80, :cyan
|
|
178
|
+
|
|
179
|
+
FileUtils.mkdir_p(dest_dir)
|
|
180
|
+
copy_file "observ.js", dest_file
|
|
181
|
+
say " Created #{dest_file}", :green
|
|
182
|
+
say "\n"
|
|
183
|
+
end
|
|
184
|
+
|
|
148
185
|
def show_post_install_message
|
|
149
186
|
say "\n"
|
|
150
187
|
say "=" * 80, :green
|
|
@@ -161,10 +198,18 @@ module Observ
|
|
|
161
198
|
end
|
|
162
199
|
|
|
163
200
|
say "Next steps:", :cyan
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
201
|
+
|
|
202
|
+
if options[:skip_vite_entrypoint]
|
|
203
|
+
say " 1. Import Observ in your application", :cyan
|
|
204
|
+
say " Add to app/javascript/application.js:", :cyan
|
|
205
|
+
say " import 'observ'", :cyan
|
|
206
|
+
say "\n"
|
|
207
|
+
else
|
|
208
|
+
say " 1. Add 'observ' to your Vite entrypoints (if not auto-detected)", :cyan
|
|
209
|
+
say " The entry point was created at: #{options[:vite_entrypoint_dest]}/observ.js", :cyan
|
|
210
|
+
say "\n"
|
|
211
|
+
end
|
|
212
|
+
|
|
168
213
|
say " 2. Restart your development server", :cyan
|
|
169
214
|
say " bin/dev or rails server", :cyan
|
|
170
215
|
say "\n"
|
|
@@ -238,6 +283,13 @@ module Observ
|
|
|
238
283
|
!index_file.exist?
|
|
239
284
|
end
|
|
240
285
|
|
|
286
|
+
# Check if Vite entry point will be created
|
|
287
|
+
# @return [Boolean] true if observ.js will be created
|
|
288
|
+
def will_create_vite_entrypoint?
|
|
289
|
+
dest_file = Rails.root.join(options[:vite_entrypoint_dest], "observ.js")
|
|
290
|
+
!dest_file.exist?
|
|
291
|
+
end
|
|
292
|
+
|
|
241
293
|
# Logger adapter for Rails generator
|
|
242
294
|
class GeneratorLogger
|
|
243
295
|
def initialize(generator)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Observ Vite entry point
|
|
2
|
+
// This file is loaded via vite_javascript_tag 'observ' in the Observ layout
|
|
3
|
+
|
|
4
|
+
// Import Turbo and Stimulus
|
|
5
|
+
import '@hotwired/turbo-rails'
|
|
6
|
+
import { Application } from '@hotwired/stimulus'
|
|
7
|
+
import { registerControllers } from 'stimulus-vite-helpers'
|
|
8
|
+
|
|
9
|
+
// Import Observ stylesheets
|
|
10
|
+
import '../stylesheets/observ/application.scss'
|
|
11
|
+
|
|
12
|
+
// Start Stimulus application
|
|
13
|
+
const application = Application.start()
|
|
14
|
+
application.debug = false
|
|
15
|
+
window.Stimulus = application
|
|
16
|
+
|
|
17
|
+
// Auto-register all Observ Stimulus controllers
|
|
18
|
+
const controllers = import.meta.glob('../controllers/observ/*_controller.js', { eager: true })
|
|
19
|
+
registerControllers(application, controllers)
|
data/lib/observ/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rubyllm-observ
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Franck D'agostini
|
|
@@ -230,7 +230,6 @@ files:
|
|
|
230
230
|
- app/assets/javascripts/observ/controllers/drawer_controller.js
|
|
231
231
|
- app/assets/javascripts/observ/controllers/expandable_controller.js
|
|
232
232
|
- app/assets/javascripts/observ/controllers/filter_controller.js
|
|
233
|
-
- app/assets/javascripts/observ/controllers/index.js
|
|
234
233
|
- app/assets/javascripts/observ/controllers/json_viewer_controller.js
|
|
235
234
|
- app/assets/javascripts/observ/controllers/message_form_controller.js
|
|
236
235
|
- app/assets/javascripts/observ/controllers/prompt_variables_controller.js
|
|
@@ -242,12 +241,15 @@ files:
|
|
|
242
241
|
- app/assets/stylesheets/observ/_dashboard.scss
|
|
243
242
|
- app/assets/stylesheets/observ/_datasets.scss
|
|
244
243
|
- app/assets/stylesheets/observ/_drawer.scss
|
|
244
|
+
- app/assets/stylesheets/observ/_filters.scss
|
|
245
245
|
- app/assets/stylesheets/observ/_json_viewer.scss
|
|
246
246
|
- app/assets/stylesheets/observ/_layout.scss
|
|
247
247
|
- app/assets/stylesheets/observ/_metrics.scss
|
|
248
|
+
- app/assets/stylesheets/observ/_namespace.scss
|
|
248
249
|
- app/assets/stylesheets/observ/_observations.scss
|
|
249
250
|
- app/assets/stylesheets/observ/_pagination.scss
|
|
250
251
|
- app/assets/stylesheets/observ/_prompts.scss
|
|
252
|
+
- app/assets/stylesheets/observ/_reset.scss
|
|
251
253
|
- app/assets/stylesheets/observ/_table.scss
|
|
252
254
|
- app/assets/stylesheets/observ/_variables.scss
|
|
253
255
|
- app/assets/stylesheets/observ/application.scss
|
|
@@ -263,6 +265,8 @@ files:
|
|
|
263
265
|
- app/controllers/observ/observations_controller.rb
|
|
264
266
|
- app/controllers/observ/prompt_versions_controller.rb
|
|
265
267
|
- app/controllers/observ/prompts_controller.rb
|
|
268
|
+
- app/controllers/observ/review_queue_controller.rb
|
|
269
|
+
- app/controllers/observ/scores_controller.rb
|
|
266
270
|
- app/controllers/observ/sessions_controller.rb
|
|
267
271
|
- app/controllers/observ/traces_controller.rb
|
|
268
272
|
- app/forms/observ/prompt_form.rb
|
|
@@ -271,15 +275,19 @@ files:
|
|
|
271
275
|
- app/helpers/observ/dashboard_helper.rb
|
|
272
276
|
- app/helpers/observ/datasets_helper.rb
|
|
273
277
|
- app/helpers/observ/pagination_helper.rb
|
|
278
|
+
- app/helpers/observ/reviews_helper.rb
|
|
274
279
|
- app/jobs/observ/application_job.rb
|
|
275
280
|
- app/jobs/observ/dataset_runner_job.rb
|
|
276
281
|
- app/mailers/observ/application_mailer.rb
|
|
277
282
|
- app/models/concerns/observ/agent_phaseable.rb
|
|
278
283
|
- app/models/concerns/observ/agent_selectable.rb
|
|
279
284
|
- app/models/concerns/observ/chat_enhancements.rb
|
|
285
|
+
- app/models/concerns/observ/json_queryable.rb
|
|
280
286
|
- app/models/concerns/observ/message_enhancements.rb
|
|
281
287
|
- app/models/concerns/observ/observability_instrumentation.rb
|
|
282
288
|
- app/models/concerns/observ/prompt_management.rb
|
|
289
|
+
- app/models/concerns/observ/reviewable.rb
|
|
290
|
+
- app/models/concerns/observ/scoreable.rb
|
|
283
291
|
- app/models/concerns/observ/trace_association.rb
|
|
284
292
|
- app/models/observ/annotation.rb
|
|
285
293
|
- app/models/observ/application_record.rb
|
|
@@ -291,6 +299,7 @@ files:
|
|
|
291
299
|
- app/models/observ/null_prompt.rb
|
|
292
300
|
- app/models/observ/observation.rb
|
|
293
301
|
- app/models/observ/prompt.rb
|
|
302
|
+
- app/models/observ/review_item.rb
|
|
294
303
|
- app/models/observ/score.rb
|
|
295
304
|
- app/models/observ/session.rb
|
|
296
305
|
- app/models/observ/span.rb
|
|
@@ -306,6 +315,7 @@ files:
|
|
|
306
315
|
- app/services/observ/evaluators/contains_evaluator.rb
|
|
307
316
|
- app/services/observ/evaluators/exact_match_evaluator.rb
|
|
308
317
|
- app/services/observ/evaluators/json_structure_evaluator.rb
|
|
318
|
+
- app/services/observ/guardrail_service.rb
|
|
309
319
|
- app/services/observ/prompt_manager.rb
|
|
310
320
|
- app/services/observ/prompt_manager/cache_statistics.rb
|
|
311
321
|
- app/services/observ/prompt_manager/caching.rb
|
|
@@ -379,10 +389,26 @@ files:
|
|
|
379
389
|
- app/views/observ/prompts/new.html.erb
|
|
380
390
|
- app/views/observ/prompts/show.html.erb
|
|
381
391
|
- app/views/observ/prompts/versions.html.erb
|
|
392
|
+
- app/views/observ/review_queue/_item.html.erb
|
|
393
|
+
- app/views/observ/review_queue/_stats.html.erb
|
|
394
|
+
- app/views/observ/review_queue/index.html.erb
|
|
395
|
+
- app/views/observ/review_queue/show.html.erb
|
|
396
|
+
- app/views/observ/review_queue/stats.html.erb
|
|
397
|
+
- app/views/observ/scores/_form.html.erb
|
|
398
|
+
- app/views/observ/scores/create.turbo_stream.erb
|
|
399
|
+
- app/views/observ/sessions/_chat.html.erb
|
|
400
|
+
- app/views/observ/sessions/_metadata.html.erb
|
|
401
|
+
- app/views/observ/sessions/_metrics.html.erb
|
|
402
|
+
- app/views/observ/sessions/_traces.html.erb
|
|
382
403
|
- app/views/observ/sessions/annotations_drawer.turbo_stream.erb
|
|
383
404
|
- app/views/observ/sessions/drawer_test.turbo_stream.erb
|
|
384
405
|
- app/views/observ/sessions/index.html.erb
|
|
385
406
|
- app/views/observ/sessions/show.html.erb
|
|
407
|
+
- app/views/observ/traces/_details.html.erb
|
|
408
|
+
- app/views/observ/traces/_input.html.erb
|
|
409
|
+
- app/views/observ/traces/_metadata.html.erb
|
|
410
|
+
- app/views/observ/traces/_observations.html.erb
|
|
411
|
+
- app/views/observ/traces/_output.html.erb
|
|
386
412
|
- app/views/observ/traces/add_to_dataset_drawer.turbo_stream.erb
|
|
387
413
|
- app/views/observ/traces/annotations_drawer.turbo_stream.erb
|
|
388
414
|
- app/views/observ/traces/index.html.erb
|
|
@@ -403,10 +429,13 @@ files:
|
|
|
403
429
|
- db/migrate/012_create_observ_dataset_runs.rb
|
|
404
430
|
- db/migrate/013_create_observ_dataset_run_items.rb
|
|
405
431
|
- db/migrate/014_create_observ_scores.rb
|
|
432
|
+
- db/migrate/015_refactor_scores_to_polymorphic.rb
|
|
433
|
+
- db/migrate/016_create_observ_review_items.rb
|
|
406
434
|
- lib/generators/observ/add_phase_tracking/add_phase_tracking_generator.rb
|
|
407
435
|
- lib/generators/observ/add_phase_tracking/templates/migration.rb.tt
|
|
408
436
|
- lib/generators/observ/install/USAGE
|
|
409
437
|
- lib/generators/observ/install/install_generator.rb
|
|
438
|
+
- lib/generators/observ/install/templates/observ.js
|
|
410
439
|
- lib/generators/observ/install_chat/install_chat_generator.rb
|
|
411
440
|
- lib/generators/observ/install_chat/templates/agents/base_agent.rb.tt
|
|
412
441
|
- lib/generators/observ/install_chat/templates/agents/simple_agent.rb.tt
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
// Auto-generated index file for Observ Stimulus controllers
|
|
2
|
-
// Register all Observ controllers with the observ-- prefix
|
|
3
|
-
//
|
|
4
|
-
// This file is designed to be imported by the host application's Stimulus setup.
|
|
5
|
-
// The host app should import this file in their controllers/index.js:
|
|
6
|
-
// import "./observ"
|
|
7
|
-
|
|
8
|
-
import AutoscrollController from "./autoscroll_controller.js"
|
|
9
|
-
import ChatFormController from "./chat_form_controller.js"
|
|
10
|
-
import CopyController from "./copy_controller.js"
|
|
11
|
-
import DashboardController from "./dashboard_controller.js"
|
|
12
|
-
import DrawerController from "./drawer_controller.js"
|
|
13
|
-
import ExpandableController from "./expandable_controller.js"
|
|
14
|
-
import FilterController from "./filter_controller.js"
|
|
15
|
-
import JsonViewerController from "./json_viewer_controller.js"
|
|
16
|
-
import MessageFormController from "./message_form_controller.js"
|
|
17
|
-
import PromptVariablesController from "./prompt_variables_controller.js"
|
|
18
|
-
import TextSelectController from "./text_select_controller.js"
|
|
19
|
-
|
|
20
|
-
// Export controllers for manual registration if needed
|
|
21
|
-
export {
|
|
22
|
-
AutoscrollController,
|
|
23
|
-
ChatFormController,
|
|
24
|
-
CopyController,
|
|
25
|
-
DashboardController,
|
|
26
|
-
DrawerController,
|
|
27
|
-
ExpandableController,
|
|
28
|
-
FilterController,
|
|
29
|
-
JsonViewerController,
|
|
30
|
-
MessageFormController,
|
|
31
|
-
PromptVariablesController,
|
|
32
|
-
TextSelectController
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
// Auto-register if Stimulus application is available globally
|
|
36
|
-
if (typeof window.Stimulus !== "undefined") {
|
|
37
|
-
const application = window.Stimulus
|
|
38
|
-
|
|
39
|
-
application.register("observ--autoscroll", AutoscrollController)
|
|
40
|
-
application.register("observ--chat-form", ChatFormController)
|
|
41
|
-
application.register("observ--copy", CopyController)
|
|
42
|
-
application.register("observ--dashboard", DashboardController)
|
|
43
|
-
application.register("observ--drawer", DrawerController)
|
|
44
|
-
application.register("observ--expandable", ExpandableController)
|
|
45
|
-
application.register("observ--filter", FilterController)
|
|
46
|
-
application.register("observ--json-viewer", JsonViewerController)
|
|
47
|
-
application.register("observ--message-form", MessageFormController)
|
|
48
|
-
application.register("observ--prompt-variables", PromptVariablesController)
|
|
49
|
-
application.register("observ--text-select", TextSelectController)
|
|
50
|
-
|
|
51
|
-
console.log("Observ Stimulus controllers registered")
|
|
52
|
-
}
|