leva 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -1
- data/app/views/leva/dataset_optimizations/new.html.erb +145 -0
- data/app/views/leva/datasets/show.html.erb +64 -2
- data/config/routes.rb +2 -0
- data/lib/leva/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e25afa90f1e4ad86dbc306bc26aea4f44c157bd18b1a666740be9ea8709e395c
|
|
4
|
+
data.tar.gz: abbcde1d2853b07a825aa5558f8f50e2c6ae92eeda3290e86af78cbf328ab2bc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c0a250ede2a3eb3868f9b3c26bc75e61d20b05dd5e28d8f71ebff40f39a027e01040b8f58a8c372fd0bbf3226e62defa158f8e08c092b6b21653638f4843518a
|
|
7
|
+
data.tar.gz: 48a2bc4b8a76606d85aabdaeb3fa59d50006d1c6ba4eedd68373839d2d24115929b82a6e657a66f4527ce8559949eb9790c39ec362d5cfa4d86edb56651f6e7c
|
data/README.md
CHANGED
|
@@ -195,10 +195,13 @@ Add the DSPy gems to your Gemfile:
|
|
|
195
195
|
|
|
196
196
|
```ruby
|
|
197
197
|
gem "dspy" # Core DSPy functionality (required)
|
|
198
|
+
gem "dspy-ruby_llm" # RubyLLM provider adapter (required)
|
|
198
199
|
gem "dspy-gepa" # GEPA optimizer (optional, recommended)
|
|
199
200
|
gem "dspy-miprov2" # MIPROv2 optimizer (optional)
|
|
200
201
|
```
|
|
201
202
|
|
|
203
|
+
You can use any DSPy provider adapter instead of `dspy-ruby_llm`, such as `dspy-openai` or `dspy-anthropic`.
|
|
204
|
+
|
|
202
205
|
### Available Optimizers
|
|
203
206
|
|
|
204
207
|
| Optimizer | Best For | Description |
|
|
@@ -215,7 +218,7 @@ optimizer = Leva::PromptOptimizer.new(
|
|
|
215
218
|
dataset: dataset,
|
|
216
219
|
optimizer: :gepa, # :bootstrap, :gepa, or :miprov2
|
|
217
220
|
mode: :medium, # :light, :medium, or :heavy
|
|
218
|
-
model: "
|
|
221
|
+
model: "claude-opus-4-5" # Any model supported by RubyLLM
|
|
219
222
|
)
|
|
220
223
|
|
|
221
224
|
# Run optimization
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
<% content_for :title, 'Optimize Prompt' %>
|
|
2
|
+
<div class="container page">
|
|
3
|
+
<nav class="breadcrumb mb-4">
|
|
4
|
+
<%= link_to "Datasets", datasets_path, class: "breadcrumb-link" %>
|
|
5
|
+
<span class="breadcrumb-sep">/</span>
|
|
6
|
+
<%= link_to @dataset.name, dataset_path(@dataset), class: "breadcrumb-link" %>
|
|
7
|
+
<span class="breadcrumb-sep">/</span>
|
|
8
|
+
<span class="breadcrumb-current">Optimize</span>
|
|
9
|
+
</nav>
|
|
10
|
+
|
|
11
|
+
<div class="page-header mb-6">
|
|
12
|
+
<div>
|
|
13
|
+
<h1>Optimize Prompt</h1>
|
|
14
|
+
<p class="text-muted text-sm mt-1">Use DSPy to automatically optimize your prompt with few-shot examples</p>
|
|
15
|
+
</div>
|
|
16
|
+
</div>
|
|
17
|
+
|
|
18
|
+
<% if @can_optimize %>
|
|
19
|
+
<%= form_with url: dataset_optimization_path(@dataset), method: :post, local: true, class: "card p-6" do |form| %>
|
|
20
|
+
<%# Basic Information %>
|
|
21
|
+
<div class="form-section">
|
|
22
|
+
<div class="form-group">
|
|
23
|
+
<%= form.label :prompt_name, "Prompt Name", class: "form-label" %>
|
|
24
|
+
<%= form.text_field :prompt_name, value: "Optimized: #{@dataset.name}", autofocus: true, class: "form-input", placeholder: "e.g., Optimized Sentiment Classifier" %>
|
|
25
|
+
<p class="form-hint">Name for the new optimized prompt that will be created.</p>
|
|
26
|
+
</div>
|
|
27
|
+
</div>
|
|
28
|
+
|
|
29
|
+
<hr class="form-divider">
|
|
30
|
+
|
|
31
|
+
<%# Optimization Settings %>
|
|
32
|
+
<div class="form-section">
|
|
33
|
+
<h4 class="form-section-title">Optimization Settings</h4>
|
|
34
|
+
|
|
35
|
+
<div class="form-row">
|
|
36
|
+
<div class="form-group flex-1">
|
|
37
|
+
<%= form.label :optimizer, "Optimizer", class: "form-label" %>
|
|
38
|
+
<%= form.select :optimizer,
|
|
39
|
+
@optimizers.map { |k, v| [v[:name], k] },
|
|
40
|
+
{},
|
|
41
|
+
class: "form-select" %>
|
|
42
|
+
<p class="form-hint">Algorithm used to optimize the prompt.</p>
|
|
43
|
+
</div>
|
|
44
|
+
|
|
45
|
+
<div class="form-group flex-1">
|
|
46
|
+
<%= form.label :mode, "Mode", class: "form-label" %>
|
|
47
|
+
<%= form.select :mode,
|
|
48
|
+
@modes.map { |k, v| ["#{v[:name]} (#{v[:description]})", k] },
|
|
49
|
+
{},
|
|
50
|
+
class: "form-select" %>
|
|
51
|
+
<p class="form-hint">Higher modes use more examples but take longer.</p>
|
|
52
|
+
</div>
|
|
53
|
+
</div>
|
|
54
|
+
|
|
55
|
+
<div class="form-group">
|
|
56
|
+
<%= form.label :model, "Model", class: "form-label" %>
|
|
57
|
+
<%= form.select :model,
|
|
58
|
+
@models.map { |m| [m.name, m.id] },
|
|
59
|
+
{},
|
|
60
|
+
class: "form-select" %>
|
|
61
|
+
<p class="form-hint">The AI model to use during optimization.</p>
|
|
62
|
+
</div>
|
|
63
|
+
</div>
|
|
64
|
+
|
|
65
|
+
<hr class="form-divider">
|
|
66
|
+
|
|
67
|
+
<%# Dataset Info %>
|
|
68
|
+
<div class="form-section">
|
|
69
|
+
<h4 class="form-section-title">Dataset Information</h4>
|
|
70
|
+
<div class="info-grid">
|
|
71
|
+
<div class="info-item">
|
|
72
|
+
<span class="info-label">Records Available</span>
|
|
73
|
+
<span class="info-value"><%= @record_count %></span>
|
|
74
|
+
</div>
|
|
75
|
+
<div class="info-item">
|
|
76
|
+
<span class="info-label">Minimum Required</span>
|
|
77
|
+
<span class="info-value">10</span>
|
|
78
|
+
</div>
|
|
79
|
+
</div>
|
|
80
|
+
</div>
|
|
81
|
+
|
|
82
|
+
<div class="form-actions">
|
|
83
|
+
<%= link_to "Cancel", dataset_path(@dataset), class: "btn btn-ghost" %>
|
|
84
|
+
<%= form.submit "Start Optimization", class: "btn btn-primary" %>
|
|
85
|
+
</div>
|
|
86
|
+
<% end %>
|
|
87
|
+
<% else %>
|
|
88
|
+
<div class="card p-6">
|
|
89
|
+
<div class="setup-required">
|
|
90
|
+
<div class="setup-required-icon">
|
|
91
|
+
<svg class="icon-xl" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
|
92
|
+
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-2.5L13.732 4c-.77-.833-1.964-.833-2.732 0L4.082 16.5c-.77.833.192 2.5 1.732 2.5z" />
|
|
93
|
+
</svg>
|
|
94
|
+
</div>
|
|
95
|
+
<h3 class="setup-required-title">More Records Needed</h3>
|
|
96
|
+
<p class="setup-required-desc">You need at least <strong>10 records</strong> to optimize a prompt. Currently you have <strong><%= @record_count %></strong>.</p>
|
|
97
|
+
<p class="setup-required-hint mt-2">Add <strong><%= @records_needed %></strong> more records to enable optimization.</p>
|
|
98
|
+
</div>
|
|
99
|
+
|
|
100
|
+
<div class="form-actions">
|
|
101
|
+
<%= link_to dataset_path(@dataset), class: "btn btn-ghost" do %>
|
|
102
|
+
<svg class="icon-sm" viewBox="0 0 20 20" fill="currentColor">
|
|
103
|
+
<path fill-rule="evenodd" d="M9.707 16.707a1 1 0 01-1.414 0l-6-6a1 1 0 010-1.414l6-6a1 1 0 011.414 1.414L5.414 9H17a1 1 0 110 2H5.414l4.293 4.293a1 1 0 010 1.414z" clip-rule="evenodd" />
|
|
104
|
+
</svg>
|
|
105
|
+
Back to Dataset
|
|
106
|
+
<% end %>
|
|
107
|
+
</div>
|
|
108
|
+
</div>
|
|
109
|
+
<% end %>
|
|
110
|
+
</div>
|
|
111
|
+
|
|
112
|
+
<style>
|
|
113
|
+
.info-grid {
|
|
114
|
+
display: grid;
|
|
115
|
+
grid-template-columns: repeat(2, 1fr);
|
|
116
|
+
gap: 1rem;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
.info-item {
|
|
120
|
+
background: var(--bg-secondary);
|
|
121
|
+
border-radius: 0.5rem;
|
|
122
|
+
padding: 1rem;
|
|
123
|
+
text-align: center;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
.info-label {
|
|
127
|
+
display: block;
|
|
128
|
+
font-size: 0.75rem;
|
|
129
|
+
color: var(--text-muted);
|
|
130
|
+
text-transform: uppercase;
|
|
131
|
+
letter-spacing: 0.04em;
|
|
132
|
+
margin-bottom: 0.25rem;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
.info-value {
|
|
136
|
+
font-size: 1.5rem;
|
|
137
|
+
font-weight: 600;
|
|
138
|
+
font-family: 'Fira Code', monospace;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
.setup-required-hint {
|
|
142
|
+
color: var(--text-muted);
|
|
143
|
+
font-size: 0.875rem;
|
|
144
|
+
}
|
|
145
|
+
</style>
|
|
@@ -84,8 +84,69 @@
|
|
|
84
84
|
<% end %>
|
|
85
85
|
</section>
|
|
86
86
|
|
|
87
|
-
<%#
|
|
88
|
-
|
|
87
|
+
<%# Prompt Optimization Section %>
|
|
88
|
+
<section class="mb-8">
|
|
89
|
+
<div class="section-header">
|
|
90
|
+
<h3 class="section-title">Prompt Optimization</h3>
|
|
91
|
+
<span class="section-count"><%= @dataset.optimization_runs.count %></span>
|
|
92
|
+
<div class="ml-auto">
|
|
93
|
+
<% optimizer = Leva::PromptOptimizer.new(dataset: @dataset) %>
|
|
94
|
+
<% if optimizer.can_optimize? %>
|
|
95
|
+
<%= link_to new_dataset_optimization_path(@dataset), class: "btn btn-primary btn-sm" do %>
|
|
96
|
+
<svg class="icon-sm" viewBox="0 0 20 20" fill="currentColor">
|
|
97
|
+
<path fill-rule="evenodd" d="M11.3 1.046A1 1 0 0112 2v5h4a1 1 0 01.82 1.573l-7 10A1 1 0 018 18v-5H4a1 1 0 01-.82-1.573l7-10a1 1 0 011.12-.38z" clip-rule="evenodd" />
|
|
98
|
+
</svg>
|
|
99
|
+
Optimize Prompt
|
|
100
|
+
<% end %>
|
|
101
|
+
<% else %>
|
|
102
|
+
<button class="btn btn-ghost btn-sm" disabled title="Need <%= optimizer.records_needed %> more records">
|
|
103
|
+
<svg class="icon-sm" viewBox="0 0 20 20" fill="currentColor">
|
|
104
|
+
<path fill-rule="evenodd" d="M11.3 1.046A1 1 0 0112 2v5h4a1 1 0 01.82 1.573l-7 10A1 1 0 018 18v-5H4a1 1 0 01-.82-1.573l7-10a1 1 0 011.12-.38z" clip-rule="evenodd" />
|
|
105
|
+
</svg>
|
|
106
|
+
Need <%= optimizer.records_needed %> more records
|
|
107
|
+
</button>
|
|
108
|
+
<% end %>
|
|
109
|
+
</div>
|
|
110
|
+
</div>
|
|
111
|
+
|
|
112
|
+
<% if @dataset.optimization_runs.any? %>
|
|
113
|
+
<div class="table-wrapper">
|
|
114
|
+
<div class="table-scroll">
|
|
115
|
+
<table class="table table-clickable">
|
|
116
|
+
<thead>
|
|
117
|
+
<tr>
|
|
118
|
+
<th>Prompt Name</th>
|
|
119
|
+
<th>Optimizer</th>
|
|
120
|
+
<th>Mode</th>
|
|
121
|
+
<th>Status</th>
|
|
122
|
+
<th class="text-right">Created</th>
|
|
123
|
+
</tr>
|
|
124
|
+
</thead>
|
|
125
|
+
<tbody>
|
|
126
|
+
<% @dataset.optimization_runs.order(created_at: :desc).each do |run| %>
|
|
127
|
+
<tr class="clickable-row" onclick="window.location='<%= optimization_run_path(run) %>'">
|
|
128
|
+
<td><span class="row-title"><%= run.prompt_name %></span></td>
|
|
129
|
+
<td><%= run.optimizer&.titleize || 'Bootstrap' %></td>
|
|
130
|
+
<td><%= run.mode&.titleize || 'Light' %></td>
|
|
131
|
+
<td>
|
|
132
|
+
<span class="badge badge-<%= run.status == 'completed' ? 'success' : (run.status == 'failed' ? 'error' : 'warning') %>">
|
|
133
|
+
<%= run.status&.titleize || 'Pending' %>
|
|
134
|
+
</span>
|
|
135
|
+
</td>
|
|
136
|
+
<td class="text-right text-muted"><%= time_ago_in_words(run.created_at) %> ago</td>
|
|
137
|
+
</tr>
|
|
138
|
+
<% end %>
|
|
139
|
+
</tbody>
|
|
140
|
+
</table>
|
|
141
|
+
</div>
|
|
142
|
+
</div>
|
|
143
|
+
<% else %>
|
|
144
|
+
<div class="empty-state-inline">
|
|
145
|
+
<p class="text-muted text-sm">No optimization runs yet.</p>
|
|
146
|
+
<p class="text-xs text-subtle mt-2">Use DSPy to optimize your prompts with few-shot examples.</p>
|
|
147
|
+
</div>
|
|
148
|
+
<% end %>
|
|
149
|
+
</section>
|
|
89
150
|
|
|
90
151
|
<%# Experiments Section %>
|
|
91
152
|
<section>
|
|
@@ -126,6 +187,7 @@
|
|
|
126
187
|
</tr>
|
|
127
188
|
</thead>
|
|
128
189
|
<tbody>
|
|
190
|
+
<% @evaluator_classes = Leva::EvaluationResult.distinct.pluck(:evaluator_class) %>
|
|
129
191
|
<%= render partial: 'leva/experiments/experiment', collection: @dataset.experiments %>
|
|
130
192
|
</tbody>
|
|
131
193
|
</table>
|
data/config/routes.rb
CHANGED
|
@@ -5,7 +5,9 @@ Leva::Engine.routes.draw do
|
|
|
5
5
|
|
|
6
6
|
resources :datasets do
|
|
7
7
|
resources :dataset_records, path: "records", only: [ :index, :show ]
|
|
8
|
+
resource :optimization, only: [ :new, :create ], controller: "dataset_optimizations"
|
|
8
9
|
end
|
|
10
|
+
resources :optimization_runs, only: [ :show ]
|
|
9
11
|
resources :experiments, except: [ :destroy ] do
|
|
10
12
|
member do
|
|
11
13
|
post :rerun
|
data/lib/leva/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: leva
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kieran Klaassen
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-12-
|
|
11
|
+
date: 2025-12-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rails
|
|
@@ -85,6 +85,7 @@ files:
|
|
|
85
85
|
- app/services/leva/prompt_optimizer.rb
|
|
86
86
|
- app/services/leva/signature_generator.rb
|
|
87
87
|
- app/views/layouts/leva/application.html.erb
|
|
88
|
+
- app/views/leva/dataset_optimizations/new.html.erb
|
|
88
89
|
- app/views/leva/dataset_records/index.html.erb
|
|
89
90
|
- app/views/leva/dataset_records/show.html.erb
|
|
90
91
|
- app/views/leva/datasets/_dataset.html.erb
|