llm_logs 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +66 -3
- data/app/controllers/llm_logs/prompts_controller.rb +11 -2
- data/app/helpers/llm_logs/formatting_helper.rb +20 -0
- data/app/models/llm_logs/prompt.rb +7 -0
- data/app/services/llm_logs/prompt_syncer.rb +116 -0
- data/app/views/layouts/llm_logs/application.html.erb +93 -0
- data/app/views/llm_logs/prompt_versions/show.html.erb +1 -1
- data/app/views/llm_logs/prompts/_form.html.erb +9 -0
- data/app/views/llm_logs/prompts/index.html.erb +16 -0
- data/app/views/llm_logs/prompts/show.html.erb +18 -8
- data/db/migrate/006_add_tags_to_prompts.rb +6 -0
- data/lib/generators/llm_logs/templates/initializer.rb +6 -0
- data/lib/llm_logs/configuration.rb +17 -0
- data/lib/llm_logs/engine.rb +4 -0
- data/lib/llm_logs/version.rb +1 -1
- data/lib/llm_logs.rb +26 -5
- data/lib/tasks/llm_logs.rake +14 -0
- metadata +33 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6234bcff772d83a80d691f6cf61e5238f50038feb031a00b7a013c0557fea3b0
|
|
4
|
+
data.tar.gz: 6b71fd68445d3817fe004a2ebae1ad5644ddc42f51ef0d224e9cde31aba3cc23
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b696c698bcbeaa6da42e3e45a2f02bc4368fcbb7fa8fef3a49b6844cef1b949ebaa0c4f7a43ad6c4c4d4f178f8d93f4cd73af2e207b6941726afac10e8b152ac
|
|
7
|
+
data.tar.gz: eca247f5563982227a6f5a2b664ec4b68ff96fe93110ea9dde0974ecedd78cdbbb3a480542e90fc87aa6445f43ceec35b160e76242f2439f27a0eaead90a2197
|
data/README.md
CHANGED
|
@@ -29,6 +29,7 @@ mount LlmLogs::Engine, at: "/llm_logs"
|
|
|
29
29
|
LlmLogs.setup do |config|
|
|
30
30
|
config.enabled = true
|
|
31
31
|
config.auto_instrument = true
|
|
32
|
+
config.prompts_source_path = Rails.root.join("db/data/prompts")
|
|
32
33
|
end
|
|
33
34
|
```
|
|
34
35
|
|
|
@@ -119,6 +120,66 @@ prompt.version(1) # specific version
|
|
|
119
120
|
prompt.rollback_to!(1) # creates new version from v1 content
|
|
120
121
|
```
|
|
121
122
|
|
|
123
|
+
### Sync Prompts From Files
|
|
124
|
+
|
|
125
|
+
Store prompts as Markdown files and sync them into `LlmLogs::Prompt` records with the rake task.
|
|
126
|
+
|
|
127
|
+
```ruby
|
|
128
|
+
# config/initializers/llm_logs.rb
|
|
129
|
+
LlmLogs.setup do |config|
|
|
130
|
+
config.prompts_source_path = Rails.root.join("db/data/prompts")
|
|
131
|
+
config.prompt_subfolders = %w[skills fragments templates]
|
|
132
|
+
end
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
```sh
|
|
136
|
+
bin/rails llm_logs:prompts:sync
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The syncer reads `*.md` files from each configured subfolder. The subfolder name is added as a prompt tag automatically.
|
|
140
|
+
|
|
141
|
+
```text
|
|
142
|
+
db/data/prompts/
|
|
143
|
+
skills/
|
|
144
|
+
backtest-evaluation.md
|
|
145
|
+
fragments/
|
|
146
|
+
provider-notes.md
|
|
147
|
+
templates/
|
|
148
|
+
trading-memo.md
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Single-message prompts use the Markdown body as the system message:
|
|
152
|
+
|
|
153
|
+
```markdown
|
|
154
|
+
---
|
|
155
|
+
slug: backtest-evaluation
|
|
156
|
+
name: Backtest Evaluation
|
|
157
|
+
description: How to evaluate backtests
|
|
158
|
+
tags: [evaluation]
|
|
159
|
+
model: anthropic/claude-sonnet-4
|
|
160
|
+
model_params:
|
|
161
|
+
temperature: 0.3
|
|
162
|
+
---
|
|
163
|
+
Body content here.
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Multi-message prompts can reference sibling body files:
|
|
167
|
+
|
|
168
|
+
```markdown
|
|
169
|
+
---
|
|
170
|
+
slug: trading-memo
|
|
171
|
+
name: Trading Memo
|
|
172
|
+
model: deepseek/deepseek-v3.2
|
|
173
|
+
messages:
|
|
174
|
+
- role: system
|
|
175
|
+
body_file: trading_memo_system.md
|
|
176
|
+
- role: user
|
|
177
|
+
body_file: trading_memo_user.md
|
|
178
|
+
---
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Running the task creates missing prompts, updates metadata, and creates a new prompt version only when messages, model, or model parameters changed.
|
|
182
|
+
|
|
122
183
|
## Web UI
|
|
123
184
|
|
|
124
185
|
Browse traces and manage prompts at `/llm_logs`.
|
|
@@ -131,9 +192,11 @@ Browse traces and manage prompts at `/llm_logs`.
|
|
|
131
192
|
|
|
132
193
|
```ruby
|
|
133
194
|
LlmLogs.setup do |config|
|
|
134
|
-
config.enabled = true
|
|
135
|
-
config.auto_instrument = true
|
|
136
|
-
config.retention_days = 30
|
|
195
|
+
config.enabled = true # master switch for logging
|
|
196
|
+
config.auto_instrument = true # auto-prepend on RubyLLM::Chat
|
|
197
|
+
config.retention_days = 30 # for future cleanup job
|
|
198
|
+
config.prompts_source_path = Rails.root.join("db/data/prompts")
|
|
199
|
+
config.prompt_subfolders = %w[skills fragments templates]
|
|
137
200
|
end
|
|
138
201
|
```
|
|
139
202
|
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
module LlmLogs
|
|
2
2
|
class PromptsController < ApplicationController
|
|
3
3
|
def index
|
|
4
|
-
|
|
4
|
+
tag = params[:tag].is_a?(String) ? params[:tag].presence : nil
|
|
5
|
+
scope = Prompt.order(:name).includes(:versions)
|
|
6
|
+
scope = scope.with_tag(tag) if tag
|
|
7
|
+
@prompts = scope.page(params[:page]).per(25)
|
|
8
|
+
@active_tag = tag
|
|
9
|
+
@all_tags = Prompt.pluck(:tags).flatten.compact.uniq.sort
|
|
5
10
|
end
|
|
6
11
|
|
|
7
12
|
def show
|
|
@@ -54,7 +59,11 @@ module LlmLogs
|
|
|
54
59
|
private
|
|
55
60
|
|
|
56
61
|
def prompt_params
|
|
57
|
-
params.require(:prompt).permit(:slug, :name, :description)
|
|
62
|
+
raw = params.require(:prompt).permit(:slug, :name, :description, :tags_input, tags: [])
|
|
63
|
+
if raw[:tags_input].present?
|
|
64
|
+
raw[:tags] = raw[:tags_input].split(",").map(&:strip).reject(&:blank?)
|
|
65
|
+
end
|
|
66
|
+
raw.except(:tags_input)
|
|
58
67
|
end
|
|
59
68
|
|
|
60
69
|
def version_params
|
|
@@ -1,11 +1,31 @@
|
|
|
1
|
+
require "kramdown"
|
|
2
|
+
require "kramdown-parser-gfm"
|
|
3
|
+
|
|
1
4
|
module LlmLogs
|
|
2
5
|
module FormattingHelper
|
|
6
|
+
MARKDOWN_TAGS = %w[
|
|
7
|
+
a blockquote br code del em h1 h2 h3 h4 h5 h6 hr li ol p pre strong
|
|
8
|
+
table tbody td th thead tr ul
|
|
9
|
+
].freeze
|
|
10
|
+
MARKDOWN_ATTRIBUTES = %w[href title].freeze
|
|
11
|
+
|
|
3
12
|
def format_duration_ms(ms)
|
|
4
13
|
return "—" if ms.nil?
|
|
5
14
|
|
|
6
15
|
"#{number_with_delimiter(sprintf('%.0f', ms))} ms"
|
|
7
16
|
end
|
|
8
17
|
|
|
18
|
+
def render_markdown(text)
|
|
19
|
+
html = Kramdown::Document.new(
|
|
20
|
+
text.to_s,
|
|
21
|
+
input: "GFM",
|
|
22
|
+
hard_wrap: true,
|
|
23
|
+
syntax_highlighter: nil
|
|
24
|
+
).to_html
|
|
25
|
+
|
|
26
|
+
sanitize html, tags: MARKDOWN_TAGS, attributes: MARKDOWN_ATTRIBUTES
|
|
27
|
+
end
|
|
28
|
+
|
|
9
29
|
def pretty_json(data)
|
|
10
30
|
JSON.pretty_generate(deep_parse_json(data))
|
|
11
31
|
rescue
|
|
@@ -5,6 +5,13 @@ module LlmLogs
|
|
|
5
5
|
validates :slug, presence: true, uniqueness: true
|
|
6
6
|
validates :name, presence: true
|
|
7
7
|
|
|
8
|
+
scope :with_tag, ->(tag) { where("? = ANY(tags)", tag) }
|
|
9
|
+
scope :with_any_tag, ->(tags) { where("tags && ARRAY[?]::varchar[]", Array(tags)) }
|
|
10
|
+
|
|
11
|
+
def tags_string
|
|
12
|
+
Array(tags).join(", ")
|
|
13
|
+
end
|
|
14
|
+
|
|
8
15
|
def self.load(slug)
|
|
9
16
|
find_by!(slug: slug)
|
|
10
17
|
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
require "yaml"
|
|
2
|
+
require "pathname"
|
|
3
|
+
|
|
4
|
+
module LlmLogs
|
|
5
|
+
class PromptSyncer
|
|
6
|
+
REQUIRED_FIELDS = %w[slug name].freeze
|
|
7
|
+
|
|
8
|
+
def self.sync_all(root:, subfolders:)
|
|
9
|
+
root = Pathname.new(root)
|
|
10
|
+
subfolders.each do |sub|
|
|
11
|
+
dir = root / sub
|
|
12
|
+
next unless dir.directory?
|
|
13
|
+
|
|
14
|
+
Dir.glob(dir / "*.md").sort.each do |path|
|
|
15
|
+
# Skip files that are referenced as body_file targets rather than
|
|
16
|
+
# top-level prompts — those have no top-level `slug:` and are pulled
|
|
17
|
+
# in by the parent template instead.
|
|
18
|
+
pathname = Pathname.new(path)
|
|
19
|
+
next if body_file_only?(pathname)
|
|
20
|
+
new(path: pathname, auto_tag: sub).call
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.body_file_only?(path)
|
|
26
|
+
raw = File.read(path)
|
|
27
|
+
# Files referenced as body_file targets have no YAML front-matter.
|
|
28
|
+
# Files with front-matter are prompts and will be validated (missing
|
|
29
|
+
# slug/name raises a clear error rather than being silently skipped).
|
|
30
|
+
!raw.start_with?("---")
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.parse(raw)
|
|
34
|
+
_, front_yaml, body = raw.split(/^---\s*$/, 3)
|
|
35
|
+
[YAML.safe_load(front_yaml.to_s, permitted_classes: [Symbol], aliases: true) || {}, body.to_s.sub(/\A\n+/, "")]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def initialize(path:, auto_tag:)
|
|
39
|
+
@path = Pathname.new(path)
|
|
40
|
+
@auto_tag = auto_tag
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def call
|
|
44
|
+
raw = @path.read
|
|
45
|
+
raise "Missing front-matter in #{@path}" unless raw.start_with?("---")
|
|
46
|
+
|
|
47
|
+
front, body = self.class.parse(raw)
|
|
48
|
+
REQUIRED_FIELDS.each do |field|
|
|
49
|
+
raise "Missing required front-matter field '#{field}' in #{@path}" unless front[field].is_a?(String) && !front[field].strip.empty?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
ActiveRecord::Base.transaction do
|
|
53
|
+
tags = (Array(front["tags"]) + [@auto_tag]).map(&:to_s).uniq
|
|
54
|
+
prompt = LlmLogs::Prompt.find_or_initialize_by(slug: front["slug"])
|
|
55
|
+
created = prompt.new_record?
|
|
56
|
+
|
|
57
|
+
prompt.name = front["name"]
|
|
58
|
+
prompt.description = front["description"]
|
|
59
|
+
prompt.tags = tags
|
|
60
|
+
prompt.save!
|
|
61
|
+
|
|
62
|
+
messages = build_messages(front, body)
|
|
63
|
+
model = front["model"]
|
|
64
|
+
model_params = front["model_params"].is_a?(Hash) ? front["model_params"] : {}
|
|
65
|
+
model_params = model_params.deep_stringify_keys
|
|
66
|
+
|
|
67
|
+
if version_needs_update?(prompt, messages, model, model_params)
|
|
68
|
+
prompt.update_content!(messages: messages, model: model, model_params: model_params, changelog: "Synced from #{@path.basename}")
|
|
69
|
+
status = created ? "Created" : "Updated"
|
|
70
|
+
else
|
|
71
|
+
status = "Unchanged"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
log "#{status}: #{prompt.slug}"
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
|
|
80
|
+
def build_messages(front, body)
|
|
81
|
+
if front["messages"].is_a?(Array)
|
|
82
|
+
front["messages"].map do |msg|
|
|
83
|
+
msg = msg.deep_stringify_keys if msg.is_a?(Hash)
|
|
84
|
+
content =
|
|
85
|
+
if msg["body_file"]
|
|
86
|
+
body_path = @path.parent / msg["body_file"]
|
|
87
|
+
raise "Missing body_file #{msg['body_file']} referenced by #{@path}" unless body_path.exist?
|
|
88
|
+
body_path.read
|
|
89
|
+
else
|
|
90
|
+
msg["content"].to_s
|
|
91
|
+
end
|
|
92
|
+
{ "role" => msg["role"].to_s, "content" => content }
|
|
93
|
+
end
|
|
94
|
+
else
|
|
95
|
+
[{ "role" => "system", "content" => body }]
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def version_needs_update?(prompt, messages, model, model_params)
|
|
100
|
+
current = prompt.current_version
|
|
101
|
+
return true unless current
|
|
102
|
+
|
|
103
|
+
current.messages != messages ||
|
|
104
|
+
current.model.to_s != model.to_s ||
|
|
105
|
+
(current.model_params || {}).deep_stringify_keys != (model_params || {}).deep_stringify_keys
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def log(message)
|
|
109
|
+
if defined?(Rails) && Rails.logger
|
|
110
|
+
Rails.logger.info(message)
|
|
111
|
+
else
|
|
112
|
+
puts message
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -10,6 +10,99 @@
|
|
|
10
10
|
</script>
|
|
11
11
|
<style>
|
|
12
12
|
pre { white-space: pre-wrap; word-wrap: break-word; }
|
|
13
|
+
|
|
14
|
+
.markdown-body {
|
|
15
|
+
color: rgb(17 24 39);
|
|
16
|
+
line-height: 1.6;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
.markdown-body > :first-child {
|
|
20
|
+
margin-top: 0;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
.markdown-body > :last-child {
|
|
24
|
+
margin-bottom: 0;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
.markdown-body p,
|
|
28
|
+
.markdown-body ul,
|
|
29
|
+
.markdown-body ol,
|
|
30
|
+
.markdown-body blockquote,
|
|
31
|
+
.markdown-body pre,
|
|
32
|
+
.markdown-body table,
|
|
33
|
+
.markdown-body hr {
|
|
34
|
+
margin: 0.75rem 0;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
.markdown-body h1,
|
|
38
|
+
.markdown-body h2,
|
|
39
|
+
.markdown-body h3,
|
|
40
|
+
.markdown-body h4,
|
|
41
|
+
.markdown-body h5,
|
|
42
|
+
.markdown-body h6 {
|
|
43
|
+
margin: 1rem 0 0.5rem;
|
|
44
|
+
font-weight: 600;
|
|
45
|
+
line-height: 1.25;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
.markdown-body h1 { font-size: 1.25rem; }
|
|
49
|
+
.markdown-body h2 { font-size: 1.125rem; }
|
|
50
|
+
.markdown-body h3,
|
|
51
|
+
.markdown-body h4,
|
|
52
|
+
.markdown-body h5,
|
|
53
|
+
.markdown-body h6 { font-size: 1rem; }
|
|
54
|
+
|
|
55
|
+
.markdown-body ul,
|
|
56
|
+
.markdown-body ol {
|
|
57
|
+
padding-left: 1.25rem;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
.markdown-body ul { list-style: disc; }
|
|
61
|
+
.markdown-body ol { list-style: decimal; }
|
|
62
|
+
|
|
63
|
+
.markdown-body blockquote {
|
|
64
|
+
border-left: 3px solid rgb(209 213 219);
|
|
65
|
+
color: rgb(75 85 99);
|
|
66
|
+
padding-left: 0.75rem;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
.markdown-body code {
|
|
70
|
+
background: rgb(243 244 246);
|
|
71
|
+
border-radius: 0.25rem;
|
|
72
|
+
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
|
|
73
|
+
font-size: 0.875em;
|
|
74
|
+
padding: 0.125rem 0.25rem;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
.markdown-body pre {
|
|
78
|
+
background: rgb(249 250 251);
|
|
79
|
+
border-radius: 0.5rem;
|
|
80
|
+
overflow-x: auto;
|
|
81
|
+
padding: 0.75rem;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
.markdown-body pre code {
|
|
85
|
+
background: transparent;
|
|
86
|
+
border-radius: 0;
|
|
87
|
+
padding: 0;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
.markdown-body table {
|
|
91
|
+
border-collapse: collapse;
|
|
92
|
+
width: 100%;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
.markdown-body th,
|
|
96
|
+
.markdown-body td {
|
|
97
|
+
border: 1px solid rgb(229 231 235);
|
|
98
|
+
padding: 0.375rem 0.5rem;
|
|
99
|
+
text-align: left;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
.markdown-body th {
|
|
103
|
+
background: rgb(249 250 251);
|
|
104
|
+
font-weight: 600;
|
|
105
|
+
}
|
|
13
106
|
</style>
|
|
14
107
|
</head>
|
|
15
108
|
<body class="h-full">
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
<% @version.messages.each do |msg| %>
|
|
41
41
|
<div class="rounded-lg p-3 <%= msg['role'] == 'system' ? 'bg-yellow-50' : msg['role'] == 'user' ? 'bg-blue-50' : 'bg-gray-50' %>">
|
|
42
42
|
<div class="text-xs font-medium text-gray-500 uppercase mb-1"><%= msg['role'] %></div>
|
|
43
|
-
<
|
|
43
|
+
<div class="markdown-body text-sm"><%= render_markdown(msg["content"]) %></div>
|
|
44
44
|
</div>
|
|
45
45
|
<% end %>
|
|
46
46
|
</div>
|
|
@@ -27,6 +27,15 @@
|
|
|
27
27
|
<p class="mt-1 text-sm text-red-600"><%= error %></p>
|
|
28
28
|
<% end %>
|
|
29
29
|
</div>
|
|
30
|
+
|
|
31
|
+
<div>
|
|
32
|
+
<%= f.label :tags_input, "Tags", class: "block text-sm font-medium text-gray-700 mb-1" %>
|
|
33
|
+
<%= f.text_field :tags_input,
|
|
34
|
+
value: prompt.persisted? ? prompt.tags_string : nil,
|
|
35
|
+
class: "w-full rounded-md shadow-sm text-sm px-3 py-2 border border-gray-300",
|
|
36
|
+
placeholder: "comma-separated (e.g. skills, fragments)" %>
|
|
37
|
+
<p class="mt-1 text-xs text-gray-500">Used to classify and filter prompts. Free-form strings.</p>
|
|
38
|
+
</div>
|
|
30
39
|
</div>
|
|
31
40
|
|
|
32
41
|
<div class="bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5 p-6 space-y-4">
|
|
@@ -3,12 +3,23 @@
|
|
|
3
3
|
<%= link_to "New Prompt", new_prompt_path, class: "bg-indigo-600 text-white px-4 py-2 rounded-md text-sm font-medium hover:bg-indigo-500" %>
|
|
4
4
|
</div>
|
|
5
5
|
|
|
6
|
+
<% if @all_tags.present? %>
|
|
7
|
+
<div class="mb-4 flex flex-wrap gap-2 items-center text-sm">
|
|
8
|
+
<span class="text-gray-500">Filter:</span>
|
|
9
|
+
<%= link_to "All", prompts_path, class: "px-2 py-1 rounded #{@active_tag.blank? ? 'bg-indigo-600 text-white' : 'bg-gray-100 text-gray-700 hover:bg-gray-200'}" %>
|
|
10
|
+
<% @all_tags.each do |tag| %>
|
|
11
|
+
<%= link_to tag, prompts_path(tag: tag), class: "px-2 py-1 rounded #{@active_tag == tag ? 'bg-indigo-600 text-white' : 'bg-gray-100 text-gray-700 hover:bg-gray-200'}" %>
|
|
12
|
+
<% end %>
|
|
13
|
+
</div>
|
|
14
|
+
<% end %>
|
|
15
|
+
|
|
6
16
|
<div class="bg-white shadow-sm ring-1 ring-gray-900/5 rounded-lg overflow-hidden">
|
|
7
17
|
<table class="min-w-full divide-y divide-gray-200">
|
|
8
18
|
<thead class="bg-gray-50">
|
|
9
19
|
<tr>
|
|
10
20
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Name</th>
|
|
11
21
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Slug</th>
|
|
22
|
+
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Tags</th>
|
|
12
23
|
<th class="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase">Version</th>
|
|
13
24
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Updated</th>
|
|
14
25
|
</tr>
|
|
@@ -20,6 +31,11 @@
|
|
|
20
31
|
<%= link_to prompt.name, prompt_path(prompt), class: "text-indigo-600 hover:text-indigo-900 font-medium" %>
|
|
21
32
|
</td>
|
|
22
33
|
<td class="px-4 py-3 text-sm text-gray-500 font-mono"><%= prompt.slug %></td>
|
|
34
|
+
<td class="px-4 py-3 text-sm">
|
|
35
|
+
<% Array(prompt.tags).each do |tag| %>
|
|
36
|
+
<span class="inline-block bg-gray-100 text-gray-700 text-xs px-1.5 py-0.5 rounded mr-1"><%= tag %></span>
|
|
37
|
+
<% end %>
|
|
38
|
+
</td>
|
|
23
39
|
<td class="px-4 py-3 text-sm text-right">
|
|
24
40
|
<% if prompt.versions.any? %>
|
|
25
41
|
<%= link_to "v#{prompt.versions.maximum(:version_number)}", prompt_versions_path(prompt), class: "text-indigo-600 hover:text-indigo-900" %>
|
|
@@ -7,6 +7,12 @@
|
|
|
7
7
|
<% if @prompt.description.present? %>
|
|
8
8
|
· <%= @prompt.description %>
|
|
9
9
|
<% end %>
|
|
10
|
+
<% if @prompt.tags.present? %>
|
|
11
|
+
·
|
|
12
|
+
<% @prompt.tags.each do |tag| %>
|
|
13
|
+
<span class="inline-block bg-gray-100 text-gray-700 text-xs px-1.5 py-0.5 rounded ml-1"><%= tag %></span>
|
|
14
|
+
<% end %>
|
|
15
|
+
<% end %>
|
|
10
16
|
</p>
|
|
11
17
|
</div>
|
|
12
18
|
<div class="flex items-center space-x-3">
|
|
@@ -31,7 +37,7 @@
|
|
|
31
37
|
<% @current_version.messages.each do |msg| %>
|
|
32
38
|
<div class="rounded-lg p-3 <%= msg['role'] == 'system' ? 'bg-yellow-50' : msg['role'] == 'user' ? 'bg-blue-50' : 'bg-gray-50' %>">
|
|
33
39
|
<div class="text-xs font-medium text-gray-500 uppercase mb-1"><%= msg['role'] %></div>
|
|
34
|
-
<
|
|
40
|
+
<div class="markdown-body text-sm"><%= render_markdown(msg["content"]) %></div>
|
|
35
41
|
</div>
|
|
36
42
|
<% end %>
|
|
37
43
|
</div>
|
|
@@ -94,14 +100,18 @@
|
|
|
94
100
|
</div>
|
|
95
101
|
</div>
|
|
96
102
|
|
|
97
|
-
<
|
|
98
|
-
<
|
|
99
|
-
<
|
|
100
|
-
|
|
103
|
+
<details class="mt-4 bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5">
|
|
104
|
+
<summary class="cursor-pointer px-4 py-3 text-sm font-medium text-gray-900">SDK Usage</summary>
|
|
105
|
+
<div class="px-4 pb-4">
|
|
106
|
+
<pre class="text-xs bg-gray-50 rounded p-3 font-mono overflow-x-auto">prompt = LlmLogs::Prompt.load("<%= @prompt.slug %>")
|
|
107
|
+
<% if @current_version&.variables&.any? %>
|
|
108
|
+
params = prompt.build(
|
|
101
109
|
<%= @current_version.variables.map { |v| " #{v}: \"\"" }.join(",\n") %>
|
|
110
|
+
)
|
|
102
111
|
<% else %>
|
|
103
|
-
|
|
104
|
-
<% end
|
|
105
|
-
|
|
112
|
+
params = prompt.build
|
|
113
|
+
<% end %></pre>
|
|
114
|
+
</div>
|
|
115
|
+
</details>
|
|
106
116
|
</div>
|
|
107
117
|
</div>
|
|
@@ -7,4 +7,10 @@ LlmLogs.setup do |config|
|
|
|
7
7
|
|
|
8
8
|
# Number of days to keep trace data (for future cleanup job)
|
|
9
9
|
# config.retention_days = 30
|
|
10
|
+
|
|
11
|
+
# Directory used by `bin/rails llm_logs:prompts:sync`
|
|
12
|
+
# config.prompts_source_path = Rails.root.join("db/data/prompts")
|
|
13
|
+
|
|
14
|
+
# Subdirectories to sync within `prompts_source_path`
|
|
15
|
+
# config.prompt_subfolders = %w[skills fragments templates]
|
|
10
16
|
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
module LlmLogs
|
|
2
|
+
class Configuration
|
|
3
|
+
attr_accessor :enabled, :auto_instrument, :retention_days, :prompts_source_path, :prompt_subfolders
|
|
4
|
+
|
|
5
|
+
def initialize
|
|
6
|
+
@enabled = true
|
|
7
|
+
@auto_instrument = true
|
|
8
|
+
@retention_days = 30
|
|
9
|
+
@prompts_source_path = nil
|
|
10
|
+
@prompt_subfolders = %w[skills fragments templates]
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def self.configuration
|
|
15
|
+
@configuration ||= Configuration.new
|
|
16
|
+
end
|
|
17
|
+
end
|
data/lib/llm_logs/engine.rb
CHANGED
data/lib/llm_logs/version.rb
CHANGED
data/lib/llm_logs.rb
CHANGED
|
@@ -1,20 +1,41 @@
|
|
|
1
1
|
require "kaminari"
|
|
2
2
|
require "llm_logs/version"
|
|
3
|
+
require "llm_logs/configuration"
|
|
3
4
|
require "llm_logs/engine"
|
|
4
5
|
require "llm_logs/tracer"
|
|
5
6
|
require "llm_logs/prompt_renderer"
|
|
6
7
|
|
|
7
8
|
module LlmLogs
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
def self.setup
|
|
10
|
+
yield configuration
|
|
11
|
+
end
|
|
11
12
|
|
|
12
13
|
def self.enabled?
|
|
13
14
|
enabled
|
|
14
15
|
end
|
|
15
16
|
|
|
16
|
-
def self.
|
|
17
|
-
|
|
17
|
+
def self.enabled
|
|
18
|
+
configuration.enabled
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.enabled=(enabled)
|
|
22
|
+
configuration.enabled = enabled
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.auto_instrument
|
|
26
|
+
configuration.auto_instrument
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.auto_instrument=(auto_instrument)
|
|
30
|
+
configuration.auto_instrument = auto_instrument
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.retention_days
|
|
34
|
+
configuration.retention_days
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def self.retention_days=(retention_days)
|
|
38
|
+
configuration.retention_days = retention_days
|
|
18
39
|
end
|
|
19
40
|
|
|
20
41
|
def self.trace(name, **options, &block)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
namespace :llm_logs do
|
|
2
|
+
namespace :prompts do
|
|
3
|
+
desc "Sync LlmLogs::Prompt records from files in LlmLogs.configuration.prompts_source_path"
|
|
4
|
+
task sync: :environment do
|
|
5
|
+
path = LlmLogs.configuration.prompts_source_path
|
|
6
|
+
raise "LlmLogs.configuration.prompts_source_path is not set" unless path
|
|
7
|
+
|
|
8
|
+
LlmLogs::PromptSyncer.sync_all(
|
|
9
|
+
root: path,
|
|
10
|
+
subfolders: LlmLogs.configuration.prompt_subfolders
|
|
11
|
+
)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llm_logs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Anton
|
|
@@ -65,6 +65,34 @@ dependencies:
|
|
|
65
65
|
- - "~>"
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
67
|
version: '3.4'
|
|
68
|
+
- !ruby/object:Gem::Dependency
|
|
69
|
+
name: kramdown
|
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - "~>"
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '2.5'
|
|
75
|
+
type: :runtime
|
|
76
|
+
prerelease: false
|
|
77
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - "~>"
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: '2.5'
|
|
82
|
+
- !ruby/object:Gem::Dependency
|
|
83
|
+
name: kramdown-parser-gfm
|
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - "~>"
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: '1.1'
|
|
89
|
+
type: :runtime
|
|
90
|
+
prerelease: false
|
|
91
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
92
|
+
requirements:
|
|
93
|
+
- - "~>"
|
|
94
|
+
- !ruby/object:Gem::Version
|
|
95
|
+
version: '1.1'
|
|
68
96
|
description: Mountable Rails engine that provides hierarchical LLM call tracing and
|
|
69
97
|
versioned prompt management with Mustache templates.
|
|
70
98
|
executables: []
|
|
@@ -85,6 +113,7 @@ files:
|
|
|
85
113
|
- app/models/llm_logs/prompt_version.rb
|
|
86
114
|
- app/models/llm_logs/span.rb
|
|
87
115
|
- app/models/llm_logs/trace.rb
|
|
116
|
+
- app/services/llm_logs/prompt_syncer.rb
|
|
88
117
|
- app/views/kaminari/tailwind/_first_page.html.erb
|
|
89
118
|
- app/views/kaminari/tailwind/_gap.html.erb
|
|
90
119
|
- app/views/kaminari/tailwind/_last_page.html.erb
|
|
@@ -111,14 +140,17 @@ files:
|
|
|
111
140
|
- db/migrate/003_create_llm_logs_prompts.rb
|
|
112
141
|
- db/migrate/004_create_llm_logs_prompt_versions.rb
|
|
113
142
|
- db/migrate/005_add_prompt_version_to_traces.rb
|
|
143
|
+
- db/migrate/006_add_tags_to_prompts.rb
|
|
114
144
|
- lib/generators/llm_logs/install_generator.rb
|
|
115
145
|
- lib/generators/llm_logs/templates/initializer.rb
|
|
116
146
|
- lib/llm_logs.rb
|
|
147
|
+
- lib/llm_logs/configuration.rb
|
|
117
148
|
- lib/llm_logs/engine.rb
|
|
118
149
|
- lib/llm_logs/instrumentation/ruby_llm_chat.rb
|
|
119
150
|
- lib/llm_logs/prompt_renderer.rb
|
|
120
151
|
- lib/llm_logs/tracer.rb
|
|
121
152
|
- lib/llm_logs/version.rb
|
|
153
|
+
- lib/tasks/llm_logs.rake
|
|
122
154
|
homepage: https://github.com/tonic20/llm_logs
|
|
123
155
|
licenses:
|
|
124
156
|
- MIT
|