llm_logs 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.txt +21 -0
- data/README.md +67 -4
- data/app/controllers/llm_logs/prompts_controller.rb +11 -2
- data/app/helpers/llm_logs/formatting_helper.rb +20 -0
- data/app/models/llm_logs/prompt.rb +7 -0
- data/app/services/llm_logs/prompt_syncer.rb +116 -0
- data/app/views/layouts/llm_logs/application.html.erb +93 -0
- data/app/views/llm_logs/prompt_versions/show.html.erb +1 -1
- data/app/views/llm_logs/prompts/_form.html.erb +9 -0
- data/app/views/llm_logs/prompts/index.html.erb +16 -0
- data/app/views/llm_logs/prompts/show.html.erb +18 -8
- data/db/migrate/006_add_tags_to_prompts.rb +6 -0
- data/lib/generators/llm_logs/templates/initializer.rb +6 -0
- data/lib/llm_logs/configuration.rb +17 -0
- data/lib/llm_logs/engine.rb +4 -0
- data/lib/llm_logs/version.rb +1 -1
- data/lib/llm_logs.rb +26 -5
- data/lib/tasks/llm_logs.rake +14 -0
- metadata +34 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6234bcff772d83a80d691f6cf61e5238f50038feb031a00b7a013c0557fea3b0
|
|
4
|
+
data.tar.gz: 6b71fd68445d3817fe004a2ebae1ad5644ddc42f51ef0d224e9cde31aba3cc23
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b696c698bcbeaa6da42e3e45a2f02bc4368fcbb7fa8fef3a49b6844cef1b949ebaa0c4f7a43ad6c4c4d4f178f8d93f4cd73af2e207b6941726afac10e8b152ac
|
|
7
|
+
data.tar.gz: eca247f5563982227a6f5a2b664ec4b68ff96fe93110ea9dde0974ecedd78cdbbb3a480542e90fc87aa6445f43ceec35b160e76242f2439f27a0eaead90a2197
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Anton Kopylov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
CHANGED
|
@@ -7,7 +7,7 @@ Mountable Rails engine for LLM call tracing and prompt management. Auto-instrume
|
|
|
7
7
|
Add to your Gemfile:
|
|
8
8
|
|
|
9
9
|
```ruby
|
|
10
|
-
gem "llm_logs"
|
|
10
|
+
gem "llm_logs"
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
Run the install generator:
|
|
@@ -29,6 +29,7 @@ mount LlmLogs::Engine, at: "/llm_logs"
|
|
|
29
29
|
LlmLogs.setup do |config|
|
|
30
30
|
config.enabled = true
|
|
31
31
|
config.auto_instrument = true
|
|
32
|
+
config.prompts_source_path = Rails.root.join("db/data/prompts")
|
|
32
33
|
end
|
|
33
34
|
```
|
|
34
35
|
|
|
@@ -119,6 +120,66 @@ prompt.version(1) # specific version
|
|
|
119
120
|
prompt.rollback_to!(1) # creates new version from v1 content
|
|
120
121
|
```
|
|
121
122
|
|
|
123
|
+
### Sync Prompts From Files
|
|
124
|
+
|
|
125
|
+
Store prompts as Markdown files and sync them into `LlmLogs::Prompt` records with the rake task.
|
|
126
|
+
|
|
127
|
+
```ruby
|
|
128
|
+
# config/initializers/llm_logs.rb
|
|
129
|
+
LlmLogs.setup do |config|
|
|
130
|
+
config.prompts_source_path = Rails.root.join("db/data/prompts")
|
|
131
|
+
config.prompt_subfolders = %w[skills fragments templates]
|
|
132
|
+
end
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
```sh
|
|
136
|
+
bin/rails llm_logs:prompts:sync
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The syncer reads `*.md` files from each configured subfolder. The subfolder name is added as a prompt tag automatically.
|
|
140
|
+
|
|
141
|
+
```text
|
|
142
|
+
db/data/prompts/
|
|
143
|
+
skills/
|
|
144
|
+
backtest-evaluation.md
|
|
145
|
+
fragments/
|
|
146
|
+
provider-notes.md
|
|
147
|
+
templates/
|
|
148
|
+
trading-memo.md
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Single-message prompts use the Markdown body as the system message:
|
|
152
|
+
|
|
153
|
+
```markdown
|
|
154
|
+
---
|
|
155
|
+
slug: backtest-evaluation
|
|
156
|
+
name: Backtest Evaluation
|
|
157
|
+
description: How to evaluate backtests
|
|
158
|
+
tags: [evaluation]
|
|
159
|
+
model: anthropic/claude-sonnet-4
|
|
160
|
+
model_params:
|
|
161
|
+
temperature: 0.3
|
|
162
|
+
---
|
|
163
|
+
Body content here.
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Multi-message prompts can reference sibling body files:
|
|
167
|
+
|
|
168
|
+
```markdown
|
|
169
|
+
---
|
|
170
|
+
slug: trading-memo
|
|
171
|
+
name: Trading Memo
|
|
172
|
+
model: deepseek/deepseek-v3.2
|
|
173
|
+
messages:
|
|
174
|
+
- role: system
|
|
175
|
+
body_file: trading_memo_system.md
|
|
176
|
+
- role: user
|
|
177
|
+
body_file: trading_memo_user.md
|
|
178
|
+
---
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Running the task creates missing prompts, updates metadata, and creates a new prompt version only when messages, model, or model parameters changed.
|
|
182
|
+
|
|
122
183
|
## Web UI
|
|
123
184
|
|
|
124
185
|
Browse traces and manage prompts at `/llm_logs`.
|
|
@@ -131,9 +192,11 @@ Browse traces and manage prompts at `/llm_logs`.
|
|
|
131
192
|
|
|
132
193
|
```ruby
|
|
133
194
|
LlmLogs.setup do |config|
|
|
134
|
-
config.enabled = true
|
|
135
|
-
config.auto_instrument = true
|
|
136
|
-
config.retention_days = 30
|
|
195
|
+
config.enabled = true # master switch for logging
|
|
196
|
+
config.auto_instrument = true # auto-prepend on RubyLLM::Chat
|
|
197
|
+
config.retention_days = 30 # for future cleanup job
|
|
198
|
+
config.prompts_source_path = Rails.root.join("db/data/prompts")
|
|
199
|
+
config.prompt_subfolders = %w[skills fragments templates]
|
|
137
200
|
end
|
|
138
201
|
```
|
|
139
202
|
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
module LlmLogs
|
|
2
2
|
class PromptsController < ApplicationController
|
|
3
3
|
def index
|
|
4
|
-
|
|
4
|
+
tag = params[:tag].is_a?(String) ? params[:tag].presence : nil
|
|
5
|
+
scope = Prompt.order(:name).includes(:versions)
|
|
6
|
+
scope = scope.with_tag(tag) if tag
|
|
7
|
+
@prompts = scope.page(params[:page]).per(25)
|
|
8
|
+
@active_tag = tag
|
|
9
|
+
@all_tags = Prompt.pluck(:tags).flatten.compact.uniq.sort
|
|
5
10
|
end
|
|
6
11
|
|
|
7
12
|
def show
|
|
@@ -54,7 +59,11 @@ module LlmLogs
|
|
|
54
59
|
private
|
|
55
60
|
|
|
56
61
|
def prompt_params
|
|
57
|
-
params.require(:prompt).permit(:slug, :name, :description)
|
|
62
|
+
raw = params.require(:prompt).permit(:slug, :name, :description, :tags_input, tags: [])
|
|
63
|
+
if raw[:tags_input].present?
|
|
64
|
+
raw[:tags] = raw[:tags_input].split(",").map(&:strip).reject(&:blank?)
|
|
65
|
+
end
|
|
66
|
+
raw.except(:tags_input)
|
|
58
67
|
end
|
|
59
68
|
|
|
60
69
|
def version_params
|
|
@@ -1,11 +1,31 @@
|
|
|
1
|
+
require "kramdown"
|
|
2
|
+
require "kramdown-parser-gfm"
|
|
3
|
+
|
|
1
4
|
module LlmLogs
|
|
2
5
|
module FormattingHelper
|
|
6
|
+
MARKDOWN_TAGS = %w[
|
|
7
|
+
a blockquote br code del em h1 h2 h3 h4 h5 h6 hr li ol p pre strong
|
|
8
|
+
table tbody td th thead tr ul
|
|
9
|
+
].freeze
|
|
10
|
+
MARKDOWN_ATTRIBUTES = %w[href title].freeze
|
|
11
|
+
|
|
3
12
|
def format_duration_ms(ms)
|
|
4
13
|
return "—" if ms.nil?
|
|
5
14
|
|
|
6
15
|
"#{number_with_delimiter(sprintf('%.0f', ms))} ms"
|
|
7
16
|
end
|
|
8
17
|
|
|
18
|
+
def render_markdown(text)
|
|
19
|
+
html = Kramdown::Document.new(
|
|
20
|
+
text.to_s,
|
|
21
|
+
input: "GFM",
|
|
22
|
+
hard_wrap: true,
|
|
23
|
+
syntax_highlighter: nil
|
|
24
|
+
).to_html
|
|
25
|
+
|
|
26
|
+
sanitize html, tags: MARKDOWN_TAGS, attributes: MARKDOWN_ATTRIBUTES
|
|
27
|
+
end
|
|
28
|
+
|
|
9
29
|
def pretty_json(data)
|
|
10
30
|
JSON.pretty_generate(deep_parse_json(data))
|
|
11
31
|
rescue
|
|
@@ -5,6 +5,13 @@ module LlmLogs
|
|
|
5
5
|
validates :slug, presence: true, uniqueness: true
|
|
6
6
|
validates :name, presence: true
|
|
7
7
|
|
|
8
|
+
scope :with_tag, ->(tag) { where("? = ANY(tags)", tag) }
|
|
9
|
+
scope :with_any_tag, ->(tags) { where("tags && ARRAY[?]::varchar[]", Array(tags)) }
|
|
10
|
+
|
|
11
|
+
def tags_string
|
|
12
|
+
Array(tags).join(", ")
|
|
13
|
+
end
|
|
14
|
+
|
|
8
15
|
def self.load(slug)
|
|
9
16
|
find_by!(slug: slug)
|
|
10
17
|
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
require "yaml"
|
|
2
|
+
require "pathname"
|
|
3
|
+
|
|
4
|
+
module LlmLogs
|
|
5
|
+
class PromptSyncer
|
|
6
|
+
REQUIRED_FIELDS = %w[slug name].freeze
|
|
7
|
+
|
|
8
|
+
def self.sync_all(root:, subfolders:)
|
|
9
|
+
root = Pathname.new(root)
|
|
10
|
+
subfolders.each do |sub|
|
|
11
|
+
dir = root / sub
|
|
12
|
+
next unless dir.directory?
|
|
13
|
+
|
|
14
|
+
Dir.glob(dir / "*.md").sort.each do |path|
|
|
15
|
+
# Skip files that are referenced as body_file targets rather than
|
|
16
|
+
# top-level prompts — those have no top-level `slug:` and are pulled
|
|
17
|
+
# in by the parent template instead.
|
|
18
|
+
pathname = Pathname.new(path)
|
|
19
|
+
next if body_file_only?(pathname)
|
|
20
|
+
new(path: pathname, auto_tag: sub).call
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.body_file_only?(path)
|
|
26
|
+
raw = File.read(path)
|
|
27
|
+
# Files referenced as body_file targets have no YAML front-matter.
|
|
28
|
+
# Files with front-matter are prompts and will be validated (missing
|
|
29
|
+
# slug/name raises a clear error rather than being silently skipped).
|
|
30
|
+
!raw.start_with?("---")
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.parse(raw)
|
|
34
|
+
_, front_yaml, body = raw.split(/^---\s*$/, 3)
|
|
35
|
+
[YAML.safe_load(front_yaml.to_s, permitted_classes: [Symbol], aliases: true) || {}, body.to_s.sub(/\A\n+/, "")]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def initialize(path:, auto_tag:)
|
|
39
|
+
@path = Pathname.new(path)
|
|
40
|
+
@auto_tag = auto_tag
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def call
|
|
44
|
+
raw = @path.read
|
|
45
|
+
raise "Missing front-matter in #{@path}" unless raw.start_with?("---")
|
|
46
|
+
|
|
47
|
+
front, body = self.class.parse(raw)
|
|
48
|
+
REQUIRED_FIELDS.each do |field|
|
|
49
|
+
raise "Missing required front-matter field '#{field}' in #{@path}" unless front[field].is_a?(String) && !front[field].strip.empty?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
ActiveRecord::Base.transaction do
|
|
53
|
+
tags = (Array(front["tags"]) + [@auto_tag]).map(&:to_s).uniq
|
|
54
|
+
prompt = LlmLogs::Prompt.find_or_initialize_by(slug: front["slug"])
|
|
55
|
+
created = prompt.new_record?
|
|
56
|
+
|
|
57
|
+
prompt.name = front["name"]
|
|
58
|
+
prompt.description = front["description"]
|
|
59
|
+
prompt.tags = tags
|
|
60
|
+
prompt.save!
|
|
61
|
+
|
|
62
|
+
messages = build_messages(front, body)
|
|
63
|
+
model = front["model"]
|
|
64
|
+
model_params = front["model_params"].is_a?(Hash) ? front["model_params"] : {}
|
|
65
|
+
model_params = model_params.deep_stringify_keys
|
|
66
|
+
|
|
67
|
+
if version_needs_update?(prompt, messages, model, model_params)
|
|
68
|
+
prompt.update_content!(messages: messages, model: model, model_params: model_params, changelog: "Synced from #{@path.basename}")
|
|
69
|
+
status = created ? "Created" : "Updated"
|
|
70
|
+
else
|
|
71
|
+
status = "Unchanged"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
log "#{status}: #{prompt.slug}"
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
|
|
80
|
+
def build_messages(front, body)
|
|
81
|
+
if front["messages"].is_a?(Array)
|
|
82
|
+
front["messages"].map do |msg|
|
|
83
|
+
msg = msg.deep_stringify_keys if msg.is_a?(Hash)
|
|
84
|
+
content =
|
|
85
|
+
if msg["body_file"]
|
|
86
|
+
body_path = @path.parent / msg["body_file"]
|
|
87
|
+
raise "Missing body_file #{msg['body_file']} referenced by #{@path}" unless body_path.exist?
|
|
88
|
+
body_path.read
|
|
89
|
+
else
|
|
90
|
+
msg["content"].to_s
|
|
91
|
+
end
|
|
92
|
+
{ "role" => msg["role"].to_s, "content" => content }
|
|
93
|
+
end
|
|
94
|
+
else
|
|
95
|
+
[{ "role" => "system", "content" => body }]
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def version_needs_update?(prompt, messages, model, model_params)
|
|
100
|
+
current = prompt.current_version
|
|
101
|
+
return true unless current
|
|
102
|
+
|
|
103
|
+
current.messages != messages ||
|
|
104
|
+
current.model.to_s != model.to_s ||
|
|
105
|
+
(current.model_params || {}).deep_stringify_keys != (model_params || {}).deep_stringify_keys
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def log(message)
|
|
109
|
+
if defined?(Rails) && Rails.logger
|
|
110
|
+
Rails.logger.info(message)
|
|
111
|
+
else
|
|
112
|
+
puts message
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -10,6 +10,99 @@
|
|
|
10
10
|
</script>
|
|
11
11
|
<style>
|
|
12
12
|
pre { white-space: pre-wrap; word-wrap: break-word; }
|
|
13
|
+
|
|
14
|
+
.markdown-body {
|
|
15
|
+
color: rgb(17 24 39);
|
|
16
|
+
line-height: 1.6;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
.markdown-body > :first-child {
|
|
20
|
+
margin-top: 0;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
.markdown-body > :last-child {
|
|
24
|
+
margin-bottom: 0;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
.markdown-body p,
|
|
28
|
+
.markdown-body ul,
|
|
29
|
+
.markdown-body ol,
|
|
30
|
+
.markdown-body blockquote,
|
|
31
|
+
.markdown-body pre,
|
|
32
|
+
.markdown-body table,
|
|
33
|
+
.markdown-body hr {
|
|
34
|
+
margin: 0.75rem 0;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
.markdown-body h1,
|
|
38
|
+
.markdown-body h2,
|
|
39
|
+
.markdown-body h3,
|
|
40
|
+
.markdown-body h4,
|
|
41
|
+
.markdown-body h5,
|
|
42
|
+
.markdown-body h6 {
|
|
43
|
+
margin: 1rem 0 0.5rem;
|
|
44
|
+
font-weight: 600;
|
|
45
|
+
line-height: 1.25;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
.markdown-body h1 { font-size: 1.25rem; }
|
|
49
|
+
.markdown-body h2 { font-size: 1.125rem; }
|
|
50
|
+
.markdown-body h3,
|
|
51
|
+
.markdown-body h4,
|
|
52
|
+
.markdown-body h5,
|
|
53
|
+
.markdown-body h6 { font-size: 1rem; }
|
|
54
|
+
|
|
55
|
+
.markdown-body ul,
|
|
56
|
+
.markdown-body ol {
|
|
57
|
+
padding-left: 1.25rem;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
.markdown-body ul { list-style: disc; }
|
|
61
|
+
.markdown-body ol { list-style: decimal; }
|
|
62
|
+
|
|
63
|
+
.markdown-body blockquote {
|
|
64
|
+
border-left: 3px solid rgb(209 213 219);
|
|
65
|
+
color: rgb(75 85 99);
|
|
66
|
+
padding-left: 0.75rem;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
.markdown-body code {
|
|
70
|
+
background: rgb(243 244 246);
|
|
71
|
+
border-radius: 0.25rem;
|
|
72
|
+
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
|
|
73
|
+
font-size: 0.875em;
|
|
74
|
+
padding: 0.125rem 0.25rem;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
.markdown-body pre {
|
|
78
|
+
background: rgb(249 250 251);
|
|
79
|
+
border-radius: 0.5rem;
|
|
80
|
+
overflow-x: auto;
|
|
81
|
+
padding: 0.75rem;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
.markdown-body pre code {
|
|
85
|
+
background: transparent;
|
|
86
|
+
border-radius: 0;
|
|
87
|
+
padding: 0;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
.markdown-body table {
|
|
91
|
+
border-collapse: collapse;
|
|
92
|
+
width: 100%;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
.markdown-body th,
|
|
96
|
+
.markdown-body td {
|
|
97
|
+
border: 1px solid rgb(229 231 235);
|
|
98
|
+
padding: 0.375rem 0.5rem;
|
|
99
|
+
text-align: left;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
.markdown-body th {
|
|
103
|
+
background: rgb(249 250 251);
|
|
104
|
+
font-weight: 600;
|
|
105
|
+
}
|
|
13
106
|
</style>
|
|
14
107
|
</head>
|
|
15
108
|
<body class="h-full">
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
<% @version.messages.each do |msg| %>
|
|
41
41
|
<div class="rounded-lg p-3 <%= msg['role'] == 'system' ? 'bg-yellow-50' : msg['role'] == 'user' ? 'bg-blue-50' : 'bg-gray-50' %>">
|
|
42
42
|
<div class="text-xs font-medium text-gray-500 uppercase mb-1"><%= msg['role'] %></div>
|
|
43
|
-
<
|
|
43
|
+
<div class="markdown-body text-sm"><%= render_markdown(msg["content"]) %></div>
|
|
44
44
|
</div>
|
|
45
45
|
<% end %>
|
|
46
46
|
</div>
|
|
@@ -27,6 +27,15 @@
|
|
|
27
27
|
<p class="mt-1 text-sm text-red-600"><%= error %></p>
|
|
28
28
|
<% end %>
|
|
29
29
|
</div>
|
|
30
|
+
|
|
31
|
+
<div>
|
|
32
|
+
<%= f.label :tags_input, "Tags", class: "block text-sm font-medium text-gray-700 mb-1" %>
|
|
33
|
+
<%= f.text_field :tags_input,
|
|
34
|
+
value: prompt.persisted? ? prompt.tags_string : nil,
|
|
35
|
+
class: "w-full rounded-md shadow-sm text-sm px-3 py-2 border border-gray-300",
|
|
36
|
+
placeholder: "comma-separated (e.g. skills, fragments)" %>
|
|
37
|
+
<p class="mt-1 text-xs text-gray-500">Used to classify and filter prompts. Free-form strings.</p>
|
|
38
|
+
</div>
|
|
30
39
|
</div>
|
|
31
40
|
|
|
32
41
|
<div class="bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5 p-6 space-y-4">
|
|
@@ -3,12 +3,23 @@
|
|
|
3
3
|
<%= link_to "New Prompt", new_prompt_path, class: "bg-indigo-600 text-white px-4 py-2 rounded-md text-sm font-medium hover:bg-indigo-500" %>
|
|
4
4
|
</div>
|
|
5
5
|
|
|
6
|
+
<% if @all_tags.present? %>
|
|
7
|
+
<div class="mb-4 flex flex-wrap gap-2 items-center text-sm">
|
|
8
|
+
<span class="text-gray-500">Filter:</span>
|
|
9
|
+
<%= link_to "All", prompts_path, class: "px-2 py-1 rounded #{@active_tag.blank? ? 'bg-indigo-600 text-white' : 'bg-gray-100 text-gray-700 hover:bg-gray-200'}" %>
|
|
10
|
+
<% @all_tags.each do |tag| %>
|
|
11
|
+
<%= link_to tag, prompts_path(tag: tag), class: "px-2 py-1 rounded #{@active_tag == tag ? 'bg-indigo-600 text-white' : 'bg-gray-100 text-gray-700 hover:bg-gray-200'}" %>
|
|
12
|
+
<% end %>
|
|
13
|
+
</div>
|
|
14
|
+
<% end %>
|
|
15
|
+
|
|
6
16
|
<div class="bg-white shadow-sm ring-1 ring-gray-900/5 rounded-lg overflow-hidden">
|
|
7
17
|
<table class="min-w-full divide-y divide-gray-200">
|
|
8
18
|
<thead class="bg-gray-50">
|
|
9
19
|
<tr>
|
|
10
20
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Name</th>
|
|
11
21
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Slug</th>
|
|
22
|
+
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Tags</th>
|
|
12
23
|
<th class="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase">Version</th>
|
|
13
24
|
<th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Updated</th>
|
|
14
25
|
</tr>
|
|
@@ -20,6 +31,11 @@
|
|
|
20
31
|
<%= link_to prompt.name, prompt_path(prompt), class: "text-indigo-600 hover:text-indigo-900 font-medium" %>
|
|
21
32
|
</td>
|
|
22
33
|
<td class="px-4 py-3 text-sm text-gray-500 font-mono"><%= prompt.slug %></td>
|
|
34
|
+
<td class="px-4 py-3 text-sm">
|
|
35
|
+
<% Array(prompt.tags).each do |tag| %>
|
|
36
|
+
<span class="inline-block bg-gray-100 text-gray-700 text-xs px-1.5 py-0.5 rounded mr-1"><%= tag %></span>
|
|
37
|
+
<% end %>
|
|
38
|
+
</td>
|
|
23
39
|
<td class="px-4 py-3 text-sm text-right">
|
|
24
40
|
<% if prompt.versions.any? %>
|
|
25
41
|
<%= link_to "v#{prompt.versions.maximum(:version_number)}", prompt_versions_path(prompt), class: "text-indigo-600 hover:text-indigo-900" %>
|
|
@@ -7,6 +7,12 @@
|
|
|
7
7
|
<% if @prompt.description.present? %>
|
|
8
8
|
· <%= @prompt.description %>
|
|
9
9
|
<% end %>
|
|
10
|
+
<% if @prompt.tags.present? %>
|
|
11
|
+
·
|
|
12
|
+
<% @prompt.tags.each do |tag| %>
|
|
13
|
+
<span class="inline-block bg-gray-100 text-gray-700 text-xs px-1.5 py-0.5 rounded ml-1"><%= tag %></span>
|
|
14
|
+
<% end %>
|
|
15
|
+
<% end %>
|
|
10
16
|
</p>
|
|
11
17
|
</div>
|
|
12
18
|
<div class="flex items-center space-x-3">
|
|
@@ -31,7 +37,7 @@
|
|
|
31
37
|
<% @current_version.messages.each do |msg| %>
|
|
32
38
|
<div class="rounded-lg p-3 <%= msg['role'] == 'system' ? 'bg-yellow-50' : msg['role'] == 'user' ? 'bg-blue-50' : 'bg-gray-50' %>">
|
|
33
39
|
<div class="text-xs font-medium text-gray-500 uppercase mb-1"><%= msg['role'] %></div>
|
|
34
|
-
<
|
|
40
|
+
<div class="markdown-body text-sm"><%= render_markdown(msg["content"]) %></div>
|
|
35
41
|
</div>
|
|
36
42
|
<% end %>
|
|
37
43
|
</div>
|
|
@@ -94,14 +100,18 @@
|
|
|
94
100
|
</div>
|
|
95
101
|
</div>
|
|
96
102
|
|
|
97
|
-
<
|
|
98
|
-
<
|
|
99
|
-
<
|
|
100
|
-
|
|
103
|
+
<details class="mt-4 bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5">
|
|
104
|
+
<summary class="cursor-pointer px-4 py-3 text-sm font-medium text-gray-900">SDK Usage</summary>
|
|
105
|
+
<div class="px-4 pb-4">
|
|
106
|
+
<pre class="text-xs bg-gray-50 rounded p-3 font-mono overflow-x-auto">prompt = LlmLogs::Prompt.load("<%= @prompt.slug %>")
|
|
107
|
+
<% if @current_version&.variables&.any? %>
|
|
108
|
+
params = prompt.build(
|
|
101
109
|
<%= @current_version.variables.map { |v| " #{v}: \"\"" }.join(",\n") %>
|
|
110
|
+
)
|
|
102
111
|
<% else %>
|
|
103
|
-
|
|
104
|
-
<% end
|
|
105
|
-
|
|
112
|
+
params = prompt.build
|
|
113
|
+
<% end %></pre>
|
|
114
|
+
</div>
|
|
115
|
+
</details>
|
|
106
116
|
</div>
|
|
107
117
|
</div>
|
|
@@ -7,4 +7,10 @@ LlmLogs.setup do |config|
|
|
|
7
7
|
|
|
8
8
|
# Number of days to keep trace data (for future cleanup job)
|
|
9
9
|
# config.retention_days = 30
|
|
10
|
+
|
|
11
|
+
# Directory used by `bin/rails llm_logs:prompts:sync`
|
|
12
|
+
# config.prompts_source_path = Rails.root.join("db/data/prompts")
|
|
13
|
+
|
|
14
|
+
# Subdirectories to sync within `prompts_source_path`
|
|
15
|
+
# config.prompt_subfolders = %w[skills fragments templates]
|
|
10
16
|
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
module LlmLogs
|
|
2
|
+
class Configuration
|
|
3
|
+
attr_accessor :enabled, :auto_instrument, :retention_days, :prompts_source_path, :prompt_subfolders
|
|
4
|
+
|
|
5
|
+
def initialize
|
|
6
|
+
@enabled = true
|
|
7
|
+
@auto_instrument = true
|
|
8
|
+
@retention_days = 30
|
|
9
|
+
@prompts_source_path = nil
|
|
10
|
+
@prompt_subfolders = %w[skills fragments templates]
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def self.configuration
|
|
15
|
+
@configuration ||= Configuration.new
|
|
16
|
+
end
|
|
17
|
+
end
|
data/lib/llm_logs/engine.rb
CHANGED
data/lib/llm_logs/version.rb
CHANGED
data/lib/llm_logs.rb
CHANGED
|
@@ -1,20 +1,41 @@
|
|
|
1
1
|
require "kaminari"
|
|
2
2
|
require "llm_logs/version"
|
|
3
|
+
require "llm_logs/configuration"
|
|
3
4
|
require "llm_logs/engine"
|
|
4
5
|
require "llm_logs/tracer"
|
|
5
6
|
require "llm_logs/prompt_renderer"
|
|
6
7
|
|
|
7
8
|
module LlmLogs
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
def self.setup
|
|
10
|
+
yield configuration
|
|
11
|
+
end
|
|
11
12
|
|
|
12
13
|
def self.enabled?
|
|
13
14
|
enabled
|
|
14
15
|
end
|
|
15
16
|
|
|
16
|
-
def self.
|
|
17
|
-
|
|
17
|
+
def self.enabled
|
|
18
|
+
configuration.enabled
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.enabled=(enabled)
|
|
22
|
+
configuration.enabled = enabled
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.auto_instrument
|
|
26
|
+
configuration.auto_instrument
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.auto_instrument=(auto_instrument)
|
|
30
|
+
configuration.auto_instrument = auto_instrument
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.retention_days
|
|
34
|
+
configuration.retention_days
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def self.retention_days=(retention_days)
|
|
38
|
+
configuration.retention_days = retention_days
|
|
18
39
|
end
|
|
19
40
|
|
|
20
41
|
def self.trace(name, **options, &block)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
namespace :llm_logs do
|
|
2
|
+
namespace :prompts do
|
|
3
|
+
desc "Sync LlmLogs::Prompt records from files in LlmLogs.configuration.prompts_source_path"
|
|
4
|
+
task sync: :environment do
|
|
5
|
+
path = LlmLogs.configuration.prompts_source_path
|
|
6
|
+
raise "LlmLogs.configuration.prompts_source_path is not set" unless path
|
|
7
|
+
|
|
8
|
+
LlmLogs::PromptSyncer.sync_all(
|
|
9
|
+
root: path,
|
|
10
|
+
subfolders: LlmLogs.configuration.prompt_subfolders
|
|
11
|
+
)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llm_logs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Anton
|
|
@@ -65,12 +65,41 @@ dependencies:
|
|
|
65
65
|
- - "~>"
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
67
|
version: '3.4'
|
|
68
|
+
- !ruby/object:Gem::Dependency
|
|
69
|
+
name: kramdown
|
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - "~>"
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '2.5'
|
|
75
|
+
type: :runtime
|
|
76
|
+
prerelease: false
|
|
77
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - "~>"
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: '2.5'
|
|
82
|
+
- !ruby/object:Gem::Dependency
|
|
83
|
+
name: kramdown-parser-gfm
|
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - "~>"
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: '1.1'
|
|
89
|
+
type: :runtime
|
|
90
|
+
prerelease: false
|
|
91
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
92
|
+
requirements:
|
|
93
|
+
- - "~>"
|
|
94
|
+
- !ruby/object:Gem::Version
|
|
95
|
+
version: '1.1'
|
|
68
96
|
description: Mountable Rails engine that provides hierarchical LLM call tracing and
|
|
69
97
|
versioned prompt management with Mustache templates.
|
|
70
98
|
executables: []
|
|
71
99
|
extensions: []
|
|
72
100
|
extra_rdoc_files: []
|
|
73
101
|
files:
|
|
102
|
+
- LICENSE.txt
|
|
74
103
|
- README.md
|
|
75
104
|
- Rakefile
|
|
76
105
|
- app/controllers/llm_logs/application_controller.rb
|
|
@@ -84,6 +113,7 @@ files:
|
|
|
84
113
|
- app/models/llm_logs/prompt_version.rb
|
|
85
114
|
- app/models/llm_logs/span.rb
|
|
86
115
|
- app/models/llm_logs/trace.rb
|
|
116
|
+
- app/services/llm_logs/prompt_syncer.rb
|
|
87
117
|
- app/views/kaminari/tailwind/_first_page.html.erb
|
|
88
118
|
- app/views/kaminari/tailwind/_gap.html.erb
|
|
89
119
|
- app/views/kaminari/tailwind/_last_page.html.erb
|
|
@@ -110,14 +140,17 @@ files:
|
|
|
110
140
|
- db/migrate/003_create_llm_logs_prompts.rb
|
|
111
141
|
- db/migrate/004_create_llm_logs_prompt_versions.rb
|
|
112
142
|
- db/migrate/005_add_prompt_version_to_traces.rb
|
|
143
|
+
- db/migrate/006_add_tags_to_prompts.rb
|
|
113
144
|
- lib/generators/llm_logs/install_generator.rb
|
|
114
145
|
- lib/generators/llm_logs/templates/initializer.rb
|
|
115
146
|
- lib/llm_logs.rb
|
|
147
|
+
- lib/llm_logs/configuration.rb
|
|
116
148
|
- lib/llm_logs/engine.rb
|
|
117
149
|
- lib/llm_logs/instrumentation/ruby_llm_chat.rb
|
|
118
150
|
- lib/llm_logs/prompt_renderer.rb
|
|
119
151
|
- lib/llm_logs/tracer.rb
|
|
120
152
|
- lib/llm_logs/version.rb
|
|
153
|
+
- lib/tasks/llm_logs.rake
|
|
121
154
|
homepage: https://github.com/tonic20/llm_logs
|
|
122
155
|
licenses:
|
|
123
156
|
- MIT
|