tracebook 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +10 -0
- data/CHANGELOG.md +43 -0
- data/MIT-LICENSE +20 -0
- data/README.md +881 -0
- data/Rakefile +21 -0
- data/app/assets/images/tracebook/.keep +0 -0
- data/app/assets/javascripts/tracebook/application.js +88 -0
- data/app/assets/stylesheets/tracebook/application.css +173 -0
- data/app/controllers/concerns/.keep +0 -0
- data/app/controllers/tracebook/application_controller.rb +4 -0
- data/app/controllers/tracebook/exports_controller.rb +25 -0
- data/app/controllers/tracebook/interactions_controller.rb +71 -0
- data/app/helpers/tracebook/application_helper.rb +4 -0
- data/app/helpers/tracebook/interactions_helper.rb +35 -0
- data/app/jobs/tracebook/application_job.rb +5 -0
- data/app/jobs/tracebook/daily_rollups_job.rb +100 -0
- data/app/jobs/tracebook/export_job.rb +162 -0
- data/app/jobs/tracebook/persist_interaction_job.rb +160 -0
- data/app/mailers/tracebook/application_mailer.rb +6 -0
- data/app/models/concerns/.keep +0 -0
- data/app/models/tracebook/application_record.rb +5 -0
- data/app/models/tracebook/interaction.rb +100 -0
- data/app/models/tracebook/pricing_rule.rb +84 -0
- data/app/models/tracebook/redaction_rule.rb +81 -0
- data/app/models/tracebook/rollup_daily.rb +73 -0
- data/app/views/layouts/tracebook/application.html.erb +18 -0
- data/app/views/tracebook/interactions/index.html.erb +105 -0
- data/app/views/tracebook/interactions/show.html.erb +44 -0
- data/config/routes.rb +8 -0
- data/db/migrate/20241112000100_create_tracebook_interactions.rb +55 -0
- data/db/migrate/20241112000200_create_tracebook_rollups_dailies.rb +24 -0
- data/db/migrate/20241112000300_create_tracebook_pricing_rules.rb +21 -0
- data/db/migrate/20241112000400_create_tracebook_redaction_rules.rb +19 -0
- data/lib/tasks/tracebook_tasks.rake +4 -0
- data/lib/tasks/yard.rake +29 -0
- data/lib/tracebook/adapters/active_agent.rb +82 -0
- data/lib/tracebook/adapters/ruby_llm.rb +97 -0
- data/lib/tracebook/adapters.rb +6 -0
- data/lib/tracebook/config.rb +130 -0
- data/lib/tracebook/engine.rb +5 -0
- data/lib/tracebook/errors.rb +9 -0
- data/lib/tracebook/mappers/anthropic.rb +59 -0
- data/lib/tracebook/mappers/base.rb +38 -0
- data/lib/tracebook/mappers/ollama.rb +49 -0
- data/lib/tracebook/mappers/openai.rb +75 -0
- data/lib/tracebook/mappers.rb +283 -0
- data/lib/tracebook/normalized_interaction.rb +86 -0
- data/lib/tracebook/pricing/calculator.rb +39 -0
- data/lib/tracebook/pricing.rb +5 -0
- data/lib/tracebook/redaction_pipeline.rb +88 -0
- data/lib/tracebook/redactors/base.rb +29 -0
- data/lib/tracebook/redactors/card_pan.rb +15 -0
- data/lib/tracebook/redactors/email.rb +15 -0
- data/lib/tracebook/redactors/phone.rb +15 -0
- data/lib/tracebook/redactors.rb +8 -0
- data/lib/tracebook/result.rb +53 -0
- data/lib/tracebook/version.rb +3 -0
- data/lib/tracebook.rb +201 -0
- metadata +164 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "stringio"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
require "time"
|
|
7
|
+
|
|
8
|
+
module Tracebook
|
|
9
|
+
# Background job for persisting LLM interactions.
|
|
10
|
+
#
|
|
11
|
+
# Handles the complete persistence pipeline:
|
|
12
|
+
# 1. Deserialize payload
|
|
13
|
+
# 2. Apply PII redaction
|
|
14
|
+
# 3. Calculate cost based on pricing rules
|
|
15
|
+
# 4. Store interaction in database (with encryption)
|
|
16
|
+
# 5. Handle large payloads via ActiveStorage
|
|
17
|
+
# 6. Enqueue rollup job for analytics
|
|
18
|
+
#
|
|
19
|
+
# This job is enqueued by {Tracebook.record!} when `config.persist_async` is true.
|
|
20
|
+
#
|
|
21
|
+
# @example Enqueuing directly
|
|
22
|
+
# PersistInteractionJob.perform_later(
|
|
23
|
+
# provider: "openai",
|
|
24
|
+
# model: "gpt-4o",
|
|
25
|
+
# request_payload: { messages: messages },
|
|
26
|
+
# response_payload: response,
|
|
27
|
+
# input_tokens: 100,
|
|
28
|
+
# output_tokens: 50
|
|
29
|
+
# )
|
|
30
|
+
#
|
|
31
|
+
# @example Synchronous execution (testing)
|
|
32
|
+
# interaction = PersistInteractionJob.perform_now(payload)
|
|
33
|
+
#
|
|
34
|
+
# @see Tracebook.record!
|
|
35
|
+
# @see RedactionPipeline
|
|
36
|
+
# @see Pricing::Calculator
|
|
37
|
+
class PersistInteractionJob < ApplicationJob
|
|
38
|
+
# Default inline payload size threshold (64KB)
|
|
39
|
+
INLINE_THRESHOLD_BYTES = 64 * 1024
|
|
40
|
+
|
|
41
|
+
# Processes and persists an LLM interaction.
|
|
42
|
+
#
|
|
43
|
+
# @param payload [Hash] Normalized interaction attributes
|
|
44
|
+
# @return [Interaction] The persisted interaction record
|
|
45
|
+
#
|
|
46
|
+
# @raise [ActiveRecord::RecordInvalid] if interaction fails validation
|
|
47
|
+
def perform(payload)
|
|
48
|
+
normalized = from_payload(payload)
|
|
49
|
+
redacted = redaction_pipeline.call(normalized)
|
|
50
|
+
cost = Pricing::Calculator.call(
|
|
51
|
+
provider: redacted.provider,
|
|
52
|
+
model: redacted.model,
|
|
53
|
+
input_tokens: redacted.input_tokens,
|
|
54
|
+
output_tokens: redacted.output_tokens,
|
|
55
|
+
occurred_at: occurred_at(redacted)
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
interaction = persist_interaction(redacted, cost)
|
|
59
|
+
enqueue_rollup(interaction)
|
|
60
|
+
interaction
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
def from_payload(payload)
|
|
66
|
+
attributes = payload.to_h.deep_symbolize_keys
|
|
67
|
+
NormalizedInteraction.new(**attributes)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def persist_interaction(normalized, cost)
|
|
71
|
+
attributes = {
|
|
72
|
+
provider: normalized.provider,
|
|
73
|
+
model: normalized.model,
|
|
74
|
+
project: normalized.project,
|
|
75
|
+
request_text: normalized.request_text,
|
|
76
|
+
response_text: normalized.response_text,
|
|
77
|
+
input_tokens: normalized.input_tokens,
|
|
78
|
+
output_tokens: normalized.output_tokens,
|
|
79
|
+
total_tokens: total_tokens(normalized),
|
|
80
|
+
latency_ms: normalized.latency_ms,
|
|
81
|
+
status: normalized.status,
|
|
82
|
+
error_class: normalized.error_class,
|
|
83
|
+
error_message: normalized.error_message,
|
|
84
|
+
tags: normalized.tags,
|
|
85
|
+
metadata: normalized.metadata,
|
|
86
|
+
parent_id: normalized.parent_id,
|
|
87
|
+
session_id: normalized.session_id,
|
|
88
|
+
cost_input_cents: cost.input_cents,
|
|
89
|
+
cost_output_cents: cost.output_cents,
|
|
90
|
+
cost_total_cents: cost.total_cents,
|
|
91
|
+
currency: cost.currency || Tracebook.config.default_currency
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
ActiveRecord::Base.transaction do
|
|
95
|
+
Interaction.create!(attributes).tap do |interaction|
|
|
96
|
+
interaction.user = normalized.user if normalized.user
|
|
97
|
+
persist_payloads(interaction, normalized)
|
|
98
|
+
interaction.save! if interaction.changed?
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def total_tokens(normalized)
|
|
104
|
+
[ normalized.input_tokens.to_i, normalized.output_tokens.to_i ].compact.sum
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def persist_payloads(interaction, normalized)
|
|
108
|
+
store_payload(:request, interaction, normalized.request_payload)
|
|
109
|
+
store_payload(:response, interaction, normalized.response_payload)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def store_payload(type, interaction, payload)
|
|
113
|
+
return if payload.nil?
|
|
114
|
+
|
|
115
|
+
serialized = JSON.generate(payload)
|
|
116
|
+
if serialized.bytesize > inline_threshold
|
|
117
|
+
blob = create_blob(serialized, "#{type}-payload")
|
|
118
|
+
interaction.public_send("#{type}_payload_store=", "active_storage")
|
|
119
|
+
interaction.public_send("#{type}_payload_blob_id=", blob.id)
|
|
120
|
+
interaction.public_send("#{type}_payload=", nil)
|
|
121
|
+
else
|
|
122
|
+
interaction.public_send("#{type}_payload_store=", "inline")
|
|
123
|
+
interaction.public_send("#{type}_payload=", payload)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def inline_threshold
|
|
128
|
+
Tracebook.config.inline_payload_bytes
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def create_blob(contents, label)
|
|
132
|
+
ActiveStorage::Blob.create_and_upload!(
|
|
133
|
+
io: StringIO.new(contents),
|
|
134
|
+
filename: "tracebook-#{label}-#{SecureRandom.uuid}.json",
|
|
135
|
+
content_type: "application/json"
|
|
136
|
+
)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def enqueue_rollup(interaction)
|
|
140
|
+
DailyRollupsJob.perform_later(
|
|
141
|
+
date: interaction.created_at.to_date,
|
|
142
|
+
provider: interaction.provider,
|
|
143
|
+
model: interaction.model,
|
|
144
|
+
project: interaction.project
|
|
145
|
+
)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def redaction_pipeline
|
|
149
|
+
@redaction_pipeline ||= RedactionPipeline.new(config: Tracebook.config)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def occurred_at(normalized)
|
|
153
|
+
normalized.metadata && normalized.metadata["timestamp"] ? Time.parse(normalized.metadata["timestamp"].to_s) : Time.current
|
|
154
|
+
rescue ArgumentError
|
|
155
|
+
Time.current
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
TraceBook = Tracebook unless defined?(TraceBook)
|
|
File without changes
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tracebook
|
|
4
|
+
# ActiveRecord model for LLM interactions.
|
|
5
|
+
#
|
|
6
|
+
# Stores all data related to an LLM API call including request/response payloads
|
|
7
|
+
# (encrypted), usage metrics, cost, review state, and relationships.
|
|
8
|
+
#
|
|
9
|
+
# ## Encrypted Fields
|
|
10
|
+
# - `request_payload` - Full request sent to provider
|
|
11
|
+
# - `response_payload` - Full response from provider
|
|
12
|
+
# - `review_comment` - Reviewer's comments
|
|
13
|
+
#
|
|
14
|
+
# ## Enums
|
|
15
|
+
# - `status`: `:success`, `:error`, `:canceled`
|
|
16
|
+
# - `review_state`: `:pending`, `:approved`, `:flagged`, `:rejected`
|
|
17
|
+
#
|
|
18
|
+
# ## Associations
|
|
19
|
+
# - `parent` - Parent interaction for hierarchical chains
|
|
20
|
+
# - `user` - Polymorphic association to user who triggered the call
|
|
21
|
+
# - `request_payload_blob` - ActiveStorage blob for large requests
|
|
22
|
+
# - `response_payload_blob` - ActiveStorage blob for large responses
|
|
23
|
+
#
|
|
24
|
+
# @example Finding interactions
|
|
25
|
+
# Interaction.by_provider("openai")
|
|
26
|
+
# .by_model("gpt-4o")
|
|
27
|
+
# .by_review_state(:pending)
|
|
28
|
+
# .between_dates(30.days.ago, Date.current)
|
|
29
|
+
#
|
|
30
|
+
# @example Filtering with parameters
|
|
31
|
+
# Interaction.filtered(
|
|
32
|
+
# provider: "anthropic",
|
|
33
|
+
# model: "claude-3-5-sonnet",
|
|
34
|
+
# review_state: "flagged",
|
|
35
|
+
# from: "2025-01-01",
|
|
36
|
+
# to: "2025-01-31"
|
|
37
|
+
# )
|
|
38
|
+
class Interaction < ApplicationRecord
|
|
39
|
+
self.table_name = "tracebook_interactions"
|
|
40
|
+
|
|
41
|
+
# @!attribute [rw] parent
|
|
42
|
+
# @return [Tracebook::Interaction, nil] Parent interaction for hierarchical chains
|
|
43
|
+
belongs_to :parent, class_name: "Tracebook::Interaction", optional: true
|
|
44
|
+
|
|
45
|
+
# @!attribute [rw] user
|
|
46
|
+
# @return [ActiveRecord::Base, nil] Polymorphic user who triggered this interaction
|
|
47
|
+
belongs_to :user, polymorphic: true, optional: true
|
|
48
|
+
|
|
49
|
+
# @!attribute [rw] request_payload_blob
|
|
50
|
+
# @return [ActiveStorage::Blob, nil] Blob for large request payloads
|
|
51
|
+
belongs_to :request_payload_blob, class_name: "ActiveStorage::Blob", optional: true
|
|
52
|
+
|
|
53
|
+
# @!attribute [rw] response_payload_blob
|
|
54
|
+
# @return [ActiveStorage::Blob, nil] Blob for large response payloads
|
|
55
|
+
belongs_to :response_payload_blob, class_name: "ActiveStorage::Blob", optional: true
|
|
56
|
+
|
|
57
|
+
# @!attribute [rw] status
|
|
58
|
+
# @return [Symbol] Request status (:success, :error, :canceled)
|
|
59
|
+
enum :status, { success: 0, error: 1, canceled: 2 }, prefix: true
|
|
60
|
+
|
|
61
|
+
# @!attribute [rw] review_state
|
|
62
|
+
# @return [Symbol] Review state (:pending, :approved, :flagged, :rejected)
|
|
63
|
+
enum :review_state, { pending: 0, approved: 1, flagged: 2, rejected: 3 }, prefix: true
|
|
64
|
+
|
|
65
|
+
attribute :tags, :json, default: []
|
|
66
|
+
attribute :metadata, :json, default: {}
|
|
67
|
+
attribute :request_payload, :json, default: {}
|
|
68
|
+
attribute :response_payload, :json, default: {}
|
|
69
|
+
|
|
70
|
+
validates :provider, presence: true
|
|
71
|
+
validates :model, presence: true
|
|
72
|
+
|
|
73
|
+
scope :by_provider, ->(provider) { where(provider: provider) if provider.present? }
|
|
74
|
+
scope :by_model, ->(model) { where(model: model) if model.present? }
|
|
75
|
+
scope :by_project, ->(project) { where(project: project) if project.present? }
|
|
76
|
+
scope :by_status, ->(status) { where(status: status) if status.present? }
|
|
77
|
+
scope :by_review_state, ->(state) { where(review_state: state) if state.present? }
|
|
78
|
+
scope :between_dates, ->(from, to) {
|
|
79
|
+
scope = all
|
|
80
|
+
scope = scope.where("created_at >= ?", Date.parse(from.to_s).beginning_of_day) if from.present?
|
|
81
|
+
scope = scope.where("created_at <= ?", Date.parse(to.to_s).end_of_day) if to.present?
|
|
82
|
+
scope
|
|
83
|
+
}
|
|
84
|
+
scope :tagged_with, ->(tag) {
|
|
85
|
+
where("tags LIKE ?", "%#{sanitize_sql_like(tag)}%") if tag.present?
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
def self.filtered(params)
|
|
89
|
+
by_provider(params[:provider])
|
|
90
|
+
.by_model(params[:model])
|
|
91
|
+
.by_project(params[:project])
|
|
92
|
+
.by_status(params[:status])
|
|
93
|
+
.by_review_state(params[:review_state])
|
|
94
|
+
.tagged_with(params[:tag])
|
|
95
|
+
.between_dates(params[:from], params[:to])
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
TraceBook = Tracebook unless defined?(TraceBook)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tracebook
|
|
4
|
+
# Pricing rule for calculating LLM interaction costs.
|
|
5
|
+
#
|
|
6
|
+
# Defines cost per 1000 tokens for a provider/model pattern. Supports
|
|
7
|
+
# glob patterns for matching multiple models and date-based effective periods.
|
|
8
|
+
#
|
|
9
|
+
# ## Fields
|
|
10
|
+
# - `provider` - Provider name (e.g., "openai", "anthropic")
|
|
11
|
+
# - `model_glob` - Glob pattern for matching models (e.g., "gpt-4o*", "claude-3-5-*")
|
|
12
|
+
# - `input_per_1k` - Cost per 1000 input tokens
|
|
13
|
+
# - `output_per_1k` - Cost per 1000 output tokens
|
|
14
|
+
# - `currency` - Currency code (e.g., "USD")
|
|
15
|
+
# - `effective_from` - Date this pricing takes effect
|
|
16
|
+
# - `effective_to` - Optional end date for this pricing
|
|
17
|
+
#
|
|
18
|
+
# @example Creating pricing rules
|
|
19
|
+
# PricingRule.create!(
|
|
20
|
+
# provider: "openai",
|
|
21
|
+
# model_glob: "gpt-4o",
|
|
22
|
+
# input_per_1k: 2.50,
|
|
23
|
+
# output_per_1k: 10.00,
|
|
24
|
+
# currency: "USD",
|
|
25
|
+
# effective_from: Date.new(2024, 8, 6)
|
|
26
|
+
# )
|
|
27
|
+
#
|
|
28
|
+
# @example Glob patterns
|
|
29
|
+
# # Exact match
|
|
30
|
+
# model_glob: "gpt-4o"
|
|
31
|
+
#
|
|
32
|
+
# # All GPT-4o variants
|
|
33
|
+
# model_glob: "gpt-4o*"
|
|
34
|
+
#
|
|
35
|
+
# # All Claude 3.5 models
|
|
36
|
+
# model_glob: "claude-3-5-*"
|
|
37
|
+
#
|
|
38
|
+
# # Fallback for any model
|
|
39
|
+
# model_glob: "*"
|
|
40
|
+
#
|
|
41
|
+
# @example Finding applicable rule
|
|
42
|
+
# rule = PricingRule
|
|
43
|
+
# .where(provider: "openai")
|
|
44
|
+
# .where("? >= effective_from", Date.current)
|
|
45
|
+
# .where("effective_to IS NULL OR ? < effective_to", Date.current)
|
|
46
|
+
# .find { |r| r.matches_model?("gpt-4o-mini") }
|
|
47
|
+
class PricingRule < ApplicationRecord
|
|
48
|
+
self.table_name = "tracebook_pricing_rules"
|
|
49
|
+
|
|
50
|
+
validates :provider, presence: true
|
|
51
|
+
validates :model_glob, presence: true
|
|
52
|
+
validates :effective_from, presence: true
|
|
53
|
+
|
|
54
|
+
# Returns true if this rule is active on the given date.
|
|
55
|
+
#
|
|
56
|
+
# @param date [Date] The date to check
|
|
57
|
+
# @return [Boolean] true if rule is effective on this date
|
|
58
|
+
#
|
|
59
|
+
# @example
|
|
60
|
+
# rule.active_on?(Date.current) # => true
|
|
61
|
+
# rule.active_on?(Date.new(2020, 1, 1)) # => false
|
|
62
|
+
def active_on?(date)
|
|
63
|
+
date >= effective_from && (effective_to.nil? || date < effective_to)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Returns true if this rule's glob pattern matches the given model.
|
|
67
|
+
#
|
|
68
|
+
# Uses case-insensitive file glob matching.
|
|
69
|
+
#
|
|
70
|
+
# @param model [String] Model identifier to match
|
|
71
|
+
# @return [Boolean] true if pattern matches
|
|
72
|
+
#
|
|
73
|
+
# @example
|
|
74
|
+
# rule = PricingRule.new(model_glob: "gpt-4o*")
|
|
75
|
+
# rule.matches_model?("gpt-4o") # => true
|
|
76
|
+
# rule.matches_model?("gpt-4o-mini") # => true
|
|
77
|
+
# rule.matches_model?("claude-3-5-sonnet") # => false
|
|
78
|
+
def matches_model?(model)
|
|
79
|
+
File.fnmatch?(model_glob, model, File::FNM_CASEFOLD)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
TraceBook = Tracebook unless defined?(TraceBook)
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tracebook
|
|
4
|
+
# Rule for redacting PII from interaction payloads.
|
|
5
|
+
#
|
|
6
|
+
# Defines a regex pattern to detect and replace sensitive data before
|
|
7
|
+
# persistence. Runs on request, response, or both payloads.
|
|
8
|
+
#
|
|
9
|
+
# ## Fields
|
|
10
|
+
# - `name` - Human-readable name for this rule
|
|
11
|
+
# - `pattern` - Regular expression pattern to match
|
|
12
|
+
# - `replacement` - Replacement string (e.g., "[REDACTED]", "[EMAIL]")
|
|
13
|
+
# - `applies_to` - Where to apply: `:request`, `:response`, `:both`, `:metadata`
|
|
14
|
+
# - `enabled` - Whether this rule is active
|
|
15
|
+
#
|
|
16
|
+
# ## Built-in Rules
|
|
17
|
+
# TraceBook includes default redactors for:
|
|
18
|
+
# - Email addresses
|
|
19
|
+
# - Phone numbers (US format)
|
|
20
|
+
# - Credit card PANs
|
|
21
|
+
#
|
|
22
|
+
# @example Creating a custom redaction rule
|
|
23
|
+
# RedactionRule.create!(
|
|
24
|
+
# name: "API Keys",
|
|
25
|
+
# pattern: 'api_key["\s]*[:=]["\s]*\K[\w-]+',
|
|
26
|
+
# replacement: "[API_KEY]",
|
|
27
|
+
# applies_to: :both,
|
|
28
|
+
# enabled: true
|
|
29
|
+
# )
|
|
30
|
+
#
|
|
31
|
+
# @example Email redaction
|
|
32
|
+
# RedactionRule.create!(
|
|
33
|
+
# name: "Email Addresses",
|
|
34
|
+
# pattern: '\b[\w\.-]+@[\w\.-]+\.\w{2,}\b',
|
|
35
|
+
# replacement: "[EMAIL]",
|
|
36
|
+
# applies_to: :both,
|
|
37
|
+
# enabled: true
|
|
38
|
+
# )
|
|
39
|
+
#
|
|
40
|
+
# @example SSN redaction
|
|
41
|
+
# RedactionRule.create!(
|
|
42
|
+
# name: "Social Security Numbers",
|
|
43
|
+
# pattern: '\b\d{3}-\d{2}-\d{4}\b',
|
|
44
|
+
# replacement: "[SSN]",
|
|
45
|
+
# applies_to: :both,
|
|
46
|
+
# enabled: true
|
|
47
|
+
# )
|
|
48
|
+
#
|
|
49
|
+
# @see Redactors::Email
|
|
50
|
+
# @see Redactors::Phone
|
|
51
|
+
# @see Redactors::CardPAN
|
|
52
|
+
class RedactionRule < ApplicationRecord
|
|
53
|
+
self.table_name = "tracebook_redaction_rules"
|
|
54
|
+
|
|
55
|
+
# @!attribute [rw] applies_to
|
|
56
|
+
# @return [Symbol] Where to apply redaction (:request, :response, :both, :metadata)
|
|
57
|
+
enum :applies_to, { request: 0, response: 1, both: 2, metadata: 3 }
|
|
58
|
+
|
|
59
|
+
validates :name, presence: true
|
|
60
|
+
validates :pattern, presence: true
|
|
61
|
+
validates :replacement, presence: true
|
|
62
|
+
|
|
63
|
+
# Returns the compiled regex pattern.
|
|
64
|
+
#
|
|
65
|
+
# Caches the compiled pattern for performance. If pattern is invalid,
|
|
66
|
+
# falls back to escaped literal match.
|
|
67
|
+
#
|
|
68
|
+
# @return [Regexp] Compiled regular expression with MULTILINE flag
|
|
69
|
+
#
|
|
70
|
+
# @example
|
|
71
|
+
# rule = RedactionRule.new(pattern: '\b\d{3}-\d{2}-\d{4}\b')
|
|
72
|
+
# rule.compiled_pattern.match("123-45-6789") # => MatchData
|
|
73
|
+
def compiled_pattern
|
|
74
|
+
@compiled_pattern ||= Regexp.new(pattern, Regexp::MULTILINE)
|
|
75
|
+
rescue RegexpError
|
|
76
|
+
Regexp.new(Regexp.escape(pattern.to_s))
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
TraceBook = Tracebook unless defined?(TraceBook)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tracebook
|
|
4
|
+
# Daily aggregated metrics for LLM interactions.
|
|
5
|
+
#
|
|
6
|
+
# Stores summarized statistics by date/provider/model/project for analytics
|
|
7
|
+
# and cost reporting. Updated nightly by {DailyRollupsJob}.
|
|
8
|
+
#
|
|
9
|
+
# ## Fields
|
|
10
|
+
# - `date` - Date of rollup (part of composite PK)
|
|
11
|
+
# - `provider` - Provider name (part of composite PK)
|
|
12
|
+
# - `model` - Model identifier (part of composite PK)
|
|
13
|
+
# - `project` - Project name (part of composite PK, nullable)
|
|
14
|
+
# - `interactions_count` - Total number of interactions
|
|
15
|
+
# - `success_count` - Number of successful interactions
|
|
16
|
+
# - `error_count` - Number of failed interactions
|
|
17
|
+
# - `input_tokens_sum` - Sum of input tokens
|
|
18
|
+
# - `output_tokens_sum` - Sum of output tokens
|
|
19
|
+
# - `cost_cents_sum` - Total cost in cents
|
|
20
|
+
#
|
|
21
|
+
# ## Primary Key
|
|
22
|
+
# Composite PK: `(date, provider, model, project)`
|
|
23
|
+
#
|
|
24
|
+
# @example Querying daily metrics
|
|
25
|
+
# RollupDaily
|
|
26
|
+
# .where(provider: "openai", date: Date.current)
|
|
27
|
+
# .sum(:cost_cents_sum) / 100.0 # Total cost in dollars
|
|
28
|
+
#
|
|
29
|
+
# @example Finding top models by cost
|
|
30
|
+
# RollupDaily
|
|
31
|
+
# .where(date: 30.days.ago..Date.current)
|
|
32
|
+
# .group(:provider, :model)
|
|
33
|
+
# .sum(:cost_cents_sum)
|
|
34
|
+
# .sort_by { |_, cents| -cents }
|
|
35
|
+
# .first(10)
|
|
36
|
+
#
|
|
37
|
+
# @example Error rate for a model
|
|
38
|
+
# rollup = RollupDaily.find_by(date: Date.current, provider: "openai", model: "gpt-4o")
|
|
39
|
+
# error_rate = rollup.error_count.to_f / rollup.interactions_count
|
|
40
|
+
#
|
|
41
|
+
# @see DailyRollupsJob
|
|
42
|
+
class RollupDaily < ApplicationRecord
|
|
43
|
+
self.table_name = "tracebook_rollups_dailies"
|
|
44
|
+
|
|
45
|
+
# @!attribute [rw] interactions_count
|
|
46
|
+
# @return [Integer] Total number of interactions (default: 0)
|
|
47
|
+
attribute :interactions_count, :integer, default: 0
|
|
48
|
+
|
|
49
|
+
# @!attribute [rw] success_count
|
|
50
|
+
# @return [Integer] Number of successful interactions (default: 0)
|
|
51
|
+
attribute :success_count, :integer, default: 0
|
|
52
|
+
|
|
53
|
+
# @!attribute [rw] error_count
|
|
54
|
+
# @return [Integer] Number of failed interactions (default: 0)
|
|
55
|
+
attribute :error_count, :integer, default: 0
|
|
56
|
+
|
|
57
|
+
# @!attribute [rw] input_tokens_sum
|
|
58
|
+
# @return [Integer] Sum of input tokens across all interactions (default: 0)
|
|
59
|
+
attribute :input_tokens_sum, :integer, default: 0
|
|
60
|
+
|
|
61
|
+
# @!attribute [rw] output_tokens_sum
|
|
62
|
+
# @return [Integer] Sum of output tokens across all interactions (default: 0)
|
|
63
|
+
attribute :output_tokens_sum, :integer, default: 0
|
|
64
|
+
|
|
65
|
+
# @!attribute [rw] cost_cents_sum
|
|
66
|
+
# @return [Integer] Total cost in cents (default: 0)
|
|
67
|
+
attribute :cost_cents_sum, :integer, default: 0
|
|
68
|
+
|
|
69
|
+
validates :date, presence: true
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
TraceBook = Tracebook unless defined?(TraceBook)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>Tracebook</title>
|
|
5
|
+
<%= csrf_meta_tags %>
|
|
6
|
+
<%= csp_meta_tag %>
|
|
7
|
+
|
|
8
|
+
<%= yield :head %>
|
|
9
|
+
|
|
10
|
+
<%= stylesheet_link_tag "tracebook/application", media: "all", "data-turbo-track": "reload" %>
|
|
11
|
+
<%= javascript_include_tag "tracebook/application", defer: true, "data-turbo-track": "reload" %>
|
|
12
|
+
</head>
|
|
13
|
+
<body>
|
|
14
|
+
|
|
15
|
+
<%= yield %>
|
|
16
|
+
|
|
17
|
+
</body>
|
|
18
|
+
</html>
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
<div class="tb-container">
|
|
2
|
+
<header class="tb-header">
|
|
3
|
+
<h1>TraceBook Interactions</h1>
|
|
4
|
+
<div class="tb-kpis">
|
|
5
|
+
<div class="tb-kpi">
|
|
6
|
+
<span>Total</span>
|
|
7
|
+
<strong><%= @kpis[:total] %></strong>
|
|
8
|
+
</div>
|
|
9
|
+
<div class="tb-kpi">
|
|
10
|
+
<span>Success</span>
|
|
11
|
+
<strong><%= @kpis[:success] %></strong>
|
|
12
|
+
</div>
|
|
13
|
+
<div class="tb-kpi">
|
|
14
|
+
<span>Total Tokens</span>
|
|
15
|
+
<strong><%= @kpis[:input_tokens] + @kpis[:output_tokens] %></strong>
|
|
16
|
+
</div>
|
|
17
|
+
<div class="tb-kpi">
|
|
18
|
+
<span>Cost (¢)</span>
|
|
19
|
+
<strong><%= @kpis[:cost_cents] %></strong>
|
|
20
|
+
</div>
|
|
21
|
+
</div>
|
|
22
|
+
</header>
|
|
23
|
+
|
|
24
|
+
<section class="tb-filters">
|
|
25
|
+
<%= form_with url: interactions_path, method: :get, scope: :filters, data: { turbo_frame: "interactions_table" }, class: "tb-filter-form" do |form| %>
|
|
26
|
+
<div class="tb-filter-grid">
|
|
27
|
+
<div>
|
|
28
|
+
<%= form.label :provider %>
|
|
29
|
+
<%= form.select :provider, options_for_select(@providers, @filters[:provider]), include_blank: "All" %>
|
|
30
|
+
</div>
|
|
31
|
+
<div>
|
|
32
|
+
<%= form.label :model %>
|
|
33
|
+
<%= form.select :model, options_for_select(@models, @filters[:model]), include_blank: "All" %>
|
|
34
|
+
</div>
|
|
35
|
+
<div>
|
|
36
|
+
<%= form.label :project %>
|
|
37
|
+
<%= form.select :project, options_for_select(@projects, @filters[:project]), include_blank: "All" %>
|
|
38
|
+
</div>
|
|
39
|
+
<div>
|
|
40
|
+
<%= form.label :status %>
|
|
41
|
+
<%= form.select :status, options_for_select(TraceBook::Interaction.statuses.keys, @filters[:status]), include_blank: "All" %>
|
|
42
|
+
</div>
|
|
43
|
+
<div>
|
|
44
|
+
<%= form.label :review_state %>
|
|
45
|
+
<%= form.select :review_state, options_for_select(TraceBook::Interaction.review_states.keys, @filters[:review_state]), include_blank: "All" %>
|
|
46
|
+
</div>
|
|
47
|
+
<div>
|
|
48
|
+
<%= form.label :tag %>
|
|
49
|
+
<%= form.text_field :tag, value: @filters[:tag] %>
|
|
50
|
+
</div>
|
|
51
|
+
<div>
|
|
52
|
+
<%= form.label :from, "From (date)" %>
|
|
53
|
+
<%= form.date_field :from, value: @filters[:from] %>
|
|
54
|
+
</div>
|
|
55
|
+
<div>
|
|
56
|
+
<%= form.label :to, "To (date)" %>
|
|
57
|
+
<%= form.date_field :to, value: @filters[:to] %>
|
|
58
|
+
</div>
|
|
59
|
+
</div>
|
|
60
|
+
<div class="tb-filter-actions">
|
|
61
|
+
<%= form.submit "Apply" %>
|
|
62
|
+
<%= link_to "Reset", interactions_path, class: "tb-link" %>
|
|
63
|
+
</div>
|
|
64
|
+
<% end %>
|
|
65
|
+
</section>
|
|
66
|
+
|
|
67
|
+
<section class="tb-table-wrapper">
|
|
68
|
+
<%= form_with url: bulk_review_interactions_path, method: :post, data: { controller: "keyboard" }, html: { tabindex: 0 } do %>
|
|
69
|
+
<%= hidden_field_tag :review_state, "approved", data: { keyboard_target: "reviewState" } %>
|
|
70
|
+
<div class="tb-table-actions">
|
|
71
|
+
<button type="submit">Bulk Approve</button>
|
|
72
|
+
<button type="submit" formaction="<%= exports_path(format: :csv, filters: @filters.to_h) %>" formmethod="post">Export CSV</button>
|
|
73
|
+
</div>
|
|
74
|
+
|
|
75
|
+
<turbo-frame id="interactions_table">
|
|
76
|
+
<table class="tb-table" data-keyboard-target="table">
|
|
77
|
+
<thead>
|
|
78
|
+
<tr>
|
|
79
|
+
<th><input type="checkbox" data-keyboard-target="toggleAll"></th>
|
|
80
|
+
<th>Timestamp</th>
|
|
81
|
+
<th>Provider</th>
|
|
82
|
+
<th>Model</th>
|
|
83
|
+
<th>Status</th>
|
|
84
|
+
<th>Tokens</th>
|
|
85
|
+
<th>Cost (¢)</th>
|
|
86
|
+
</tr>
|
|
87
|
+
</thead>
|
|
88
|
+
<tbody>
|
|
89
|
+
<% @interactions.each do |interaction| %>
|
|
90
|
+
<tr data-controller="row" data-keyboard-target="row">
|
|
91
|
+
<td><%= check_box_tag "interaction_ids[]", interaction.id, false, data: { keyboard_target: "checkbox" } %></td>
|
|
92
|
+
<td><%= link_to interaction.created_at.strftime("%Y-%m-%d %H:%M"), interaction_path(interaction) %></td>
|
|
93
|
+
<td><%= interaction.provider %></td>
|
|
94
|
+
<td><%= interaction.model %></td>
|
|
95
|
+
<td><%= interaction.status %></td>
|
|
96
|
+
<td><%= interaction.total_tokens %></td>
|
|
97
|
+
<td><%= interaction.cost_total_cents %></td>
|
|
98
|
+
</tr>
|
|
99
|
+
<% end %>
|
|
100
|
+
</tbody>
|
|
101
|
+
</table>
|
|
102
|
+
</turbo-frame>
|
|
103
|
+
<% end %>
|
|
104
|
+
</section>
|
|
105
|
+
</div>
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
<div class="tb-container">
|
|
2
|
+
<nav class="tb-breadcrumbs">
|
|
3
|
+
<%= link_to "← Back", interactions_path %>
|
|
4
|
+
</nav>
|
|
5
|
+
|
|
6
|
+
<header class="tb-header">
|
|
7
|
+
<h1>Interaction <%= @interaction.id %></h1>
|
|
8
|
+
<div class="tb-meta-grid">
|
|
9
|
+
<div><span>Provider</span><strong><%= @interaction.provider %></strong></div>
|
|
10
|
+
<div><span>Model</span><strong><%= @interaction.model %></strong></div>
|
|
11
|
+
<div><span>Status</span><strong><%= @interaction.status %></strong></div>
|
|
12
|
+
<div><span>Review</span><strong><%= @interaction.review_state %></strong></div>
|
|
13
|
+
<div><span>Tokens</span><strong><%= @interaction.total_tokens %></strong></div>
|
|
14
|
+
<div><span>Cost (¢)</span><strong><%= @interaction.cost_total_cents %></strong></div>
|
|
15
|
+
</div>
|
|
16
|
+
</header>
|
|
17
|
+
|
|
18
|
+
<% request_payload = payload_for(@interaction, :request) %>
|
|
19
|
+
<section class="tb-section" data-controller="json-viewer" data-json-viewer-collapsed-value="true">
|
|
20
|
+
<h2>Request</h2>
|
|
21
|
+
<pre data-json-viewer-target="content"><%= formatted_payload(request_payload, @interaction.request_text) %></pre>
|
|
22
|
+
</section>
|
|
23
|
+
|
|
24
|
+
<% response_payload = payload_for(@interaction, :response) %>
|
|
25
|
+
<section class="tb-section" data-controller="json-viewer">
|
|
26
|
+
<h2>Response</h2>
|
|
27
|
+
<pre data-json-viewer-target="content"><%= formatted_payload(response_payload, @interaction.response_text) %></pre>
|
|
28
|
+
</section>
|
|
29
|
+
|
|
30
|
+
<section class="tb-section">
|
|
31
|
+
<h2>Metadata</h2>
|
|
32
|
+
<pre><%= formatted_payload(@interaction.metadata, "{}") %></pre>
|
|
33
|
+
</section>
|
|
34
|
+
|
|
35
|
+
<section class="tb-section">
|
|
36
|
+
<%= form_with url: review_interaction_path(@interaction), method: :post, class: "tb-review-form" do %>
|
|
37
|
+
<div>
|
|
38
|
+
<%= label_tag :review_state, "Review State" %>
|
|
39
|
+
<%= select_tag :review_state, options_for_select(TraceBook::Interaction.review_states.keys, @interaction.review_state) %>
|
|
40
|
+
</div>
|
|
41
|
+
<%= submit_tag "Update Review" %>
|
|
42
|
+
<% end %>
|
|
43
|
+
</section>
|
|
44
|
+
</div>
|