tracebook 0.1.1 → 1.0.0

data/app/jobs/tracebook/daily_rollups_job.rb

@@ -1,100 +0,0 @@
-# frozen_string_literal: true
-
-module Tracebook
-  # Background job for aggregating daily metrics.
-  #
-  # Summarizes interactions by date/provider/model/project into {RollupDaily}
-  # records for analytics and cost reporting. Should be scheduled nightly for
-  # each active provider/model combination.
-  #
-  # ## Aggregated Metrics
-  # - Total interaction count
-  # - Success/error counts
-  # - Input/output token sums
-  # - Total cost in cents
-  #
-  # @example Schedule with Sidekiq Cron
-  #   Sidekiq::Cron::Job.create(
-  #     name: "TraceBook OpenAI rollups",
-  #     cron: "0 2 * * *",
-  #     class: "Tracebook::DailyRollupsJob",
-  #     kwargs: { date: Date.yesterday, provider: "openai", model: nil, project: nil }
-  #   )
-  #
-  # @example Run manually for specific date/model
-  #   DailyRollupsJob.perform_now(
-  #     date: Date.yesterday,
-  #     provider: "openai",
-  #     model: "gpt-4o",
-  #     project: "support"
-  #   )
-  #
-  # @see RollupDaily
-  class DailyRollupsJob < ApplicationJob
-    # Aggregates metrics for a specific date/provider/model/project.
-    #
-    # Creates or updates a {RollupDaily} record with summarized statistics.
-    #
-    # @param date [Date] Date to aggregate (usually Date.yesterday)
-    # @param provider [String] Provider name (e.g., "openai")
-    # @param model [String, nil] Model identifier (nil for all models)
-    # @param project [String, nil] Project name (nil for all projects)
-    #
-    # @return [void]
-    #
-    # @raise [ActiveRecord::RecordInvalid] if rollup fails validation
-    def perform(date:, provider:, model:, project: nil)
-      scope = Interaction.where(provider: provider, model: model)
-      scope = scope.where(project: project) if project
-      scope = scope.where(created_at: date.beginning_of_day..date.end_of_day)
-
-      counts = normalize_status_counts(scope.group(:status).count)
-      tokens = scope.pluck(Arel.sql("COALESCE(input_tokens, 0)"), Arel.sql("COALESCE(output_tokens, 0)"))
-      costs = scope.pluck(Arel.sql("COALESCE(cost_total_cents, 0)"))
-
-      input_sum = tokens.sum { |(input, _)| input.to_i }
-      output_sum = tokens.sum { |(_, output)| output.to_i }
-      cost_sum = costs.sum(&:to_i)
-
-      rollup = RollupDaily.find_or_initialize_by(date: date, provider: provider, model: model, project: project)
-      rollup.interactions_count = scope.count
-      rollup.success_count = counts.fetch("success", 0)
-      rollup.error_count = counts.fetch("error", 0)
-      rollup.input_tokens_sum = input_sum
-      rollup.output_tokens_sum = output_sum
-      rollup.cost_cents_sum = cost_sum
-      rollup.currency = determine_currency(scope) || rollup.currency
-      rollup.save!
-    end
-
-    private
-
-    def normalize_status_counts(counts)
-      counts.each_with_object(Hash.new(0)) do |(raw_key, count), normalized|
-        status_name = status_name_for(raw_key)
-        next unless status_name
-
-        normalized[status_name] += count
-      end
-    end
-
-    def status_name_for(raw_key)
-      key_string = raw_key.to_s
-      return key_string if Interaction.statuses.key?(key_string)
-
-      integer_string?(key_string) ? Interaction.statuses.invert[key_string.to_i] : nil
-    end
-
-    def integer_string?(value)
-      value.match?(/\A-?\d+\z/)
-    end
-
-    def determine_currency(scope)
-      scope.pick(:currency)
-    rescue NoMethodError
-      scope.first&.currency
-    end
-  end
-end
-
-TraceBook = Tracebook unless defined?(TraceBook)

data/app/jobs/tracebook/export_job.rb

@@ -1,162 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-
-module Tracebook
-  # Background job for exporting interactions to CSV or NDJSON.
-  #
-  # Streams filtered interactions to an export file stored in ActiveStorage.
-  # Supports filtering by provider, model, project, date range, tags, etc.
-  #
-  # ## Supported Formats
-  # - **CSV** - Comma-separated values with headers
-  # - **NDJSON** - Newline-delimited JSON (one interaction per line)
-  #
-  # ## Exported Fields
-  # - timestamp, project, provider, model, status
-  # - input_tokens, output_tokens, cost_total_cents
-  # - tags (pipe-separated)
-  # - request_payload, response_payload (JSON)
-  # - metadata
-  #
-  # @example Enqueue export job
-  #   blob = ExportJob.perform_now(
-  #     format: :csv,
-  #     filters: {
-  #       provider: "openai",
-  #       from: 30.days.ago,
-  #       to: Date.current
-  #     }
-  #   )
-  #   download_url = Rails.application.routes.url_helpers.rails_blob_url(blob)
-  #
-  # @example Export specific project
-  #   ExportJob.perform_later(
-  #     format: :ndjson,
-  #     filters: { project: "support", review_state: "approved" }
-  #   )
-  #
-  # @see Interaction.filtered
-  class ExportJob < ApplicationJob
-    # Exports filtered interactions to specified format.
-    #
-    # @param format [Symbol, String] Export format (:csv or :ndjson)
-    # @param filters [Hash] Filters to apply (see {Interaction.filtered})
-    #
-    # @option filters [String] :provider Provider name
-    # @option filters [String] :model Model identifier
-    # @option filters [String] :project Project name
-    # @option filters [Symbol, String] :status Status filter
-    # @option filters [Symbol, String] :review_state Review state filter
-    # @option filters [String] :tag Tag to filter by
-    # @option filters [Date, String] :from Start date
-    # @option filters [Date, String] :to End date
-    #
-    # @return [ActiveStorage::Blob] The created export blob
-    #
-    # @raise [ArgumentError] if format is not supported
-    def perform(format:, filters: {})
-      interactions = Interaction.filtered(filters).order(:created_at)
-      data = export_as(interactions, format.to_sym)
-      create_blob(data, format)
-    end
-
-    private
-
-    def export_as(interactions, format)
-      case format
-      when :csv
-        csv_for(interactions)
-      when :ndjson
-        ndjson_for(interactions)
-      else
-        raise ArgumentError, "Unsupported export format: #{format}"
-      end
-    end
-
-    def csv_for(interactions)
-      CSV.generate do |csv|
-        csv << csv_headers
-        interactions.find_each do |interaction|
-          csv << serialize_interaction(interaction).values_at(*csv_headers)
-        end
-      end
-    end
-
-    def ndjson_for(interactions)
-      Enumerator.new do |yielder|
-        interactions.find_each do |interaction|
-          yielder << serialize_interaction(interaction).to_json
-        end
-      end.to_a.join("\n")
-    end
-
-    def serialize_interaction(interaction)
-      payload = {
-        "timestamp" => interaction.created_at.iso8601,
-        "project" => interaction.project,
-        "provider" => interaction.provider,
-        "model" => interaction.model,
-        "status" => interaction.status,
-        "input_tokens" => interaction.input_tokens,
-        "output_tokens" => interaction.output_tokens,
-        "cost_total_cents" => interaction.cost_total_cents,
-        "tags" => Array(interaction.tags).join("|"),
-        "request_payload" => load_payload(interaction, :request),
-        "response_payload" => load_payload(interaction, :response)
-      }
-
-      payload.merge("metadata" => interaction.metadata)
-    end
-
-    def load_payload(interaction, type)
-      store = interaction.public_send("#{type}_payload_store")
-      if store == "active_storage"
-        blob = interaction.public_send("#{type}_payload_blob")
-        return JSON.parse(blob.download) if blob
-      end
-
-      interaction.public_send("#{type}_payload")
-    rescue JSON::ParserError
-      interaction.public_send("#{type}_payload")
-    end
-
-    def create_blob(data, format)
-      ActiveStorage::Blob.create_and_upload!(
-        io: StringIO.new(data),
-        filename: "tracebook-export-#{Time.current.to_i}.#{format}",
-        content_type: content_type_for(format)
-      )
-    end
-
-    def content_type_for(format)
-      case format.to_sym
-      when :csv
-        "text/csv"
-      when :ndjson
-        "application/x-ndjson"
-      else
-        "application/octet-stream"
-      end
-    end
-
-    def csv_headers
-      @csv_headers ||= [
-        "timestamp",
-        "project",
-        "provider",
-        "model",
-        "status",
-        "input_tokens",
-        "output_tokens",
-        "cost_total_cents",
-        "tags",
-        "request_payload",
-        "response_payload",
-        "metadata"
-      ]
-    end
-  end
-end
-
-TraceBook = Tracebook unless defined?(TraceBook)

data/app/jobs/tracebook/persist_interaction_job.rb

@@ -1,160 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "stringio"
-require "securerandom"
-require "time"
-
-module Tracebook
-  # Background job for persisting LLM interactions.
-  #
-  # Handles the complete persistence pipeline:
-  # 1. Deserialize payload
-  # 2. Apply PII redaction
-  # 3. Calculate cost based on pricing rules
-  # 4. Store interaction in database (with encryption)
-  # 5. Handle large payloads via ActiveStorage
-  # 6. Enqueue rollup job for analytics
-  #
-  # This job is enqueued by {Tracebook.record!} when `config.persist_async` is true.
-  #
-  # @example Enqueuing directly
-  #   PersistInteractionJob.perform_later(
-  #     provider: "openai",
-  #     model: "gpt-4o",
-  #     request_payload: { messages: messages },
-  #     response_payload: response,
-  #     input_tokens: 100,
-  #     output_tokens: 50
-  #   )
-  #
-  # @example Synchronous execution (testing)
-  #   interaction = PersistInteractionJob.perform_now(payload)
-  #
-  # @see Tracebook.record!
-  # @see RedactionPipeline
-  # @see Pricing::Calculator
-  class PersistInteractionJob < ApplicationJob
-    # Default inline payload size threshold (64KB)
-    INLINE_THRESHOLD_BYTES = 64 * 1024
-
-    # Processes and persists an LLM interaction.
-    #
-    # @param payload [Hash] Normalized interaction attributes
-    # @return [Interaction] The persisted interaction record
-    #
-    # @raise [ActiveRecord::RecordInvalid] if interaction fails validation
-    def perform(payload)
-      normalized = from_payload(payload)
-      redacted = redaction_pipeline.call(normalized)
-      cost = Pricing::Calculator.call(
-        provider: redacted.provider,
-        model: redacted.model,
-        input_tokens: redacted.input_tokens,
-        output_tokens: redacted.output_tokens,
-        occurred_at: occurred_at(redacted)
-      )
-
-      interaction = persist_interaction(redacted, cost)
-      enqueue_rollup(interaction)
-      interaction
-    end
-
-    private
-
-    def from_payload(payload)
-      attributes = payload.to_h.deep_symbolize_keys
-      NormalizedInteraction.new(**attributes)
-    end
-
-    def persist_interaction(normalized, cost)
-      attributes = {
-        provider: normalized.provider,
-        model: normalized.model,
-        project: normalized.project,
-        request_text: normalized.request_text,
-        response_text: normalized.response_text,
-        input_tokens: normalized.input_tokens,
-        output_tokens: normalized.output_tokens,
-        total_tokens: total_tokens(normalized),
-        latency_ms: normalized.latency_ms,
-        status: normalized.status,
-        error_class: normalized.error_class,
-        error_message: normalized.error_message,
-        tags: normalized.tags,
-        metadata: normalized.metadata,
-        parent_id: normalized.parent_id,
-        session_id: normalized.session_id,
-        cost_input_cents: cost.input_cents,
-        cost_output_cents: cost.output_cents,
-        cost_total_cents: cost.total_cents,
-        currency: cost.currency || Tracebook.config.default_currency
-      }
-
-      ActiveRecord::Base.transaction do
-        Interaction.create!(attributes).tap do |interaction|
-          interaction.user = normalized.user if normalized.user
-          persist_payloads(interaction, normalized)
-          interaction.save! if interaction.changed?
-        end
-      end
-    end
-
-    def total_tokens(normalized)
-      [ normalized.input_tokens.to_i, normalized.output_tokens.to_i ].compact.sum
-    end
-
-    def persist_payloads(interaction, normalized)
-      store_payload(:request, interaction, normalized.request_payload)
-      store_payload(:response, interaction, normalized.response_payload)
-    end
-
-    def store_payload(type, interaction, payload)
-      return if payload.nil?
-
-      serialized = JSON.generate(payload)
-      if serialized.bytesize > inline_threshold
-        blob = create_blob(serialized, "#{type}-payload")
-        interaction.public_send("#{type}_payload_store=", "active_storage")
-        interaction.public_send("#{type}_payload_blob_id=", blob.id)
-        interaction.public_send("#{type}_payload=", nil)
-      else
-        interaction.public_send("#{type}_payload_store=", "inline")
-        interaction.public_send("#{type}_payload=", payload)
-      end
-    end
-
-    def inline_threshold
-      Tracebook.config.inline_payload_bytes
-    end
-
-    def create_blob(contents, label)
-      ActiveStorage::Blob.create_and_upload!(
-        io: StringIO.new(contents),
-        filename: "tracebook-#{label}-#{SecureRandom.uuid}.json",
-        content_type: "application/json"
-      )
-    end
-
-    def enqueue_rollup(interaction)
-      DailyRollupsJob.perform_later(
-        date: interaction.created_at.to_date,
-        provider: interaction.provider,
-        model: interaction.model,
-        project: interaction.project
-      )
-    end
-
-    def redaction_pipeline
-      @redaction_pipeline ||= RedactionPipeline.new(config: Tracebook.config)
-    end
-
-    def occurred_at(normalized)
-      normalized.metadata && normalized.metadata["timestamp"] ? Time.parse(normalized.metadata["timestamp"].to_s) : Time.current
-    rescue ArgumentError
-      Time.current
-    end
-  end
-end
-
-TraceBook = Tracebook unless defined?(TraceBook)

data/app/mailers/tracebook/application_mailer.rb

@@ -1,6 +0,0 @@
-module Tracebook
-  class ApplicationMailer < ActionMailer::Base
-    default from: "from@example.com"
-    layout "mailer"
-  end
-end

data/app/models/tracebook/interaction.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-module Tracebook
-  # ActiveRecord model for LLM interactions.
-  #
-  # Stores all data related to an LLM API call including request/response payloads,
-  # usage metrics, cost, review state, and relationships.
-  #
-  # ## Payload Fields
-  # - `request_payload` - Full request sent to provider (JSON)
-  # - `response_payload` - Full response from provider (JSON)
-  # - `review_comment` - Reviewer's comments
-  #
-  # Note: Encryption is optional. See README for instructions on enabling
-  # ActiveRecord::Encryption for these fields.
-  #
-  # ## Enums
-  # - `status`: `:success`, `:error`, `:canceled`
-  # - `review_state`: `:pending`, `:approved`, `:flagged`, `:rejected`
-  #
-  # ## Associations
-  # - `parent` - Parent interaction for hierarchical chains
-  # - `user` - Polymorphic association to user who triggered the call
-  # - `request_payload_blob` - ActiveStorage blob for large requests
-  # - `response_payload_blob` - ActiveStorage blob for large responses
-  #
-  # @example Finding interactions
-  #   Interaction.by_provider("openai")
-  #     .by_model("gpt-4o")
-  #     .by_review_state(:pending)
-  #     .between_dates(30.days.ago, Date.current)
-  #
-  # @example Filtering with parameters
-  #   Interaction.filtered(
-  #     provider: "anthropic",
-  #     model: "claude-3-5-sonnet",
-  #     review_state: "flagged",
-  #     from: "2025-01-01",
-  #     to: "2025-01-31"
-  #   )
-  class Interaction < ApplicationRecord
-    self.table_name = "tracebook_interactions"
-
-    # @!attribute [rw] parent
-    #   @return [Tracebook::Interaction, nil] Parent interaction for hierarchical chains
-    belongs_to :parent, class_name: "Tracebook::Interaction", optional: true
-
-    # @!attribute [rw] user
-    #   @return [ActiveRecord::Base, nil] Polymorphic user who triggered this interaction
-    belongs_to :user, polymorphic: true, optional: true
-
-    # @!attribute [rw] request_payload_blob
-    #   @return [ActiveStorage::Blob, nil] Blob for large request payloads
-    belongs_to :request_payload_blob, class_name: "ActiveStorage::Blob", optional: true
-
-    # @!attribute [rw] response_payload_blob
-    #   @return [ActiveStorage::Blob, nil] Blob for large response payloads
-    belongs_to :response_payload_blob, class_name: "ActiveStorage::Blob", optional: true
-
-    # @!attribute [rw] status
-    #   @return [Symbol] Request status (:success, :error, :canceled)
-    enum :status, { success: 0, error: 1, canceled: 2 }, prefix: true
-
-    # @!attribute [rw] review_state
-    #   @return [Symbol] Review state (:pending, :approved, :flagged, :rejected)
-    enum :review_state, { pending: 0, approved: 1, flagged: 2, rejected: 3 }, prefix: true
-
-    attribute :tags, :json, default: []
-    attribute :metadata, :json, default: {}
-    attribute :request_payload, :json, default: {}
-    attribute :response_payload, :json, default: {}
-
-    validates :provider, presence: true
-    validates :model, presence: true
-
-    scope :by_provider, ->(provider) { where(provider: provider) if provider.present? }
-    scope :by_model, ->(model) { where(model: model) if model.present? }
-    scope :by_project, ->(project) { where(project: project) if project.present? }
-    scope :by_status, ->(status) { where(status: status) if status.present? }
-    scope :by_review_state, ->(state) { where(review_state: state) if state.present? }
-    scope :between_dates, ->(from, to) {
-      scope = all
-      scope = scope.where("created_at >= ?", Date.parse(from.to_s).beginning_of_day) if from.present?
-      scope = scope.where("created_at <= ?", Date.parse(to.to_s).end_of_day) if to.present?
-      scope
-    }
-    scope :tagged_with, ->(tag) {
-      where("tags LIKE ?", "%#{sanitize_sql_like(tag)}%") if tag.present?
-    }
-
-    def self.filtered(params)
-      by_provider(params[:provider])
-        .by_model(params[:model])
-        .by_project(params[:project])
-        .by_status(params[:status])
-        .by_review_state(params[:review_state])
-        .tagged_with(params[:tag])
-        .between_dates(params[:from], params[:to])
-    end
-  end
-end
-
-TraceBook = Tracebook unless defined?(TraceBook)

data/app/models/tracebook/redaction_rule.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-module Tracebook
-  # Rule for redacting PII from interaction payloads.
-  #
-  # Defines a regex pattern to detect and replace sensitive data before
-  # persistence. Runs on request, response, or both payloads.
-  #
-  # ## Fields
-  # - `name` - Human-readable name for this rule
-  # - `pattern` - Regular expression pattern to match
-  # - `replacement` - Replacement string (e.g., "[REDACTED]", "[EMAIL]")
-  # - `applies_to` - Where to apply: `:request`, `:response`, `:both`, `:metadata`
-  # - `enabled` - Whether this rule is active
-  #
-  # ## Built-in Rules
-  # TraceBook includes default redactors for:
-  # - Email addresses
-  # - Phone numbers (US format)
-  # - Credit card PANs
-  #
-  # @example Creating a custom redaction rule
-  #   RedactionRule.create!(
-  #     name: "API Keys",
-  #     pattern: 'api_key["\s]*[:=]["\s]*\K[\w-]+',
-  #     replacement: "[API_KEY]",
-  #     applies_to: :both,
-  #     enabled: true
-  #   )
-  #
-  # @example Email redaction
-  #   RedactionRule.create!(
-  #     name: "Email Addresses",
-  #     pattern: '\b[\w\.-]+@[\w\.-]+\.\w{2,}\b',
-  #     replacement: "[EMAIL]",
-  #     applies_to: :both,
-  #     enabled: true
-  #   )
-  #
-  # @example SSN redaction
-  #   RedactionRule.create!(
-  #     name: "Social Security Numbers",
-  #     pattern: '\b\d{3}-\d{2}-\d{4}\b',
-  #     replacement: "[SSN]",
-  #     applies_to: :both,
-  #     enabled: true
-  #   )
-  #
-  # @see Redactors::Email
-  # @see Redactors::Phone
-  # @see Redactors::CardPAN
-  class RedactionRule < ApplicationRecord
-    self.table_name = "tracebook_redaction_rules"
-
-    # @!attribute [rw] applies_to
-    #   @return [Symbol] Where to apply redaction (:request, :response, :both, :metadata)
-    enum :applies_to, { request: 0, response: 1, both: 2, metadata: 3 }
-
-    validates :name, presence: true
-    validates :pattern, presence: true
-    validates :replacement, presence: true
-
-    # Returns the compiled regex pattern.
-    #
-    # Caches the compiled pattern for performance. If pattern is invalid,
-    # falls back to escaped literal match.
-    #
-    # @return [Regexp] Compiled regular expression with MULTILINE flag
-    #
-    # @example
-    #   rule = RedactionRule.new(pattern: '\b\d{3}-\d{2}-\d{4}\b')
-    #   rule.compiled_pattern.match("123-45-6789") # => MatchData
-    def compiled_pattern
-      @compiled_pattern ||= Regexp.new(pattern, Regexp::MULTILINE)
-    rescue RegexpError
-      Regexp.new(Regexp.escape(pattern.to_s))
-    end
-  end
-end
-
-TraceBook = Tracebook unless defined?(TraceBook)

data/app/models/tracebook/rollup_daily.rb

@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-module Tracebook
-  # Daily aggregated metrics for LLM interactions.
-  #
-  # Stores summarized statistics by date/provider/model/project for analytics
-  # and cost reporting. Updated nightly by {DailyRollupsJob}.
-  #
-  # ## Fields
-  # - `date` - Date of rollup (part of composite PK)
-  # - `provider` - Provider name (part of composite PK)
-  # - `model` - Model identifier (part of composite PK)
-  # - `project` - Project name (part of composite PK, nullable)
-  # - `interactions_count` - Total number of interactions
-  # - `success_count` - Number of successful interactions
-  # - `error_count` - Number of failed interactions
-  # - `input_tokens_sum` - Sum of input tokens
-  # - `output_tokens_sum` - Sum of output tokens
-  # - `cost_cents_sum` - Total cost in cents
-  #
-  # ## Primary Key
-  # Composite PK: `(date, provider, model, project)`
-  #
-  # @example Querying daily metrics
-  #   RollupDaily
-  #     .where(provider: "openai", date: Date.current)
-  #     .sum(:cost_cents_sum) / 100.0 # Total cost in dollars
-  #
-  # @example Finding top models by cost
-  #   RollupDaily
-  #     .where(date: 30.days.ago..Date.current)
-  #     .group(:provider, :model)
-  #     .sum(:cost_cents_sum)
-  #     .sort_by { |_, cents| -cents }
-  #     .first(10)
-  #
-  # @example Error rate for a model
-  #   rollup = RollupDaily.find_by(date: Date.current, provider: "openai", model: "gpt-4o")
-  #   error_rate = rollup.error_count.to_f / rollup.interactions_count
-  #
-  # @see DailyRollupsJob
-  class RollupDaily < ApplicationRecord
-    self.table_name = "tracebook_rollups_dailies"
-
-    # @!attribute [rw] interactions_count
-    #   @return [Integer] Total number of interactions (default: 0)
-    attribute :interactions_count, :integer, default: 0
-
-    # @!attribute [rw] success_count
-    #   @return [Integer] Number of successful interactions (default: 0)
-    attribute :success_count, :integer, default: 0
-
-    # @!attribute [rw] error_count
-    #   @return [Integer] Number of failed interactions (default: 0)
-    attribute :error_count, :integer, default: 0
-
-    # @!attribute [rw] input_tokens_sum
-    #   @return [Integer] Sum of input tokens across all interactions (default: 0)
-    attribute :input_tokens_sum, :integer, default: 0
-
-    # @!attribute [rw] output_tokens_sum
-    #   @return [Integer] Sum of output tokens across all interactions (default: 0)
-    attribute :output_tokens_sum, :integer, default: 0
-
-    # @!attribute [rw] cost_cents_sum
-    #   @return [Integer] Total cost in cents (default: 0)
-    attribute :cost_cents_sum, :integer, default: 0
-
-    validates :date, presence: true
-  end
-end
-
-TraceBook = Tracebook unless defined?(TraceBook)