ruby_llm-agents 1.3.4 → 2.1.0

data/lib/generators/ruby_llm_agents/templates/split_execution_details_migration.rb.tt

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+# Idempotent migration to split large payload columns from executions to execution_details.
+#
+# Handles all upgrade scenarios:
+# - execution_details table doesn't exist: creates it and migrates data
+# - execution_details table exists but old columns remain on executions: migrates data and removes columns
+# - already clean: no-ops safely
+class SplitExecutionDetailsFromExecutions < ActiveRecord::Migration<%= migration_version %>
+  # Columns that belong on execution_details, not executions
+  DETAIL_COLUMNS = %i[
+    error_message system_prompt user_prompt response messages_summary
+    tool_calls attempts fallback_chain parameters routed_to
+    classification_result cached_at cache_creation_tokens
+  ].freeze
+
+  # Niche columns moved to metadata JSON
+  NICHE_COLUMNS = %i[
+    span_id response_cache_key time_to_first_token_ms
+    retryable rate_limited fallback_reason
+  ].freeze
+
+  # Polymorphic tenant columns removed from executions (access via Tenant model)
+  TENANT_RECORD_COLUMNS = %i[tenant_record_type tenant_record_id].freeze
+
+  def up
+    create_details_table_if_needed
+    backfill_and_remove_old_columns
+    remove_niche_columns
+    remove_tenant_record_columns
+    ensure_required_columns
+    cleanup_indexes
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration,
+          "This migration cannot be reversed. Use rails db:schema:load to restore."
+  end
+
+  private
+
+  def create_details_table_if_needed
+    return if table_exists?(:ruby_llm_agents_execution_details)
+
+    create_table :ruby_llm_agents_execution_details do |t|
+      t.references :execution, null: false,
+                   foreign_key: { to_table: :ruby_llm_agents_executions, on_delete: :cascade },
+                   index: { unique: true }
+
+      t.text     :error_message
+      t.text     :system_prompt
+      t.text     :user_prompt
+      t.json     :response,             default: {}
+      t.json     :messages_summary,     default: {}, null: false
+      t.json     :tool_calls,           default: [], null: false
+      t.json     :attempts,             default: [], null: false
+      t.json     :fallback_chain
+      t.json     :parameters,           default: {}, null: false
+      t.string   :routed_to
+      t.json     :classification_result
+      t.datetime :cached_at
+      t.integer  :cache_creation_tokens, default: 0
+
+      t.timestamps
+    end
+  end
+
+  def backfill_and_remove_old_columns
+    # Only proceed if detail columns still exist on executions
+    columns_present = DETAIL_COLUMNS.select { |col| column_exists?(:ruby_llm_agents_executions, col) }
+    return if columns_present.empty?
+
+    # Backfill data from executions to execution_details
+    say_with_time "Backfilling execution_details from executions" do
+      backfill_execution_details(columns_present)
+    end
+
+    # Remove old columns from executions
+    columns_present.each do |col|
+      remove_column :ruby_llm_agents_executions, col
+    end
+  end
+
+  def backfill_execution_details(columns_present)
+    batch_size = 1000
+    count = 0
+
+    # Build WHERE clause to only copy rows that have data
+    has_data_conditions = columns_present.map { |col| "e.#{col} IS NOT NULL" }.join(" OR ")
+
+    loop do
+      ids = exec_query(<<~SQL).rows.flatten
+        SELECT e.id FROM ruby_llm_agents_executions e
+        LEFT JOIN ruby_llm_agents_execution_details d ON d.execution_id = e.id
+        WHERE d.id IS NULL AND (#{has_data_conditions})
+        ORDER BY e.id
+        LIMIT #{batch_size}
+      SQL
+
+      break if ids.empty?
+
+      # Build dynamic column lists based on what actually exists
+      detail_cols = %w[execution_id created_at updated_at] + columns_present.map(&:to_s)
+      select_exprs = %w[id created_at updated_at] + columns_present.map { |col|
+        case col
+        when :messages_summary then "COALESCE(messages_summary, '{}')"
+        when :tool_calls then "COALESCE(tool_calls, '[]')"
+        when :attempts then "COALESCE(attempts, '[]')"
+        when :parameters then "COALESCE(parameters, '{}')"
+        else col.to_s
+        end
+      }
+
+      execute <<~SQL
+        INSERT INTO ruby_llm_agents_execution_details (#{detail_cols.join(', ')})
+        SELECT #{select_exprs.join(', ')}
+        FROM ruby_llm_agents_executions
+        WHERE id IN (#{ids.join(',')})
+      SQL
+
+      count += ids.size
+    end
+
+    count
+  end
+
+  def remove_niche_columns
+    NICHE_COLUMNS.each do |col|
+      if column_exists?(:ruby_llm_agents_executions, col)
+        remove_column :ruby_llm_agents_executions, col
+      end
+    end
+  end
+
+  def remove_tenant_record_columns
+    remove_index :ruby_llm_agents_executions, column: TENANT_RECORD_COLUMNS,
+                 name: "index_executions_on_tenant_record", if_exists: true
+
+    TENANT_RECORD_COLUMNS.each do |col|
+      if column_exists?(:ruby_llm_agents_executions, col)
+        remove_column :ruby_llm_agents_executions, col
+      end
+    end
+  end
+
+  def ensure_required_columns
+    unless column_exists?(:ruby_llm_agents_executions, :execution_type)
+      add_column :ruby_llm_agents_executions, :execution_type, :string, null: false, default: "chat"
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :chosen_model_id)
+      add_column :ruby_llm_agents_executions, :chosen_model_id, :string
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :messages_count)
+      add_column :ruby_llm_agents_executions, :messages_count, :integer, default: 0, null: false
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :attempts_count)
+      add_column :ruby_llm_agents_executions, :attempts_count, :integer, default: 1, null: false
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :tool_calls_count)
+      add_column :ruby_llm_agents_executions, :tool_calls_count, :integer, default: 0, null: false
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :streaming)
+      add_column :ruby_llm_agents_executions, :streaming, :boolean, default: false
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :finish_reason)
+      add_column :ruby_llm_agents_executions, :finish_reason, :string
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :cache_hit)
+      add_column :ruby_llm_agents_executions, :cache_hit, :boolean, default: false
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :trace_id)
+      add_column :ruby_llm_agents_executions, :trace_id, :string
+      add_index :ruby_llm_agents_executions, :trace_id
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :request_id)
+      add_column :ruby_llm_agents_executions, :request_id, :string
+      add_index :ruby_llm_agents_executions, :request_id
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :parent_execution_id)
+      add_column :ruby_llm_agents_executions, :parent_execution_id, :bigint
+      add_index :ruby_llm_agents_executions, :parent_execution_id
+      add_foreign_key :ruby_llm_agents_executions, :ruby_llm_agents_executions,
+                      column: :parent_execution_id, on_delete: :nullify
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :root_execution_id)
+      add_column :ruby_llm_agents_executions, :root_execution_id, :bigint
+      add_index :ruby_llm_agents_executions, :root_execution_id
+      add_foreign_key :ruby_llm_agents_executions, :ruby_llm_agents_executions,
+                      column: :root_execution_id, on_delete: :nullify
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :tenant_id)
+      add_column :ruby_llm_agents_executions, :tenant_id, :string
+    end
+    unless column_exists?(:ruby_llm_agents_executions, :cached_tokens)
+      add_column :ruby_llm_agents_executions, :cached_tokens, :integer, default: 0
+    end
+  end
+
+  def cleanup_indexes
+    # Remove single-column indexes that are redundant with composite ones
+    %i[duration_ms total_cost messages_count attempts_count tool_calls_count
+       chosen_model_id execution_type response_cache_key agent_type tenant_id].each do |col|
+      remove_index :ruby_llm_agents_executions, col, if_exists: true
+    end
+
+    # Ensure composite tenant indexes exist
+    unless index_exists?(:ruby_llm_agents_executions, [:tenant_id, :created_at])
+      add_index :ruby_llm_agents_executions, [:tenant_id, :created_at]
+    end
+    unless index_exists?(:ruby_llm_agents_executions, [:tenant_id, :status])
+      add_index :ruby_llm_agents_executions, [:tenant_id, :status]
+    end
+
+    # Remove workflow indexes (feature removed)
+    remove_index :ruby_llm_agents_executions, [:workflow_id, :workflow_step], if_exists: true
+    remove_index :ruby_llm_agents_executions, :workflow_id, if_exists: true
+    remove_index :ruby_llm_agents_executions, :workflow_type, if_exists: true
+
+    # Remove workflow columns if present
+    %i[workflow_id workflow_type workflow_step].each do |col|
+      if column_exists?(:ruby_llm_agents_executions, col)
+        remove_column :ruby_llm_agents_executions, col
+      end
+    end
+
+    # Remove agent_version if present
+    if column_exists?(:ruby_llm_agents_executions, :agent_version)
+      remove_index :ruby_llm_agents_executions, [:agent_type, :agent_version], if_exists: true
+      remove_column :ruby_llm_agents_executions, :agent_version
+    end
+  end
+end

data/lib/generators/ruby_llm_agents/upgrade_generator.rb

@@ -2,8 +2,6 @@
 
 require "rails/generators"
 require "rails/generators/active_record"
-require "fileutils"
-
 module RubyLlmAgents
   # Upgrade generator for ruby_llm-agents
   #
@@ -11,142 +9,37 @@ module RubyLlmAgents
   #   rails generate ruby_llm_agents:upgrade
   #
   # This will create any missing migrations for upgrading from older versions.
+  # It handles all upgrade scenarios:
+  #
+  # - v0.x/v1.x -> v2.0: Splits detail columns from executions to execution_details,
+  #   removes deprecated columns, renames tenant_budgets to tenants
+  # - v2.0 -> latest: No-ops safely if already up to date
   #
   class UpgradeGenerator < ::Rails::Generators::Base
     include ::ActiveRecord::Generators::Migration
 
     source_root File.expand_path("templates", __dir__)
 
-    def create_add_prompts_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :system_prompt)
-        say_status :skip, "system_prompt column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_prompts_migration.rb.tt",
-        File.join(db_migrate_path, "add_prompts_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_attempts_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :attempts)
-        say_status :skip, "attempts column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_attempts_migration.rb.tt",
-        File.join(db_migrate_path, "add_attempts_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_streaming_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :streaming)
-        say_status :skip, "streaming column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_streaming_migration.rb.tt",
-        File.join(db_migrate_path, "add_streaming_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_tracing_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :trace_id)
-        say_status :skip, "trace_id column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_tracing_migration.rb.tt",
-        File.join(db_migrate_path, "add_tracing_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_routing_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :fallback_reason)
-        say_status :skip, "fallback_reason column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_routing_migration.rb.tt",
-        File.join(db_migrate_path, "add_routing_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_finish_reason_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :finish_reason)
-        say_status :skip, "finish_reason column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_finish_reason_migration.rb.tt",
-        File.join(db_migrate_path, "add_finish_reason_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_caching_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :cache_hit)
-        say_status :skip, "cache_hit column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_caching_migration.rb.tt",
-        File.join(db_migrate_path, "add_caching_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_tool_calls_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :tool_calls)
-        say_status :skip, "tool_calls column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_tool_calls_migration.rb.tt",
-        File.join(db_migrate_path, "add_tool_calls_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_workflow_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :workflow_id)
-        say_status :skip, "workflow_id column already exists", :yellow
-        return
-      end
-
-      migration_template(
-        "add_workflow_migration.rb.tt",
-        File.join(db_migrate_path, "add_workflow_to_ruby_llm_agents_executions.rb")
-      )
-    end
-
-    def create_add_execution_type_migration
-      # Check if columns already exist
-      if column_exists?(:ruby_llm_agents_executions, :execution_type)
-        say_status :skip, "execution_type column already exists", :yellow
+    # Main upgrade: split execution_details from executions table
+    #
+    # This single migration handles ALL schema transitions:
+    # - Creates execution_details table if missing
+    # - Migrates data from old columns on executions to execution_details
+    # - Removes deprecated columns (detail, niche, workflow, agent_version)
+    # - Adds any missing columns that should stay on executions
+    def create_split_execution_details_migration
+      if already_split?
+        say_status :skip, "execution_details already split from executions", :yellow
         return
       end
 
       migration_template(
-        "add_execution_type_migration.rb.tt",
-        File.join(db_migrate_path, "add_execution_type_to_ruby_llm_agents_executions.rb")
+        "split_execution_details_migration.rb.tt",
+        File.join(db_migrate_path, "split_execution_details_from_executions.rb")
       )
     end
 
+    # Rename tenant_budgets to tenants (v1.x -> v2.0 upgrade)
     def create_rename_tenant_budgets_migration
       # Skip if already using new table name
       if table_exists?(:ruby_llm_agents_tenants)
@@ -167,49 +60,35 @@ module RubyLlmAgents
       )
     end
 
-    def migrate_agents_directory
-      root_dir = RubyLLM::Agents.configuration.root_directory
-      namespace = RubyLLM::Agents.configuration.root_namespace
-      migrate_directory("agents", "#{root_dir}/agents", namespace)
-    end
+    def suggest_config_consolidation
+      ruby_llm_initializer = File.join(destination_root, "config/initializers/ruby_llm.rb")
+      agents_initializer = File.join(destination_root, "config/initializers/ruby_llm_agents.rb")
 
-    def migrate_tools_directory
-      root_dir = RubyLLM::Agents.configuration.root_directory
-      namespace = RubyLLM::Agents.configuration.root_namespace
-      migrate_directory("tools", "#{root_dir}/tools", namespace)
-    end
+      return unless File.exist?(ruby_llm_initializer) && File.exist?(agents_initializer)
 
-    def show_post_upgrade_message
       say ""
-      say "RubyLLM::Agents upgrade complete!", :green
-      say ""
-      say "Next steps:"
-      say "  1. Run migrations: rails db:migrate"
-      say "  2. Update class references in your controllers, views, and tests"
-      say "  3. Run your test suite to find any broken references"
+      say "Optional: You can now consolidate your API key configuration.", :yellow
       say ""
-    end
-
-    def show_migration_summary
-      namespace = RubyLLM::Agents.configuration.root_namespace
-      root_dir = RubyLLM::Agents.configuration.root_directory
-
-      return unless @agents_migrated || @tools_migrated
-
+      say "Move your API keys from config/initializers/ruby_llm.rb"
+      say "into config/initializers/ruby_llm_agents.rb:"
       say ""
-      say "=" * 60
-      say "  File Migration Summary", :green
-      say "=" * 60
+      say "  RubyLLM::Agents.configure do |config|"
+      say "    config.openai_api_key = ENV['OPENAI_API_KEY']"
+      say "    config.anthropic_api_key = ENV['ANTHROPIC_API_KEY']"
+      say "    # ... rest of your agent config"
+      say "  end"
       say ""
-      say "Your agents and tools have been migrated to the new structure:"
+      say "Then delete config/initializers/ruby_llm.rb if it only contained API keys."
       say ""
-      say "  app/agents/  →  app/#{root_dir}/agents/" if @agents_migrated
-      say "  app/tools/   →  app/#{root_dir}/tools/" if @tools_migrated
+    end
+
+    def show_post_upgrade_message
       say ""
-      say "Classes are now namespaced under #{namespace}::"
+      say "RubyLLM::Agents upgrade complete!", :green
       say ""
-      say "  Before: GeneralAgent.call(...)"
-      say "  After:  #{namespace}::GeneralAgent.call(...)"
+      say "Next steps:"
+      say "  1. Run migrations: rails db:migrate"
+      say "  2. Run your test suite to verify everything works"
       say ""
     end
 
@@ -223,118 +102,57 @@ module RubyLlmAgents
       "db/migrate"
     end
 
-    def column_exists?(table, column)
-      return false unless ActiveRecord::Base.connection.table_exists?(table)
-
-      ActiveRecord::Base.connection.column_exists?(table, column)
-    rescue StandardError
-      false
-    end
+    # Check if the split has already been completed:
+    # - execution_details table exists
+    # - No detail columns remain on executions
+    # - No deprecated columns remain on executions
+    def already_split?
+      return false unless table_exists?(:ruby_llm_agents_execution_details)
+      return false if has_detail_columns_on_executions?
+      return false if has_deprecated_columns_on_executions?
 
-    def table_exists?(table)
-      ActiveRecord::Base.connection.table_exists?(table)
-    rescue StandardError
-      false
+      true
     end
 
-    def migrate_directory(old_dir, new_dir, namespace)
-      source = Rails.root.join("app", old_dir)
-      destination = Rails.root.join("app", new_dir)
-
-      # Skip if source doesn't exist
-      unless File.directory?(source)
-        say_status :skip, "app/#{old_dir}/ does not exist", :yellow
-        return
-      end
-
-      # Skip if source and destination are the same
-      if source.to_s == destination.to_s
-        say_status :skip, "app/#{old_dir}/ is already at destination", :yellow
-        return
-      end
-
-      files_to_migrate = Dir.glob("#{source}/**/*.rb")
-      if files_to_migrate.empty?
-        say_status :skip, "app/#{old_dir}/ has no Ruby files to migrate", :yellow
-        return
-      end
-
-      if options[:pretend]
-        say_status :preview, "Would move #{files_to_migrate.size} files from app/#{old_dir}/ → app/#{new_dir}/", :yellow
-        files_to_migrate.each do |file|
-          relative = file.sub("#{source}/", "")
-          say_status :would_move, relative, :cyan
-        end
-        return
-      end
-
-      # Create destination directory
-      FileUtils.mkdir_p(destination)
-
-      # Track conflicts and migrated files
-      conflicts = []
-      migrated = []
-
-      files_to_migrate.each do |file|
-        relative_path = file.sub("#{source}/", "")
-        dest_file = File.join(destination, relative_path)
-
-        # Skip if destination file already exists (conflict)
-        if File.exist?(dest_file)
-          conflicts << relative_path
-          say_status :conflict, "app/#{new_dir}/#{relative_path} already exists, skipping", :red
-          next
-        end
-
-        FileUtils.mkdir_p(File.dirname(dest_file))
-        FileUtils.mv(file, dest_file)
-
-        wrap_in_namespace(dest_file, namespace)
-        say_status :migrated, "app/#{new_dir}/#{relative_path}", :green
-        migrated << relative_path
-      end
+    # Detail columns that should only exist on execution_details, not executions
+    DETAIL_COLUMNS = %i[
+      error_message system_prompt user_prompt response messages_summary
+      tool_calls attempts fallback_chain parameters routed_to
+      classification_result cached_at cache_creation_tokens
+    ].freeze
 
-      # Cleanup empty source directory
-      cleanup_empty_directory(source, old_dir)
+    # Niche columns that should be in metadata JSON, not separate columns
+    NICHE_COLUMNS = %i[
+      span_id response_cache_key time_to_first_token_ms
+      retryable rate_limited fallback_reason
+    ].freeze
 
-      # Track that migration happened for summary
-      instance_variable_set("@#{old_dir}_migrated", migrated.any?)
+    # Deprecated columns
+    DEPRECATED_COLUMNS = %i[
+      agent_version workflow_id workflow_type workflow_step
+      tenant_record_type tenant_record_id
+    ].freeze
 
-      { migrated: migrated, conflicts: conflicts }
+    def has_detail_columns_on_executions?
+      DETAIL_COLUMNS.any? { |col| column_exists?(:ruby_llm_agents_executions, col) }
     end
 
-    def wrap_in_namespace(file, namespace)
-      content = File.read(file)
-
-      # Skip if already namespaced
-      return if content.include?("module #{namespace}")
-
-      # Wrap content in namespace with proper indentation
-      indented = content.lines.map { |line| line.empty? || line.strip.empty? ? line : "  #{line}" }.join
-      wrapped = "module #{namespace}\n#{indented}end\n"
-
-      File.write(file, wrapped)
+    def has_deprecated_columns_on_executions?
+      (NICHE_COLUMNS + DEPRECATED_COLUMNS).any? { |col| column_exists?(:ruby_llm_agents_executions, col) }
     end
 
-    def cleanup_empty_directory(dir, dir_name)
-      return unless File.directory?(dir)
-
-      # Remove empty subdirectories first (deepest first)
-      Dir.glob("#{dir}/**/", File::FNM_DOTMATCH).sort_by(&:length).reverse.each do |subdir|
-        next if subdir.end_with?(".", "..")
+    def column_exists?(table, column)
+      return false unless ActiveRecord::Base.connection.table_exists?(table)
 
-        FileUtils.rmdir(subdir) if File.directory?(subdir) && Dir.empty?(subdir)
-      rescue Errno::ENOTEMPTY, Errno::ENOENT
-        # Directory not empty or already removed
-      end
+      ActiveRecord::Base.connection.column_exists?(table, column)
+    rescue StandardError
+      false
+    end
 
-      # Remove main directory if empty
-      if File.directory?(dir) && Dir.empty?(dir)
-        FileUtils.rmdir(dir)
-        say_status :removed, "app/#{dir_name}/ (empty)", :yellow
-      end
-    rescue Errno::ENOTEMPTY
-      say_status :kept, "app/#{dir_name}/ (not empty)", :yellow
+    def table_exists?(table)
+      ActiveRecord::Base.connection.table_exists?(table)
+    rescue StandardError
+      false
     end
   end
 end

data/lib/ruby_llm/agents/audio/speaker.rb

@@ -353,7 +353,6 @@ module RubyLLM
           "ruby_llm_agents",
           "speech",
           self.class.name,
-          self.class.version,
           resolved_provider,
           resolved_model,
           resolved_voice,

data/lib/ruby_llm/agents/audio/transcriber.rb

@@ -355,7 +355,6 @@ module RubyLLM
           "ruby_llm_agents",
           "transcription",
           self.class.name,
-          self.class.version,
           resolved_model,
           resolved_language,
           self.class.output_format,