raif 1.0.0 → 1.2.0

data/lib/raif/json_schema_builder.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Raif
+  class JsonSchemaBuilder
+    attr_reader :properties, :required_properties, :items_schema
+
+    def initialize
+      @properties = {}
+      @required_properties = []
+      @items_schema = nil
+    end
+
+    def string(name, options = {})
+      add_property(name, "string", options)
+    end
+
+    def integer(name, options = {})
+      add_property(name, "integer", options)
+    end
+
+    def number(name, options = {})
+      add_property(name, "number", options)
+    end
+
+    def boolean(name, options = {})
+      add_property(name, "boolean", options)
+    end
+
+    def object(name = nil, options = {}, &block)
+      schema = {}
+
+      if block_given?
+        nested_builder = self.class.new
+        nested_builder.instance_eval(&block)
+
+        schema[:properties] = nested_builder.properties
+        schema[:additionalProperties] = false
+
+        # We currently use strict mode, which means that all properties are required
+        schema[:required] = nested_builder.required_properties
+      end
+
+      # If name is nil, we're inside an array and should return the schema directly
+      if name.nil?
+        @items_schema = { type: "object" }.merge(schema)
+      else
+        add_property(name, "object", options.merge(schema))
+      end
+    end
+
+    def array(name, options = {}, &block)
+      items_schema = options.delete(:items) || {}
+
+      if block_given?
+        nested_builder = self.class.new
+        nested_builder.instance_eval(&block)
+
+        # If items were directly set using the items method
+        if nested_builder.items_schema.present?
+          items_schema = nested_builder.items_schema
+        # If there are properties defined, it's an object schema
+        elsif nested_builder.properties.any?
+          items_schema = {
+            type: "object",
+            properties: nested_builder.properties,
+            additionalProperties: false
+          }
+
+          # We currently use strict mode, which means that all properties are required
+          items_schema[:required] = nested_builder.required_properties
+        end
+      end
+
+      options[:items] = items_schema unless items_schema.empty?
+      add_property(name, "array", options)
+    end
+
+    # Allow setting array items directly
+    def items(options = {})
+      @items_schema = options
+    end
+
+    def to_schema
+      {
+        type: "object",
+        additionalProperties: false,
+        properties: @properties,
+        required: @required_properties
+      }
+    end
+
+  private
+
+    def add_property(name, type, options = {})
+      property = { type: type }
+
+      # We currently use strict mode, which means that all properties are required
+      @required_properties << name.to_s
+
+      property.merge!(options)
+      @properties[name] = property
+    end
+  end
+end

data/lib/raif/llm_registry.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module Raif
+  def self.llm_registry
+    @llm_registry ||= {}
+  end
+
+  def self.register_llm(llm_class, llm_config)
+    llm = llm_class.new(**llm_config)
+
+    unless llm.valid?
+      raise ArgumentError, "The LLM you tried to register is invalid: #{llm.errors.full_messages.join(", ")}"
+    end
+
+    @llm_registry ||= {}
+    @llm_registry[llm.key] = llm_config.merge(llm_class: llm_class)
+  end
+
+  def self.llm(model_key)
+    llm_config = llm_registry[model_key]
+
+    if llm_config.nil?
+      raise ArgumentError, "No LLM found for model key: #{model_key}. Available models: #{available_llm_keys.join(", ")}"
+    end
+
+    llm_class = llm_config[:llm_class]
+    llm_class.new(**llm_config.except(:llm_class))
+  end
+
+  def self.available_llms
+    llm_registry.values
+  end
+
+  def self.available_llm_keys
+    llm_registry.keys
+  end
+
+  def self.llm_config(model_key)
+    llm_registry[model_key]
+  end
+
+  def self.default_llms
+    open_ai_models = [
+      {
+        key: :open_ai_gpt_4o_mini,
+        api_name: "gpt-4o-mini",
+        input_token_cost: 0.15 / 1_000_000,
+        output_token_cost: 0.6 / 1_000_000,
+      },
+      {
+        key: :open_ai_gpt_4o,
+        api_name: "gpt-4o",
+        input_token_cost: 2.5 / 1_000_000,
+        output_token_cost: 10.0 / 1_000_000,
+      },
+      {
+        key: :open_ai_gpt_3_5_turbo,
+        api_name: "gpt-3.5-turbo",
+        input_token_cost: 0.5 / 1_000_000,
+        output_token_cost: 1.5 / 1_000_000,
+        model_provider_settings: { supports_structured_outputs: false }
+      },
+      {
+        key: :open_ai_gpt_4_1,
+        api_name: "gpt-4.1",
+        input_token_cost: 2.0 / 1_000_000,
+        output_token_cost: 8.0 / 1_000_000,
+      },
+      {
+        key: :open_ai_gpt_4_1_mini,
+        api_name: "gpt-4.1-mini",
+        input_token_cost: 0.4 / 1_000_000,
+        output_token_cost: 1.6 / 1_000_000,
+      },
+      {
+        key: :open_ai_gpt_4_1_nano,
+        api_name: "gpt-4.1-nano",
+        input_token_cost: 0.1 / 1_000_000,
+        output_token_cost: 0.4 / 1_000_000,
+      },
+      {
+        key: :open_ai_o1,
+        api_name: "o1",
+        input_token_cost: 15.0 / 1_000_000,
+        output_token_cost: 60.0 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
+      {
+        key: :open_ai_o1_mini,
+        api_name: "o1-mini",
+        input_token_cost: 1.5 / 1_000_000,
+        output_token_cost: 6.0 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
+      {
+        key: :open_ai_o3,
+        api_name: "o3",
+        input_token_cost: 2.0 / 1_000_000,
+        output_token_cost: 8.0 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
+      {
+        key: :open_ai_o3_mini,
+        api_name: "o3-mini",
+        input_token_cost: 1.1 / 1_000_000,
+        output_token_cost: 4.4 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
+      {
+        key: :open_ai_o4_mini,
+        api_name: "o4-mini",
+        input_token_cost: 1.1 / 1_000_000,
+        output_token_cost: 4.4 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
+    ]
+
+    open_ai_responses_models = open_ai_models.dup.map.with_index do |model, _index|
+      model.merge(
+        key: model[:key].to_s.gsub("open_ai_", "open_ai_responses_").to_sym,
+        supported_provider_managed_tools: [
+          Raif::ModelTools::ProviderManaged::WebSearch,
+          Raif::ModelTools::ProviderManaged::CodeExecution,
+          Raif::ModelTools::ProviderManaged::ImageGeneration
+        ]
+      )
+    end
+
+    # o1-mini is not supported by the OpenAI Responses API.
+    open_ai_responses_models.delete_if{|model| model[:key] == :open_ai_o1_mini }
+
+    # o1-pro and o3-pro are not supported by the OpenAI Completions API, but it is supported by the OpenAI Responses API.
+    open_ai_responses_models << {
+      key: :open_ai_responses_o1_pro,
+      api_name: "o1-pro",
+      input_token_cost: 150.0 / 1_000_000,
+      output_token_cost: 600.0 / 1_000_000,
+      model_provider_settings: { supports_temperature: false },
+    }
+
+    open_ai_responses_models << {
+      key: :open_ai_responses_o3_pro,
+      api_name: "o3-pro",
+      input_token_cost: 20.0 / 1_000_000,
+      output_token_cost: 80.0 / 1_000_000,
+      model_provider_settings: { supports_temperature: false },
+    }
+
+    {
+      Raif::Llms::OpenAiCompletions => open_ai_models,
+      Raif::Llms::OpenAiResponses => open_ai_responses_models,
+      Raif::Llms::Anthropic => [
+        {
+          key: :anthropic_claude_4_sonnet,
+          api_name: "claude-sonnet-4-20250514",
+          input_token_cost: 3.0 / 1_000_000,
+          output_token_cost: 15.0 / 1_000_000,
+          max_completion_tokens: 8192,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :anthropic_claude_4_opus,
+          api_name: "claude-opus-4-20250514",
+          input_token_cost: 15.0 / 1_000_000,
+          output_token_cost: 75.0 / 1_000_000,
+          max_completion_tokens: 8192,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :anthropic_claude_3_7_sonnet,
+          api_name: "claude-3-7-sonnet-latest",
+          input_token_cost: 3.0 / 1_000_000,
+          output_token_cost: 15.0 / 1_000_000,
+          max_completion_tokens: 8192,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :anthropic_claude_3_5_sonnet,
+          api_name: "claude-3-5-sonnet-latest",
+          input_token_cost: 3.0 / 1_000_000,
+          output_token_cost: 15.0 / 1_000_000,
+          max_completion_tokens: 8192,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :anthropic_claude_3_5_haiku,
+          api_name: "claude-3-5-haiku-latest",
+          input_token_cost: 0.8 / 1_000_000,
+          output_token_cost: 4.0 / 1_000_000,
+          max_completion_tokens: 8192,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :anthropic_claude_3_opus,
+          api_name: "claude-3-opus-latest",
+          input_token_cost: 15.0 / 1_000_000,
+          output_token_cost: 75.0 / 1_000_000,
+          max_completion_tokens: 4096
+        },
+      ],
+      Raif::Llms::Bedrock => [
+        {
+          key: :bedrock_claude_4_sonnet,
+          api_name: "anthropic.claude-sonnet-4-20250514-v1:0",
+          input_token_cost: 0.003 / 1000,
+          output_token_cost: 0.015 / 1000,
+          max_completion_tokens: 8192
+        },
+        {
+          key: :bedrock_claude_4_opus,
+          api_name: "anthropic.claude-opus-4-20250514-v1:0",
+          input_token_cost: 0.015 / 1000,
+          output_token_cost: 0.075 / 1000,
+          max_completion_tokens: 8192
+        },
+        {
+          key: :bedrock_claude_3_5_sonnet,
+          api_name: "anthropic.claude-3-5-sonnet-20241022-v2:0",
+          input_token_cost: 0.003 / 1000,
+          output_token_cost: 0.015 / 1000,
+          max_completion_tokens: 8192
+        },
+        {
+          key: :bedrock_claude_3_7_sonnet,
+          api_name: "anthropic.claude-3-7-sonnet-20250219-v1:0",
+          input_token_cost: 0.003 / 1000,
+          output_token_cost: 0.015 / 1000,
+          max_completion_tokens: 8192
+        },
+        {
+          key: :bedrock_claude_3_5_haiku,
+          api_name: "anthropic.claude-3-5-haiku-20241022-v1:0",
+          input_token_cost: 0.0008 / 1000,
+          output_token_cost: 0.004 / 1000,
+          max_completion_tokens: 8192
+        },
+        {
+          key: :bedrock_claude_3_opus,
+          api_name: "anthropic.claude-3-opus-20240229-v1:0",
+          input_token_cost: 0.015 / 1000,
+          output_token_cost: 0.075 / 1000,
+          max_completion_tokens: 4096
+        },
+        {
+          key: :bedrock_amazon_nova_micro,
+          api_name: "amazon.nova-micro-v1:0",
+          input_token_cost: 0.0000115 / 1000,
+          output_token_cost: 0.000184 / 1000,
+          max_completion_tokens: 4096
+        },
+        {
+          key: :bedrock_amazon_nova_lite,
+          api_name: "amazon.nova-lite-v1:0",
+          input_token_cost: 0.0000195 / 1000,
+          output_token_cost: 0.000312 / 1000,
+          max_completion_tokens: 4096
+        },
+        {
+          key: :bedrock_amazon_nova_pro,
+          api_name: "amazon.nova-pro-v1:0",
+          input_token_cost: 0.0002625 / 1000,
+          output_token_cost: 0.0042 / 1000,
+          max_completion_tokens: 4096
+        }
+      ],
+      Raif::Llms::OpenRouter => [
+        {
+          key: :open_router_claude_3_7_sonnet,
+          api_name: "anthropic/claude-3.7-sonnet",
+          input_token_cost: 3.0 / 1_000_000,
+          output_token_cost: 15.0 / 1_000_000,
+        },
+        {
+          key: :open_router_llama_3_3_70b_instruct,
+          api_name: "meta-llama/llama-3.3-70b-instruct",
+          input_token_cost: 0.10 / 1_000_000,
+          output_token_cost: 0.25 / 1_000_000,
+        },
+        {
+          key: :open_router_llama_3_1_8b_instruct,
+          api_name: "meta-llama/llama-3.1-8b-instruct",
+          input_token_cost: 0.02 / 1_000_000,
+          output_token_cost: 0.03 / 1_000_000,
+        },
+        {
+          key: :open_router_gemini_2_0_flash,
+          api_name: "google/gemini-2.0-flash-001",
+          input_token_cost: 0.1 / 1_000_000,
+          output_token_cost: 0.4 / 1_000_000,
+        },
+        {
+          key: :open_router_deepseek_chat_v3,
+          api_name: "deepseek/deepseek-chat-v3-0324",
+          input_token_cost: 0.27 / 1_000_000,
+          output_token_cost: 1.1 / 1_000_000,
+        },
+      ]
+    }
+  end
+end

data/lib/raif/migration_checker.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Raif
+  module MigrationChecker
+    class << self
+      def uninstalled_migrations
+        engine_migration_names = engine_migration_names_from_context
+        ran_migration_names = ran_migration_names_from_host
+
+        engine_migration_names - ran_migration_names
+      end
+
+      def check_and_warn!
+        return unless defined?(Rails) && Rails.application
+
+        uninstalled = uninstalled_migrations
+        return if uninstalled.empty?
+
+        warning_message = build_warning_message(uninstalled)
+
+        # Output to both logger and STDOUT to ensure visibility
+        Rails.logger&.warn(warning_message)
+        warn warning_message
+      end
+
+    private
+
+      def engine_migration_names_from_context
+        engine_paths = Raif::Engine.paths["db/migrate"].existent
+        return [] if engine_paths.empty?
+
+        ActiveRecord::MigrationContext.new(engine_paths).migrations.map(&:name)
+      rescue => e
+        Rails.logger&.debug("Raif: Could not load engine migrations: #{e.message}")
+        []
+      end
+
+      def ran_migration_names_from_host
+        return [] unless defined?(Rails) && Rails.application
+
+        app_paths = Rails.application.paths["db/migrate"].expanded
+        return [] if app_paths.empty?
+
+        ctx = ActiveRecord::MigrationContext.new(app_paths)
+        ran_versions = ctx.get_all_versions
+        ctx.migrations.select{|m| ran_versions.include?(m.version) }.map(&:name)
+      rescue ActiveRecord::NoDatabaseError
+        # Database doesn't exist yet, so no migrations have been run
+        []
+      rescue => e
+        Rails.logger&.debug("Raif: Could not load migration status: #{e.message}")
+        []
+      end
+
+      def build_warning_message(uninstalled_migration_names)
+        <<~WARNING
+          \e[33m
+          ⚠️  RAIF MIGRATION WARNING ⚠️
+
+          The following Raif migrations have not been run in your application:
+
+          #{uninstalled_migration_names.map { |name| "  • #{name}" }.join("\n")}
+
+          To install and run these migrations:
+
+            rails raif:install:migrations
+            rails db:migrate
+
+          \e[0m
+        WARNING
+      end
+    end
+  end
+end

data/lib/raif/utils/html_fragment_processor.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# Utility class for processing HTML fragments with various cleaning and transformation operations.
+#
+# This class provides methods for sanitizing HTML content, converting markdown links to HTML,
+# processing existing HTML links (adding target="_blank", stripping tracking parameters),
+# and removing tracking parameters from URLs.
+class Raif::Utils::HtmlFragmentProcessor
+  # List of common tracking parameters to remove from URLs
+  TRACKING_PARAMS = %w[
+    utm_source
+    utm_medium
+    utm_campaign
+    utm_term
+    utm_content
+    utm_id
+  ]
+
+  class << self
+    # Cleans and sanitizes an HTML fragment by removing empty text nodes and dangerous content.
+    #
+    # @param html [String, Nokogiri::HTML::DocumentFragment] The HTML content to clean
+    # @param allowed_tags [Array<String>, nil] Array of allowed HTML tags. Defaults to Rails HTML5 safe list
+    # @param allowed_attributes [Array<String>, nil] Array of allowed HTML attributes. Defaults to Rails HTML5 safe list
+    # @return [String] Cleaned and sanitized HTML string
+    #
+    # @example
+    #   clean_html_fragment("<script>alert('xss')</script><p>Safe content</p>")
+    #   # => "<p>Safe content</p>"
+    #
+    # @example With custom allowed tags
+    #   clean_html_fragment("<p>Para</p><div>Div</div>", allowed_tags: %w[p])
+    #   # => "<p>Para</p>Div"
+    def clean_html_fragment(html, allowed_tags: nil, allowed_attributes: nil)
+      fragment = html.is_a?(Nokogiri::HTML::DocumentFragment) ? html : Nokogiri::HTML.fragment(html)
+
+      fragment.traverse do |node|
+        if node.text? && node.text.strip.empty?
+          node.remove
+        end
+      end
+
+      allowed_tags = allowed_tags.presence || Rails::HTML5::SafeListSanitizer.allowed_tags
+      allowed_attributes = allowed_attributes.presence || Rails::HTML5::SafeListSanitizer.allowed_attributes
+
+      ActionController::Base.helpers.sanitize(fragment.to_html, tags: allowed_tags, attributes: allowed_attributes).strip
+    end
+
+    # Converts markdown-style links to HTML anchor tags with target="_blank" and rel="noopener".
+    #
+    # Converts [text](url) format to <a href="url" target="_blank" rel="noopener">text</a>.
+    # Also strips tracking parameters from the URLs.
+    #
+    # @param text [String] The text content that may contain markdown links
+    # @return [String] HTML with markdown links converted to anchor tags
+    #
+    # @example
+    #   convert_markdown_links_to_html("Check out [Google](https://google.com) for search.")
+    #   # => 'Check out <a href="https://google.com" target="_blank" rel="noopener">Google</a> for search.'
+    #
+    # @example With tracking parameters
+    #   convert_markdown_links_to_html("[Example](https://example.com?utm_source=test&param=keep)")
+    #   # => '<a href="https://example.com?param=keep" target="_blank" rel="noopener">Example</a>'
+    def convert_markdown_links_to_html(text)
+      # Convert markdown links [text](url) to HTML links <a href="url" target="_blank" rel="noopener">text</a>
+      text.gsub(/\[([^\]]*)\]\(([^)]+)\)/) do |_match|
+        text = ::Regexp.last_match(1)
+        url = ::Regexp.last_match(2)
+        clean_url = strip_tracking_parameters(url)
+        %(<a href="#{CGI.escapeHTML(clean_url)}" target="_blank" rel="noopener">#{CGI.escapeHTML(text)}</a>)
+      end
+    end
+
+    # Processes existing HTML links by optionally adding target="_blank" and stripping tracking parameters.
+    #
+    # This method provides fine-grained control over link processing with configurable options
+    # for both target="_blank" addition and tracking parameter removal.
+    #
+    # @param html [String, Nokogiri::HTML::DocumentFragment] The HTML content containing links to process
+    # @param add_target_blank [Boolean] Whether to add target="_blank" and rel="noopener" to links (required)
+    # @param strip_tracking_parameters [Boolean] Whether to remove tracking parameters from URLs (required)
+    # @return [String] Processed HTML with modified links
+    #
+    # @example Default behavior (adds target="_blank" and strips tracking params)
+    #   process_links('<a href="https://example.com?utm_source=test">Link</a>', add_target_blank: true, strip_tracking_parameters: true)
+    #   # => '<a href="https://example.com" target="_blank" rel="noopener">Link</a>'
+    #
+    # @example Only strip tracking parameters
+    #   process_links(html, add_target_blank: false, strip_tracking_parameters: true)
+    #   # => '<a href="https://example.com">Link</a>'
+    #
+    # @example Only add target="_blank"
+    #   process_links(html, add_target_blank: true, strip_tracking_parameters: false)
+    #   # => '<a href="https://example.com?utm_source=test" target="_blank" rel="noopener">Link</a>'
+    #
+    # @example No processing
+    #   process_links(html, add_target_blank: false, strip_tracking_parameters: false)
+    #   # => Original HTML unchanged
+    def process_links(html, add_target_blank:, strip_tracking_parameters:)
+      fragment = html.is_a?(Nokogiri::HTML::DocumentFragment) ? html : Nokogiri::HTML.fragment(html)
+
+      fragment.css("a").each do |link|
+        if add_target_blank
+          link["target"] = "_blank"
+          link["rel"] = "noopener"
+        end
+
+        if strip_tracking_parameters
+          link["href"] = strip_tracking_parameters(link["href"])
+        end
+      end
+
+      fragment.to_html
+    end
+
+    # Removes tracking parameters (UTM parameters) from a URL.
+    #
+    # Preserves all non-tracking query parameters and handles various URL formats including
+    # relative URLs, absolute URLs, and malformed URLs gracefully.
+    #
+    # @param url [String] The URL to clean
+    # @return [String] URL with tracking parameters removed, or original URL if parsing fails
+    #
+    # @example
+    #   strip_tracking_parameters("https://example.com?utm_source=google&page=1")
+    #   # => "https://example.com?page=1"
+    #
+    # @example Removes all tracking parameters
+    #   strip_tracking_parameters("https://example.com?utm_source=test&utm_medium=cpc")
+    #   # => "https://example.com"
+    #
+    # @example Preserves fragments
+    #   strip_tracking_parameters("https://example.com?utm_source=test&page=1#section")
+    #   # => "https://example.com?page=1#section"
+    #
+    # @example Handles relative URLs
+    #   strip_tracking_parameters("/path?utm_source=test&param=keep")
+    #   # => "/path?param=keep"
+    def strip_tracking_parameters(url)
+      return url unless url.include?("?")
+
+      begin
+        uri = URI.parse(url)
+        return url unless uri.query
+
+        # Only process URLs that have a valid scheme and host, or are relative URLs
+        unless uri.scheme || url.start_with?("/", "#")
+          return url
+        end
+
+        # Parse query parameters and filter out tracking ones
+        params = URI.decode_www_form(uri.query)
+        clean_params = params.reject { |param, _| TRACKING_PARAMS.include?(param.downcase) }
+
+        # Rebuild the URL
+        uri.query = if clean_params.empty?
+          nil
+        else
+          URI.encode_www_form(clean_params)
+        end
+
+        uri.to_s
+      rescue URI::InvalidURIError
+        # If URL parsing fails, return the original URL
+        url
+      end
+    end
+  end
+end

data/lib/raif/utils.rb

@@ -3,4 +3,5 @@
 module Raif::Utils
   require "raif/utils/readable_content_extractor"
   require "raif/utils/html_to_markdown_converter"
+  require "raif/utils/html_fragment_processor"
 end

data/lib/raif/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Raif
-  VERSION = "1.0.0"
+  VERSION = "1.2.0"
 end