raif 1.1.0 → 1.2.1.pre

data/lib/raif/utils/html_fragment_processor.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+# Utility class for processing HTML fragments with various cleaning and transformation operations.
+#
+# This class provides methods for sanitizing HTML content, converting markdown links to HTML,
+# processing existing HTML links (adding target="_blank", stripping tracking parameters),
+# and removing tracking parameters from URLs.
+class Raif::Utils::HtmlFragmentProcessor
+  # List of common tracking parameters to remove from URLs
+  TRACKING_PARAMS = %w[
+    utm_source
+    utm_medium
+    utm_campaign
+    utm_term
+    utm_content
+    utm_id
+  ]
+
+  class << self
+    # Cleans and sanitizes an HTML fragment by removing empty text nodes and dangerous content.
+    #
+    # @param html [String, Nokogiri::HTML::DocumentFragment] The HTML content to clean
+    # @param allowed_tags [Array<String>, nil] Array of allowed HTML tags. Defaults to Rails HTML5 safe list
+    # @param allowed_attributes [Array<String>, nil] Array of allowed HTML attributes. Defaults to Rails HTML5 safe list
+    # @return [String] Cleaned and sanitized HTML string
+    #
+    # @example
+    #   clean_html_fragment("<script>alert('xss')</script><p>Safe content</p>")
+    #   # => "<p>Safe content</p>"
+    #
+    # @example With custom allowed tags
+    #   clean_html_fragment("<p>Para</p><div>Div</div>", allowed_tags: %w[p])
+    #   # => "<p>Para</p>Div"
+    def clean_html_fragment(html, allowed_tags: nil, allowed_attributes: nil)
+      fragment = html.is_a?(Nokogiri::HTML::DocumentFragment) ? html : Nokogiri::HTML.fragment(html)
+
+      fragment.traverse do |node|
+        if node.text? && node.text.strip.empty?
+          node.remove
+        end
+      end
+
+      allowed_tags = allowed_tags.presence || Rails::HTML5::SafeListSanitizer.allowed_tags
+      allowed_attributes = allowed_attributes.presence || Rails::HTML5::SafeListSanitizer.allowed_attributes
+
+      ActionController::Base.helpers.sanitize(fragment.to_html, tags: allowed_tags, attributes: allowed_attributes).strip
+    end
+
+    # Converts markdown-style links to HTML anchor tags with target="_blank" and rel="noopener".
+    #
+    # Converts [text](url) format to <a href="url" target="_blank" rel="noopener">text</a>.
+    # Also strips tracking parameters from the URLs.
+    #
+    # @param text [String] The text content that may contain markdown links
+    # @return [String] HTML with markdown links converted to anchor tags
+    #
+    # @example
+    #   convert_markdown_links_to_html("Check out [Google](https://google.com) for search.")
+    #   # => 'Check out <a href="https://google.com" target="_blank" rel="noopener">Google</a> for search.'
+    #
+    # @example With tracking parameters
+    #   convert_markdown_links_to_html("[Example](https://example.com?utm_source=test&param=keep)")
+    #   # => '<a href="https://example.com?param=keep" target="_blank" rel="noopener">Example</a>'
+    def convert_markdown_links_to_html(text)
+      # Convert markdown links [text](url) to HTML links <a href="url" target="_blank" rel="noopener">text</a>
+      text.gsub(/\[([^\]]*)\]\(([^)]+)\)/) do |_match|
+        text = ::Regexp.last_match(1)
+        url = ::Regexp.last_match(2)
+        clean_url = strip_tracking_parameters(url)
+        %(<a href="#{CGI.escapeHTML(clean_url)}" target="_blank" rel="noopener">#{CGI.escapeHTML(text)}</a>)
+      end
+    end
+
+    # Processes existing HTML links by optionally adding target="_blank" and stripping tracking parameters.
+    #
+    # This method provides fine-grained control over link processing with configurable options
+    # for both target="_blank" addition and tracking parameter removal.
+    #
+    # @param html [String, Nokogiri::HTML::DocumentFragment] The HTML content containing links to process
+    # @param add_target_blank [Boolean] Whether to add target="_blank" and rel="noopener" to links (required)
+    # @param strip_tracking_parameters [Boolean] Whether to remove tracking parameters from URLs (required)
+    # @return [String] Processed HTML with modified links
+    #
+    # @example Default behavior (adds target="_blank" and strips tracking params)
+    #   process_links('<a href="https://example.com?utm_source=test">Link</a>', add_target_blank: true, strip_tracking_parameters: true)
+    #   # => '<a href="https://example.com" target="_blank" rel="noopener">Link</a>'
+    #
+    # @example Only strip tracking parameters
+    #   process_links(html, add_target_blank: false, strip_tracking_parameters: true)
+    #   # => '<a href="https://example.com">Link</a>'
+    #
+    # @example Only add target="_blank"
+    #   process_links(html, add_target_blank: true, strip_tracking_parameters: false)
+    #   # => '<a href="https://example.com?utm_source=test" target="_blank" rel="noopener">Link</a>'
+    #
+    # @example No processing
+    #   process_links(html, add_target_blank: false, strip_tracking_parameters: false)
+    #   # => Original HTML unchanged
+    def process_links(html, add_target_blank:, strip_tracking_parameters:)
+      fragment = html.is_a?(Nokogiri::HTML::DocumentFragment) ? html : Nokogiri::HTML.fragment(html)
+
+      fragment.css("a").each do |link|
+        if add_target_blank
+          link["target"] = "_blank"
+          link["rel"] = "noopener"
+        end
+
+        if strip_tracking_parameters
+          link["href"] = strip_tracking_parameters(link["href"])
+        end
+      end
+
+      fragment.to_html
+    end
+
+    # Removes tracking parameters (UTM parameters) from a URL.
+    #
+    # Preserves all non-tracking query parameters and handles various URL formats including
+    # relative URLs, absolute URLs, and malformed URLs gracefully.
+    #
+    # @param url [String] The URL to clean
+    # @return [String] URL with tracking parameters removed, or original URL if parsing fails
+    #
+    # @example
+    #   strip_tracking_parameters("https://example.com?utm_source=google&page=1")
+    #   # => "https://example.com?page=1"
+    #
+    # @example Removes all tracking parameters
+    #   strip_tracking_parameters("https://example.com?utm_source=test&utm_medium=cpc")
+    #   # => "https://example.com"
+    #
+    # @example Preserves fragments
+    #   strip_tracking_parameters("https://example.com?utm_source=test&page=1#section")
+    #   # => "https://example.com?page=1#section"
+    #
+    # @example Handles relative URLs
+    #   strip_tracking_parameters("/path?utm_source=test&param=keep")
+    #   # => "/path?param=keep"
+    def strip_tracking_parameters(url)
+      return url if url.blank?
+      return url unless url.include?("?")
+
+      begin
+        uri = URI.parse(url)
+        return url unless uri.query
+
+        # Only process URLs that have a valid scheme and host, or are relative URLs
+        unless uri.scheme || url.start_with?("/", "#")
+          return url
+        end
+
+        # Parse query parameters and filter out tracking ones
+        params = URI.decode_www_form(uri.query)
+        clean_params = params.reject { |param, _| TRACKING_PARAMS.include?(param.downcase) }
+
+        # Rebuild the URL
+        uri.query = if clean_params.empty?
+          nil
+        else
+          URI.encode_www_form(clean_params)
+        end
+
+        uri.to_s
+      rescue URI::InvalidURIError
+        # If URL parsing fails, return the original URL
+        url
+      end
+    end
+  end
+end

data/lib/raif/utils.rb

@@ -3,4 +3,5 @@
 module Raif::Utils
   require "raif/utils/readable_content_extractor"
   require "raif/utils/html_to_markdown_converter"
+  require "raif/utils/html_fragment_processor"
 end

data/lib/raif/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Raif
-  VERSION = "1.1.0"
+  VERSION = "1.2.1.pre"
 end

data/lib/raif.rb

@@ -9,8 +9,10 @@ require "raif/utils"
 require "raif/llm_registry"
 require "raif/embedding_model_registry"
 require "raif/json_schema_builder"
+require "raif/migration_checker"
 
 require "faraday"
+require "event_stream_parser"
 require "json-schema"
 require "loofah"
 require "pagy"

data/spec/support/complex_test_tool.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class Raif::ModelTools::ComplexTestTool < Raif::ModelTool
+  tool_arguments_schema do
+    string :title, description: "The title of the operation", minLength: 3
+
+    object :settings, description: "Configuration settings" do
+      boolean :enabled, description: "Whether the tool is enabled"
+      integer :priority, description: "Priority level (1-10)", minimum: 1, maximum: 10
+      array :tags, description: "Associated tags" do
+        items type: "string"
+      end
+    end
+
+    array :products, description: "List of products" do
+      object do
+        integer :id, description: "Product identifier"
+        string :name, description: "Product name"
+        number :price, description: "Product price", minimum: 0
+      end
+    end
+  end
+
+  example_model_invocation do
+    {
+      "name" => tool_name,
+      "arguments" => {
+        "title" => "Daily Inventory Update",
+        "settings" => {
+          "enabled" => true,
+          "priority" => 5,
+          "tags" => ["inventory", "daily-update", "retail"]
+        },
+        "items" => [
+          { "id" => 101, "name" => "Wireless Mouse", "price" => 25.99 },
+          { "id" => 102, "name" => "Keyboard", "price" => 45.0 },
+          { "id" => 103, "name" => "USB-C Cable", "price" => 9.99 }
+        ]
+      }
+    }
+  end
+
+  tool_description do
+    "An example tool demonstrating complex schema capabilities"
+  end
+
+  def self.process_invocation(tool_invocation)
+    # This would be the actual implementation
+    # For demonstration purposes, just echo back the arguments
+    tool_invocation.update!(
+      result: {
+        message: "Received complex example request",
+        arguments: tool_invocation.tool_arguments
+      }
+    )
+
+    tool_invocation.result
+  end
+
+  def self.observation_for_invocation(tool_invocation)
+    return "No results" unless tool_invocation.result.present?
+
+    "Successfully processed request for: #{tool_invocation.tool_arguments["title"]}"
+  end
+end

data/spec/support/rspec_helpers.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Raif
+  module RspecHelpers
+
+    def stubbed_llm(llm_model_key, &block)
+      test_llm = Raif.llm(llm_model_key.to_sym)
+
+      allow(test_llm).to receive(:perform_model_completion!) do |model_completion|
+        result = block.call(model_completion.messages, model_completion)
+        model_completion.raw_response = result if result.is_a?(String)
+        model_completion.completion_tokens = rand(100..2000)
+        model_completion.prompt_tokens = rand(100..2000)
+        model_completion.total_tokens = model_completion.completion_tokens + model_completion.prompt_tokens
+        model_completion.save!
+
+        model_completion
+      end
+
+      test_llm
+    end
+
+    def stub_raif_task(task, &block)
+      allow(Raif.config).to receive(:llm_api_requests_enabled){ true }
+
+      if task.is_a?(Raif::Task)
+        allow(task).to receive(:llm){ stubbed_llm(task.llm_model_key, &block) }
+      else
+        allow_any_instance_of(task).to receive(:llm) do |task_instance|
+          stubbed_llm(task_instance.llm_model_key, &block)
+        end
+      end
+    end
+
+    def stub_raif_conversation(conversation, &block)
+      allow(Raif.config).to receive(:llm_api_requests_enabled){ true }
+
+      if conversation.is_a?(Raif::Conversation)
+        allow(conversation).to receive(:llm){ stubbed_llm(conversation.llm_model_key, &block) }
+      else
+        allow_any_instance_of(conversation).to receive(:llm) do |conversation_instance|
+          stubbed_llm(conversation_instance.llm_model_key, &block)
+        end
+      end
+    end
+
+    def stub_raif_agent(agent, &block)
+      allow(Raif.config).to receive(:llm_api_requests_enabled){ true }
+
+      if agent.is_a?(Raif::Agent)
+        allow(agent).to receive(:llm){ stubbed_llm(agent.llm_model_key, &block) }
+      else
+        allow_any_instance_of(agent).to receive(:llm) do |agent_instance|
+          stubbed_llm(agent_instance.llm_model_key, &block)
+        end
+      end
+    end
+
+    def stub_raif_llm(llm, &block)
+      llm.chat_handler = block
+      allow(Raif.config).to receive(:llm_api_requests_enabled){ true }
+      llm
+    end
+
+  end
+end

data/spec/support/test_conversation.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Raif::TestConversation < Raif::Conversation
+
+  before_create :populate_available_model_tools
+
+  def populate_available_model_tools
+    self.available_model_tools = [
+      "Raif::TestModelTool",
+      "Raif::ModelTools::WikipediaSearch",
+    ]
+  end
+
+  def process_model_response_message(message:, entry:)
+    message.gsub("jerk", "[REDACTED]")
+  end
+
+end

data/spec/support/test_embedding_model.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Raif
+  module EmbeddingModels
+    class Test < Raif::EmbeddingModel
+      attr_accessor :embedding_handler
+
+      def generate_embedding!(input, dimensions: nil)
+        if input.is_a?(Array)
+          input.map { |text| generate_test_embedding!(text, dimensions:) }
+        else
+          generate_test_embedding!(input, dimensions:)
+        end
+      end
+
+      def generate_test_embedding!(input, dimensions: nil)
+        if embedding_handler.present?
+          embedding_handler.call(input, dimensions)
+        else
+          dimensions ||= default_output_vector_size
+          Array.new(dimensions) { rand(-1.0..1.0) }
+        end
+      end
+
+    end
+  end
+end

data/spec/support/test_llm.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Raif
+  module Llms
+    class TestLlm < Raif::Llm
+      include Raif::Concerns::Llms::OpenAiCompletions::MessageFormatting
+
+      attr_accessor :chat_handler
+
+      def perform_model_completion!(model_completion)
+        result = chat_handler.call(model_completion.messages, model_completion)
+        model_completion.raw_response = result if result.is_a?(String)
+        model_completion.completion_tokens = rand(100..2000)
+        model_completion.prompt_tokens = rand(100..2000)
+        model_completion.total_tokens = model_completion.completion_tokens + model_completion.prompt_tokens
+        model_completion.save!
+
+        model_completion
+      end
+    end
+  end
+end

data/spec/support/test_model_tool.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Raif::TestModelTool < Raif::ModelTool
+  tool_arguments_schema do
+    array :items do
+      object do
+        string :title, description: "The title of the item"
+        string :description
+      end
+    end
+  end
+
+  example_model_invocation do
+    {
+      "name": tool_name,
+      "arguments": { "items": [{ "title": "foo", "description": "bar" }] }
+    }
+  end
+
+  def self.process_invocation(tool_arguments)
+  end
+
+  tool_description do
+    "Mock Tool Description"
+  end
+
+  def self.observation_for_invocation(tool_invocation)
+    return if tool_invocation.result.blank?
+
+    "Mock Observation for #{tool_invocation.id}. Result was: #{tool_invocation.result["status"]}"
+  end
+end

data/spec/support/test_task.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class Raif::TestTask < Raif::Task
+  llm_temperature 0.5
+
+  def build_prompt
+    "Tell me a joke"
+  end
+
+  def build_system_prompt
+    super + "\nYou are also good at telling jokes."
+  end
+end
+
+class Raif::TestJsonTask < Raif::Task
+  llm_response_format :json
+  llm_temperature 0.75
+
+  json_response_schema do
+    string :joke
+    string :answer
+  end
+
+  def build_prompt
+    "Tell me a joke"
+  end
+
+  def build_system_prompt
+    super + "\nYou are also good at telling jokes. Your response should be a JSON object with the following keys: joke, answer."
+  end
+end
+
+class Raif::TestHtmlTask < Raif::Task
+  llm_response_format :html
+  llm_response_allowed_tags %w[p b i u s a]
+  llm_response_allowed_attributes %w[style href]
+
+  def build_prompt
+    "Tell me a joke"
+  end
+
+  def build_system_prompt
+    super + "\nYou are also good at telling jokes. Your response should be an HTML snippet that is formatted with basic HTML tags."
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: raif
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.1.pre
 platform: ruby
 authors:
 - Ben Roesch
 - Brian Leslie
 bindir: bin
 cert_chain: []
-date: 2025-05-23 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-bedrockruntime
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: event_stream_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -175,18 +189,26 @@ files:
 - app/models/raif/concerns/llm_response_parsing.rb
 - app/models/raif/concerns/llm_temperature.rb
 - app/models/raif/concerns/llms/anthropic/message_formatting.rb
-- app/models/raif/concerns/llms/bedrock_claude/message_formatting.rb
+- app/models/raif/concerns/llms/anthropic/tool_formatting.rb
+- app/models/raif/concerns/llms/bedrock/message_formatting.rb
+- app/models/raif/concerns/llms/bedrock/tool_formatting.rb
 - app/models/raif/concerns/llms/message_formatting.rb
-- app/models/raif/concerns/llms/open_ai/message_formatting.rb
+- app/models/raif/concerns/llms/open_ai/json_schema_validation.rb
+- app/models/raif/concerns/llms/open_ai_completions/message_formatting.rb
+- app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb
+- app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb
+- app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb
 - app/models/raif/conversation.rb
 - app/models/raif/conversation_entry.rb
 - app/models/raif/embedding_model.rb
-- app/models/raif/embedding_models/bedrock_titan.rb
+- app/models/raif/embedding_models/bedrock.rb
 - app/models/raif/embedding_models/open_ai.rb
 - app/models/raif/llm.rb
 - app/models/raif/llms/anthropic.rb
-- app/models/raif/llms/bedrock_claude.rb
-- app/models/raif/llms/open_ai.rb
+- app/models/raif/llms/bedrock.rb
+- app/models/raif/llms/open_ai_base.rb
+- app/models/raif/llms/open_ai_completions.rb
+- app/models/raif/llms/open_ai_responses.rb
 - app/models/raif/llms/open_router.rb
 - app/models/raif/model_completion.rb
 - app/models/raif/model_file_input.rb
@@ -195,7 +217,15 @@ files:
 - app/models/raif/model_tool_invocation.rb
 - app/models/raif/model_tools/agent_final_answer.rb
 - app/models/raif/model_tools/fetch_url.rb
+- app/models/raif/model_tools/provider_managed/base.rb
+- app/models/raif/model_tools/provider_managed/code_execution.rb
+- app/models/raif/model_tools/provider_managed/image_generation.rb
+- app/models/raif/model_tools/provider_managed/web_search.rb
 - app/models/raif/model_tools/wikipedia_search.rb
+- app/models/raif/streaming_responses/anthropic.rb
+- app/models/raif/streaming_responses/bedrock.rb
+- app/models/raif/streaming_responses/open_ai_completions.rb
+- app/models/raif/streaming_responses/open_ai_responses.rb
 - app/models/raif/task.rb
 - app/models/raif/user_tool_invocation.rb
 - app/views/layouts/raif/admin.html.erb
@@ -218,6 +248,7 @@ files:
 - app/views/raif/admin/tasks/_task.html.erb
 - app/views/raif/admin/tasks/index.html.erb
 - app/views/raif/admin/tasks/show.html.erb
+- app/views/raif/conversation_entries/_citations.html.erb
 - app/views/raif/conversation_entries/_conversation_entry.html.erb
 - app/views/raif/conversation_entries/_form.html.erb
 - app/views/raif/conversation_entries/_form_with_available_tools.html.erb
@@ -242,6 +273,9 @@ files:
 - db/migrate/20250424232946_add_created_at_indexes.rb
 - db/migrate/20250502155330_add_status_indexes_to_raif_tasks.rb
 - db/migrate/20250507155314_add_retry_count_to_raif_model_completions.rb
+- db/migrate/20250527213016_add_response_id_and_response_array_to_model_completions.rb
+- db/migrate/20250603140622_add_citations_to_raif_model_completions.rb
+- db/migrate/20250603202013_add_stream_response_to_raif_model_completions.rb
 - lib/generators/raif/agent/agent_generator.rb
 - lib/generators/raif/agent/templates/agent.rb.tt
 - lib/generators/raif/agent/templates/application_agent.rb.tt
@@ -268,16 +302,26 @@ files:
 - lib/raif/errors/invalid_model_image_input_error.rb
 - lib/raif/errors/invalid_user_tool_type_error.rb
 - lib/raif/errors/open_ai/json_schema_error.rb
+- lib/raif/errors/streaming_error.rb
 - lib/raif/errors/unsupported_feature_error.rb
 - lib/raif/json_schema_builder.rb
 - lib/raif/languages.rb
 - lib/raif/llm_registry.rb
+- lib/raif/migration_checker.rb
 - lib/raif/rspec.rb
 - lib/raif/utils.rb
+- lib/raif/utils/html_fragment_processor.rb
 - lib/raif/utils/html_to_markdown_converter.rb
 - lib/raif/utils/readable_content_extractor.rb
 - lib/raif/version.rb
 - lib/tasks/raif_tasks.rake
+- spec/support/complex_test_tool.rb
+- spec/support/rspec_helpers.rb
+- spec/support/test_conversation.rb
+- spec/support/test_embedding_model.rb
+- spec/support/test_llm.rb
+- spec/support/test_model_tool.rb
+- spec/support/test_task.rb
 homepage: https://github.com/cultivatelabs/raif
 licenses:
 - MIT
@@ -300,7 +344,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Raif (Ruby AI Framework) is a Rails engine that helps you add AI-powered
   features to your Rails apps, such as tasks, conversations, and agents.