rag-ruby 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18916baa3b84e6a947a1fa8411658475e4608ff4a127d681ca9b29332d669e81
-  data.tar.gz: f18c5e4d77640750e570ae4762122d0b687b73b2d12731e11add7b34d00b9d8c
+  metadata.gz: 2b029b1c946d1f57c25b9098864b13297520b56e8970e4882e54684a09f2bbc1
+  data.tar.gz: ff6367e701579b6e93bca9d9c4c2f58d686d9af400e974bd1627936cd80f95e5
 SHA512:
-  metadata.gz: f397468f410d2475d1d59557749e83092c6b65b6f844b4169fff2774dbaac436a7eca5d9e667a35f7750937722d4bf92f97ec550b1919fd0deadd1857a9b1531
-  data.tar.gz: 54d733930d3ba8c0a241240318fbe58c4e8dd1750e3e9f9dc5969c86b9280721386fbe24f9379c47f06a598871950e5d97e8e6b42a13c72d6f8d89da40d2f1d9
+  metadata.gz: 266712e7a269d2de55c13cc64b47309501692c2b3476d757095708b1ba0bcbf107cb278e20cdab527e216ba97776ae957927827a2d9ea8e32dc9a6b6b84caa1d
+  data.tar.gz: 91e1297be5d3e35dd7d9b119391e38a1c4507727144cfd88f5ade6ba334a6d4103a2b70c4a6869e9262899aab066e6323c36ce7ea234a0486ff0f11b0fb2c857

data/lib/rag_ruby/configuration.rb

@@ -26,13 +26,16 @@ module RagRuby
     }.freeze
 
     attr_accessor :loader_instance, :embedder_instance, :store_instance, :generator_instance,
-                  :chunk_size, :chunk_overlap, :chunk_strategy
+                  :chunk_size, :chunk_overlap, :chunk_strategy,
+                  :http_timeout, :read_timeout
 
     def initialize
       @callbacks = Hash.new { |h, k| h[k] = [] }
       @chunk_size = 1000
       @chunk_overlap = 200
       @chunk_strategy = :recursive_character
+      @http_timeout = 30
+      @read_timeout = 60
     end
 
     def loader(name, **opts)

data/lib/rag_ruby/embedders/cohere.rb

@@ -39,7 +39,10 @@ module RagRuby
           raise "Cohere API error (#{response.code}): #{response.body}"
         end
 
-        JSON.parse(response.body)["embeddings"]
+        parsed = JSON.parse(response.body)
+        embeddings = parsed["embeddings"]
+        raise RagRuby::Error, "No embeddings in Cohere response" if embeddings.nil? || embeddings.empty?
+        embeddings
       end
 
       def dimension

data/lib/rag_ruby/embedders/openai.rb

@@ -40,24 +40,44 @@ module RagRuby
       private
 
       def request(input)
-        uri = URI.parse(ENDPOINT)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = true
-        http.open_timeout = 30
-        http.read_timeout = 60
+        request_with_retry do
+          uri = URI.parse(ENDPOINT)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.open_timeout = 30
+          http.read_timeout = 60
 
-        req = Net::HTTP::Post.new(uri)
-        req["Authorization"] = "Bearer #{@api_key}"
-        req["Content-Type"] = "application/json"
-        req.body = JSON.generate(model: @model, input: input)
+          req = Net::HTTP::Post.new(uri)
+          req["Authorization"] = "Bearer #{@api_key}"
+          req["Content-Type"] = "application/json"
+          req.body = JSON.generate(model: @model, input: input)
 
-        response = http.request(req)
+          response = http.request(req)
 
-        unless response.is_a?(Net::HTTPSuccess)
-          raise "OpenAI API error (#{response.code}): #{response.body}"
+          unless response.is_a?(Net::HTTPSuccess)
+            raise "OpenAI API error (#{response.code}): #{response.body}"
+          end
+
+          JSON.parse(response.body)
         end
+      end
+
+      def request_with_retry(max_retries: 3)
+        retries = 0
+        begin
+          yield
+        rescue => e
+          retries += 1
+          if retries <= max_retries && retryable?(e)
+            sleep(2 ** (retries - 1))
+            retry
+          end
+          raise
+        end
+      end
 
-        JSON.parse(response.body)
+      def retryable?(e)
+        e.message.match?(/429|500|502|503/)
       end
     end
   end

data/lib/rag_ruby/generators/openai.rb

@@ -26,34 +26,57 @@ module RagRuby
           temperature: temperature
         }
 
-        uri = URI.parse(ENDPOINT)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = true
-        http.open_timeout = 30
-        http.read_timeout = 120
+        request_with_retry do
+          uri = URI.parse(ENDPOINT)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.open_timeout = 30
+          http.read_timeout = 120
 
-        req = Net::HTTP::Post.new(uri)
-        req["Authorization"] = "Bearer #{@api_key}"
-        req["Content-Type"] = "application/json"
-        req.body = JSON.generate(body)
+          req = Net::HTTP::Post.new(uri)
+          req["Authorization"] = "Bearer #{@api_key}"
+          req["Content-Type"] = "application/json"
+          req.body = JSON.generate(body)
 
-        response = http.request(req)
+          response = http.request(req)
 
-        unless response.is_a?(Net::HTTPSuccess)
-          raise "OpenAI API error (#{response.code}): #{response.body}"
-        end
+          unless response.is_a?(Net::HTTPSuccess)
+            raise "OpenAI API error (#{response.code}): #{response.body}"
+          end
 
-        data = JSON.parse(response.body)
-        text = data.dig("choices", 0, "message", "content")
-        usage = data["usage"] || {}
+          data = JSON.parse(response.body)
+          text = data.dig("choices", 0, "message", "content")
+          raise RagRuby::Error, "Empty response from OpenAI" if text.nil?
+          usage = data["usage"] || {}
 
-        {
-          text: text,
-          tokens_used: {
-            prompt: usage["prompt_tokens"],
-            completion: usage["completion_tokens"]
+          {
+            text: text,
+            tokens_used: {
+              prompt: usage["prompt_tokens"],
+              completion: usage["completion_tokens"]
+            }
           }
-        }
+        end
+      end
+
+      private
+
+      def request_with_retry(max_retries: 3)
+        retries = 0
+        begin
+          yield
+        rescue => e
+          retries += 1
+          if retries <= max_retries && retryable?(e)
+            sleep(2 ** (retries - 1))
+            retry
+          end
+          raise
+        end
+      end
+
+      def retryable?(e)
+        e.message.match?(/429|500|502|503/)
       end
     end
   end

data/lib/rag_ruby/loaders/url.rb

@@ -31,7 +31,7 @@ module RagRuby
       private
 
       def fetch(uri, redirect_limit: 5)
-        raise "Too many redirects" if redirect_limit == 0
+        raise RagRuby::Error, "Too many redirects for #{uri}" if redirect_limit == 0
 
         http = Net::HTTP.new(uri.host, uri.port)
         http.use_ssl = uri.scheme == "https"
@@ -43,7 +43,9 @@ module RagRuby
 
         case response
         when Net::HTTPRedirection
-          fetch(URI.parse(response["location"]), redirect_limit: redirect_limit - 1)
+          location = response["location"]
+          raise RagRuby::Error, "Redirect with no location header" if location.nil?
+          fetch(URI.parse(location), redirect_limit: redirect_limit - 1)
         when Net::HTTPSuccess
           response
         else

data/lib/rag_ruby/pipeline.rb

@@ -134,6 +134,9 @@ module RagRuby
     def embed_chunks(chunks)
       texts = chunks.map(&:text)
       embeddings = @config.embedder_instance.embed_batch(texts)
+      if embeddings.length != chunks.length
+        raise RagRuby::Error, "Embedding count (#{embeddings.length}) doesn't match chunk count (#{chunks.length})"
+      end
       chunks.each_with_index do |chunk, i|
         chunk.embedding = embeddings[i]
       end
@@ -150,8 +153,9 @@ module RagRuby
     def build_context(sources, max_chars: 12000)
       context = ""
       sources.each do |source|
-        candidate = context + "\n---\n" + source.text
-        break if candidate.length > max_chars
+        separator = context.empty? ? "" : "\n---\n"
+        candidate = context + separator + source.text
+        break if candidate.length > max_chars && !context.empty?
         context = candidate
       end
       context.strip

data/lib/rag_ruby/rails/indexable.rb

@@ -47,8 +47,10 @@ module RagRuby
     end
 
     def rag_remove_record
-      # Remove from store by metadata filter
-      # Implementation depends on store capabilities
+      store = self.class.rag_pipeline&.config&.store_instance
+      if store && store.respond_to?(:delete)
+        store.delete(id.to_s)
+      end
     end
 
     class InlineLoader < Loaders::Base

data/lib/rag_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RagRuby
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/rag_ruby.rb

@@ -27,6 +27,9 @@ require_relative "rag_ruby/pipeline"
 
 module RagRuby
   class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class EmbeddingError < Error; end
+  class GenerationError < Error; end
 
   class << self
     def pipeline

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rag-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Johannes Dwi Cahyo