rager 0.6.0 → 0.8.0

data/lib/rager/context.rb

@@ -15,18 +15,46 @@ module Rager
     sig { returns(T.nilable(String)) }
     attr_reader :name
 
+    sig { returns(T::Array[String]) }
+    attr_reader :tags
+
     sig { returns(Integer) }
     attr_reader :max_retries
 
-    sig { returns(Float) }
+    sig { returns(Numeric) }
     attr_reader :backoff
 
-    sig { params(id: T.nilable(String), name: T.nilable(String), max_retries: T.nilable(Integer), backoff: T.nilable(Float)).void }
-    def initialize(id: nil, name: nil, max_retries: nil, backoff: nil)
+    sig {
+      params(
+        id: T.nilable(String),
+        name: T.nilable(String),
+        tags: T::Array[String],
+        max_retries: T.nilable(Integer),
+        backoff: T.nilable(Numeric)
+      ).void
+    }
+    def initialize(id: nil, name: nil, tags: [], max_retries: nil, backoff: nil)
       @id = T.let(id || SecureRandom.uuid, String)
       @name = T.let(name, T.nilable(String))
+      @tags = T.let(tags, T::Array[String])
       @max_retries = T.let(max_retries || 0, Integer)
-      @backoff = T.let(backoff || 1.0, Float)
+      @backoff = T.let(backoff || 1.0, Numeric)
+    end
+
+    sig do
+      params(
+        max_retries: T.nilable(Integer),
+        backoff: T.nilable(Numeric)
+      ).returns(Rager::Context)
+    end
+    def with_options(max_retries: nil, backoff: nil)
+      Context.new(
+        id: @id,
+        name: @name,
+        tags: @tags,
+        max_retries: max_retries || @max_retries,
+        backoff: backoff || @backoff
+      )
     end
 
     sig do
@@ -36,26 +64,32 @@ module Rager
       ).returns(Rager::Result)
     end
     def chat(messages, **kwargs)
-      execute(
-        Rager::Operation::Chat,
-        Rager::Chat::Options,
-        kwargs,
-        messages
-      ) { |options, normalized_input|
-        final_input = if normalized_input.is_a?(String)
-          [
-            Rager::Chat::Message.new(
-              role: Rager::Chat::MessageRole::User,
-              content: normalized_input
-            )
-          ]
-        else
-          normalized_input
-        end
+      context_options = Rager::ContextOptions.new(**Rager::ContextOptions.extract_kwargs(kwargs))
+      options = Rager::Chat::Options.new(**kwargs)
+      options.validate
 
-        provider = Rager::Providers.get_provider(:chat, options.provider, options)
-        provider.chat(final_input, options)
-      }
+      normalized_input, input_ids = normalize(messages, context_options.input_ids)
+
+      if normalized_input.is_a?(String)
+        normalized_input = [
+          Rager::Chat::Message.new(
+            role: Rager::Chat::MessageRole::User,
+            content: normalized_input
+          )
+        ]
+      end
+
+      execute(
+        operation: Rager::Operation::Chat,
+        input: normalized_input,
+        options: options,
+        context_options: context_options,
+        input_ids: input_ids
+      ) do |input, opts|
+        Rager::Providers
+          .get_provider(Rager::Operation::Chat, opts.provider, opts)
+          .chat(input, opts)
+      end
     end
 
     sig do
@@ -65,57 +99,77 @@ module Rager
       ).returns(Rager::Result)
     end
     def embed(text, **kwargs)
-      execute(
-        Rager::Operation::Embed,
-        Rager::Embed::Options,
-        kwargs,
-        text
-      ) { |options, normalized_input|
-        final_input = if normalized_input.is_a?(String)
-          [normalized_input]
-        else
-          normalized_input
-        end
+      context_options = Rager::ContextOptions.new(**Rager::ContextOptions.extract_kwargs(kwargs))
+      options = Rager::Embed::Options.new(**kwargs)
+      options.validate
 
-        provider = Rager::Providers.get_provider(:embed, options.provider, options)
-        provider.embed(final_input, options)
-      }
+      normalized_input, input_ids = normalize(text, context_options.input_ids)
+
+      normalized_input = [normalized_input] if normalized_input.is_a?(String)
+
+      execute(
+        operation: Rager::Operation::Embed,
+        input: normalized_input,
+        options: options,
+        context_options: context_options,
+        input_ids: input_ids
+      ) do |input, opts|
+        Rager::Providers
+          .get_provider(Rager::Operation::Embed, opts.provider, opts)
+          .embed(input, opts)
+      end
     end
 
     sig do
       params(
-        prompt: T.any(Rager::Types::ImageGenInput, Rager::Result),
+        prompt: T.any(Rager::Types::ImageInput, Rager::Result),
         kwargs: T.untyped
       ).returns(Rager::Result)
     end
-    def image_gen(prompt, **kwargs)
+    def image(prompt, **kwargs)
+      context_options = Rager::ContextOptions.new(**Rager::ContextOptions.extract_kwargs(kwargs))
+      options = Rager::Image::Options.new(**kwargs)
+      options.validate
+
+      normalized_input, input_ids = normalize(prompt, context_options.input_ids)
+
       execute(
-        Rager::Operation::ImageGen,
-        Rager::ImageGen::Options,
-        kwargs,
-        prompt
-      ) { |options, normalized_input|
-        provider = Rager::Providers.get_provider(:image_gen, options.provider, options)
-        provider.image_gen(T.cast(normalized_input, Rager::Types::ImageGenInput), options)
-      }
+        operation: Rager::Operation::Image,
+        input: normalized_input,
+        options: options,
+        context_options: context_options,
+        input_ids: input_ids
+      ) do |input, opts|
+        Rager::Providers
+          .get_provider(Rager::Operation::Image, opts.provider, opts)
+          .image(input, opts)
+      end
     end
 
     sig do
       params(
-        prompt: T.any(Rager::Types::MeshGenInput, Rager::Result),
+        prompt: T.any(Rager::Types::MeshInput, Rager::Result),
         kwargs: T.untyped
       ).returns(Rager::Result)
     end
-    def mesh_gen(prompt, **kwargs)
+    def mesh(prompt, **kwargs)
+      context_options = Rager::ContextOptions.new(**Rager::ContextOptions.extract_kwargs(kwargs))
+      options = Rager::Mesh::Options.new(**kwargs)
+      options.validate
+
+      normalized_input, input_ids = normalize(prompt, context_options.input_ids)
+
       execute(
-        Rager::Operation::MeshGen,
-        Rager::MeshGen::Options,
-        kwargs,
-        prompt
-      ) { |options, normalized_input|
-        provider = Rager::Providers.get_provider(:mesh_gen, options.provider, options)
-        provider.mesh_gen(T.cast(normalized_input, Rager::Types::MeshGenInput), options)
-      }
+        operation: Rager::Operation::Mesh,
+        input: normalized_input,
+        options: options,
+        context_options: context_options,
+        input_ids: input_ids
+      ) do |input, opts|
+        Rager::Providers
+          .get_provider(Rager::Operation::Mesh, opts.provider, opts)
+          .mesh(input, opts)
+      end
     end
 
     sig do
@@ -126,16 +180,29 @@ module Rager
       ).returns(Rager::Result)
     end
     def rerank(query, documents, **kwargs)
+      context_options = Rager::ContextOptions.new(**Rager::ContextOptions.extract_kwargs(kwargs))
+      options = Rager::Rerank::Options.new(**kwargs)
+      options.validate
+
+      normalized_query, input_ids = normalize(query, context_options.input_ids)
+      normalized_documents, input_ids = normalize(documents, input_ids)
+
+      rerank_input = Rager::Rerank::Input.new(
+        query: normalized_query,
+        documents: normalized_documents
+      )
+
       execute(
-        Rager::Operation::Rerank,
-        Rager::Rerank::Options,
-        kwargs,
-        {query: query, documents: documents}
-      ) { |options, normalized_input|
-        provider = Rager::Providers.get_provider(:rerank, options.provider, options)
-        input_hash = T.cast(normalized_input, T::Hash[Symbol, T.untyped])
-        provider.rerank(input_hash[:query], input_hash[:documents], options)
-      }
+        operation: Rager::Operation::Rerank,
+        input: rerank_input,
+        options: options,
+        context_options: context_options,
+        input_ids: input_ids
+      ) do |input, opts|
+        Rager::Providers
+          .get_provider(Rager::Operation::Rerank, opts.provider, opts)
+          .rerank(input, opts)
+      end
     end
 
     sig do
@@ -145,15 +212,23 @@ module Rager
       ).returns(Rager::Result)
     end
     def search(query, **kwargs)
+      context_options = Rager::ContextOptions.new(**Rager::ContextOptions.extract_kwargs(kwargs))
+      options = Rager::Search::Options.new(**kwargs)
+      options.validate
+
+      normalized_input, input_ids = normalize(query, context_options.input_ids)
+
       execute(
-        Rager::Operation::Search,
-        Rager::Search::Options,
-        kwargs,
-        query
-      ) { |options, normalized_input|
-        provider = Rager::Providers.get_provider(:search, options.provider, options)
-        provider.search(T.cast(normalized_input, Rager::Types::SearchInput), options)
-      }
+        operation: Rager::Operation::Search,
+        input: normalized_input,
+        options: options,
+        context_options: context_options,
+        input_ids: input_ids
+      ) do |input, opts|
+        Rager::Providers
+          .get_provider(Rager::Operation::Search, opts.provider, opts)
+          .search(input, opts)
+      end
     end
 
     sig do
@@ -164,147 +239,188 @@ module Rager
       ).returns(Rager::Result)
     end
     def template(template, variables, **kwargs)
+      context_options = Rager::ContextOptions.new(**Rager::ContextOptions.extract_kwargs(kwargs))
+      options = Rager::Template::Options.new(**kwargs)
+      options.validate
+
+      normalized_template, input_ids = normalize(template, context_options.input_ids)
+      normalized_variables, input_ids = normalize(variables, input_ids)
+
+      template_input = Rager::Template::Input.new(
+        template: normalized_template,
+        variables: normalized_variables
+      )
+
       execute(
-        Rager::Operation::Template,
-        Rager::Template::Options,
-        kwargs,
-        {template: template, variables: variables}
-      ) { |options, normalized_input|
-        provider = Rager::Providers.get_provider(:template, options.provider, options)
-        input_hash = T.cast(normalized_input, T::Hash[Symbol, T.untyped])
-        provider.template(input_hash[:template], input_hash[:variables], options)
+        operation: Rager::Operation::Template,
+        input: template_input,
+        options: options,
+        context_options: context_options,
+        input_ids: input_ids
+      ) do |input, opts|
+        Rager::Providers
+          .get_provider(Rager::Operation::Template, opts.provider, opts)
+          .template(input, opts)
+      end
+    end
+
+    sig do
+      params(
+        value: Float,
+        feedback: T.nilable(String),
+        tags: T::Array[String]
+      ).void
+    end
+    def outcome(value, feedback: nil, tags: [])
+      Rager::Utils::Http.log_remote(
+        "outcomes",
+        {
+          version: "1",
+          data: Rager::Outcome.new(
+            context_id: @id,
+            value: value,
+            feedback: feedback,
+            tags: tags
+          )
+        }.to_json
+      )
+    end
+
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def to_h
+      {
+        id: @id,
+        name: @name,
+        tags: @tags,
+        max_retries: @max_retries,
+        backoff: @backoff
       }
     end
 
+    sig { params(hash: T::Hash[Symbol, T.untyped], max_retries: T.nilable(Integer), backoff: T.nilable(Float)).returns(Rager::Context) }
+    def self.from_h(hash, max_retries: nil, backoff: nil)
+      new(
+        id: hash[:id],
+        name: hash[:name],
+        tags: hash[:tags],
+        max_retries: max_retries || hash[:max_retries],
+        backoff: backoff || hash[:backoff]
+      )
+    end
+
     private
 
     sig do
       params(
         operation: Rager::Operation,
-        options_struct: T::Class[Rager::Options],
-        kwargs: T.untyped,
-        input: T.any(Rager::Types::Input, Rager::Result, T::Hash[Symbol, T.untyped]),
-        block: T.proc.params(options: T.untyped, normalized_input: Rager::Types::Input).returns(T.untyped)
+        input: Rager::Types::Input,
+        options: Rager::Types::Options,
+        context_options: Rager::ContextOptions,
+        input_ids: T::Array[String],
+        block: T.proc.params(
+          input: Rager::Types::Input,
+          options: Rager::Types::Options
+        ).returns(Rager::Types::Output)
       ).returns(Rager::Result)
     end
-    def execute(operation, options_struct, kwargs, input, &block)
-      name = kwargs.delete(:name)
-      tags = kwargs.delete(:tags) || []
-
-      existing_input_ids = kwargs.delete(:input_ids) || []
-      normalized_input, new_input_ids = normalize_input(input)
-      input_ids = (existing_input_ids + new_input_ids).uniq
-
-      options = options_struct.new(**kwargs)
-      options.validate
-
+    def execute(operation:, input:, options:, context_options:, input_ids:, &block)
       start_time = Time.now
-      errors = []
       attempt = 0
+      errors = []
+      last_error = nil
 
       begin
-        output = yield(options, normalized_input)
-
-        Result.new(
-          id: SecureRandom.uuid,
-          context_id: @id,
-          operation: operation,
-          input: normalized_input,
-          output: output,
-          options: options,
-          start_time: start_time.to_i,
-          end_time: Time.now.to_i,
-          name: name,
-          context_name: @name,
-          tags: tags,
-          input_ids: input_ids,
-          errors: errors,
-          attempt: attempt
-        ).tap(&:log)
-      rescue => e
+        output = yield(input, options)
+      rescue Rager::Errors::HttpError => e
         errors << e.message
+        last_error = e
         attempt += 1
 
-        if attempt < @max_retries
-          delay = @backoff * 2**attempt
-          Rager::Utils::Http.sleep(delay)
+        if attempt <= @max_retries
+          jitter = @backoff * rand(0.0..1.0)
+          delay = @backoff * (2**(attempt - 1)) + jitter
+          Rager.config.logger.warn("Retrying in #{delay.round(2)}s after failure #{attempt}/#{@max_retries}: #{e.message}")
+
+          Rager::Utils::Runtime.sleep(delay)
+
           retry
         else
-          Result.new(
-            id: SecureRandom.uuid,
-            context_id: @id,
-            operation: operation,
-            input: normalized_input,
-            output: nil,
-            options: options,
-            start_time: start_time.to_i,
-            end_time: Time.now.to_i,
-            name: name,
-            context_name: @name,
-            tags: tags,
-            input_ids: input_ids,
-            errors: errors,
-            attempt: attempt
-          ).tap(&:log)
-
-          raise e
+          output = nil
         end
       end
+
+      result = Result.new(
+        id: SecureRandom.uuid,
+        context_id: @id,
+        operation: operation,
+        input: input,
+        output: output,
+        options: options,
+        start_time: start_time.to_i,
+        end_time: Time.now.to_i,
+        attempt: attempt,
+        errors: errors,
+        input_ids: input_ids.uniq,
+        name: context_options.name,
+        tags: context_options.tags,
+        context_name: @name,
+        context_tags: @tags
+      ).tap(&:log)
+
+      raise last_error if last_error
+
+      result
+    end
+
+    sig {
+      params(
+        input: T.untyped,
+        ids: T::Array[String]
+      ).returns([T.untyped, T::Array[String]])
+    }
+    def normalize(input, ids = [])
+      collected_ids = ids.to_set
+      visited = Set.new
+
+      normalized_input = normalize_item(input, collected_ids, visited)
+      [normalized_input, collected_ids.to_a]
     end
 
-    sig { params(input: T.untyped, ids: T::Array[String]).returns([T.untyped, T::Array[String]]) }
-    def normalize_input(input, ids: [])
-      case input
+    sig {
+      params(
+        item: T.untyped,
+        collected_ids: T::Set[String],
+        visited: T::Set[Integer]
+      ).returns(T.untyped)
+    }
+    def normalize_item(item, collected_ids, visited)
+      case item
       when Rager::Result
-        ids << input.id unless ids.include?(input.id)
-
-        materialized_output = input.mat
-        normalized_output = case materialized_output
-        when Rager::Search::Output
-          materialized_output.contents
-        when Rager::Rerank::Output
-          materialized_output.documents
-        else
-          materialized_output
-        end
+        collected_ids.add(item.id)
+        normalize_item(item.value, collected_ids, visited)
+      when Array
+        object_id = item.object_id
+        return item if visited.include?(object_id)
+        visited.add(object_id)
 
-        [normalized_output, ids]
+        item.map { |i| normalize_item(i, collected_ids, visited) }
       when Hash
-        normalized_hash = {}
-        input.each do |key, value|
-          normalized_value, ids = normalize_input(value, ids: ids)
-          normalized_hash[key] = normalized_value
-        end
-        [normalized_hash, ids]
-      when Array
-        normalized_array = []
-        input.each do |item|
-          normalized_item, ids = normalize_input(item, ids: ids)
-          normalized_array << normalized_item
-        end
-        [normalized_array, ids]
-      else
-        [input, ids]
-      end
-    end
+        object_id = item.object_id
+        return item if visited.include?(object_id)
+        visited.add(object_id)
 
-    sig { returns(String) }
-    def to_json
-      {
-        id: @id,
-        name: @name,
-        max_retries: @max_retries,
-        backoff: @backoff
-      }.to_json
-    end
+        item.transform_values { |value| normalize_item(value, collected_ids, visited) }
+      when T::Struct
+        object_id = item.object_id
+        return item if visited.include?(object_id)
+        visited.add(object_id)
 
-    sig { params(json: String, id: T.nilable(String), name: T.nilable(String), max_retries: T.nilable(Integer), backoff: T.nilable(Float)).returns(Rager::Context) }
-    def from_json(json, id: nil, name: nil, max_retries: nil, backoff: nil)
-      JSON.parse(json, symbolize_names: true).tap do |data|
-        @id = id || data[:id]
-        @name = name || data[:name]
-        @max_retries = max_retries || data[:max_retries]
-        @backoff = backoff || data[:backoff]
+        transformed_props = item.class.props.each_key.each_with_object({}) do |prop_name, acc|
+          acc[prop_name] = normalize_item(item.send(prop_name), collected_ids, visited)
+        end
+        item.class.new(transformed_props)
+      else
+        item
       end
     end
   end

data/lib/rager/context_options.rb

@@ -0,0 +1,23 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module Rager
+  class ContextOptions < T::Struct
+    extend T::Sig
+
+    const :input_ids, T::Array[String], default: []
+    const :tags, T::Array[String], default: []
+    const :name, T.nilable(String)
+
+    sig { params(kwargs: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
+    def self.extract_kwargs(kwargs)
+      {
+        input_ids: kwargs.delete(:input_ids),
+        tags: kwargs.delete(:tags),
+        name: kwargs.delete(:name)
+      }.compact
+    end
+  end
+end

data/lib/rager/embed/options.rb

@@ -9,11 +9,11 @@ module Rager
       extend T::Sig
       include Rager::Options
 
-      const :provider, String, default: "openai"
-      const :url, T.nilable(String)
+      const :provider, Symbol, default: :openai
       const :model, T.nilable(String)
-      const :api_key, T.nilable(String)
       const :seed, T.nilable(Integer)
+      const :url, T.nilable(String)
+      const :api_key, T.nilable(String)
       const :timeout, T.nilable(Numeric)
     end
   end

data/lib/rager/embed/providers/openai.rb

@@ -22,9 +22,13 @@ module Rager
           raise Rager::Errors::CredentialsError.new("OpenAI", env_var: ["OPENAI_API_KEY"]) if api_key.nil?
 
           base_url = options.url || ENV["OPENAI_URL"] || "https://api.openai.com/v1"
+          url = "#{base_url}/embeddings"
 
-          headers = {"Content-Type" => "application/json"}
-          headers["Authorization"] = "Bearer #{api_key}" if api_key
+          headers = {
+            "Content-Type" => "application/json"
+          }.tap do |h|
+            h["Authorization"] = "Bearer #{api_key}" if api_key
+          end
 
           body = {
             model: options.model || "text-embedding-3-large",
@@ -33,7 +37,7 @@ module Rager
 
           request = Rager::Http::Request.new(
             verb: Rager::Http::Verb::Post,
-            url: "#{base_url}/embeddings",
+            url: url,
             headers: headers,
             body: body.to_json,
             timeout: options.timeout || Rager.config.timeout

data/lib/rager/errors/options_error.rb

@@ -8,7 +8,7 @@ module Rager
     class OptionsError < Rager::Error
       extend T::Sig
 
-      sig { params(options: Rager::Options, invalid_keys: T::Array[String], details: T.nilable(String)).void }
+      sig { params(options: Rager::Types::Options, invalid_keys: T::Array[String], details: T.nilable(String)).void }
       def initialize(options, invalid_keys, details: nil)
         error_data = {
           type: "options",

data/lib/rager/http/adapters/async_http.rb

@@ -90,8 +90,6 @@ module Rager
           )
         end
 
-        private
-
         sig { params(timeout: T.nilable(Numeric), block: T.proc.returns(T.untyped)).returns(T.untyped) }
         def wrap_if_timeout(timeout, &block)
           if timeout

data/lib/rager/http/adapters/mock.rb

@@ -35,8 +35,6 @@ module Rager
           cached_entry ? build_response_from_cache(cached_entry) : fetch_and_cache_response(request, key)
         end
 
-        private
-
         sig { params(request: Rager::Http::Request, key: String).returns(Rager::Http::Response) }
         def fetch_and_cache_response(request, key)
           response = @fallback_adapter.make_request(request)