exaonruby 1.0.0 → 1.2.0

data/lib/exa/utils/parallel.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+require "concurrent"
+
+module Exa
+  module Utils
+    # Parallel request executor for batch operations
+    #
+    # Execute multiple Exa API calls concurrently with configurable
+    # concurrency limits and automatic error handling.
+    #
+    # @example Parallel searches
+    #   queries = ["AI research", "ML trends", "NLP papers"]
+    #   
+    #   results = Exa::Utils::Parallel.map(queries, client: client) do |query|
+    #     client.search(query, num_results: 10)
+    #   end
+    #
+    # @example With concurrency limit
+    #   results = Exa::Utils::Parallel.map(urls, concurrency: 5) do |url|
+    #     client.get_contents([url])
+    #   end
+    #
+    # @example Error handling
+    #   results = Exa::Utils::Parallel.map(queries, on_error: :skip) do |query|
+    #     client.search(query)
+    #   end
+    class Parallel
+      # Default concurrency limit
+      DEFAULT_CONCURRENCY = 10
+
+      class << self
+        # Execute block for each item in parallel
+        #
+        # @param items [Array] Items to process
+        # @param concurrency [Integer] Max concurrent requests
+        # @param on_error [Symbol] Error handling: :raise, :skip, :return_nil
+        # @param timeout [Integer, nil] Timeout per request in seconds
+        #
+        # @yield [item] Block to execute for each item
+        # @return [Array] Results in same order as input
+        #
+        # @example
+        #   Exa::Utils::Parallel.map(queries) { |q| client.search(q) }
+        def map(items, concurrency: DEFAULT_CONCURRENCY, on_error: :raise, timeout: nil)
+          return [] if items.empty?
+
+          pool = Concurrent::FixedThreadPool.new(concurrency)
+          futures = []
+
+          items.each_with_index do |item, idx|
+            future = Concurrent::Future.execute(executor: pool) do
+              if timeout
+                Concurrent::Promises.future { yield(item) }.value!(timeout)
+              else
+                yield(item)
+              end
+            end
+            futures << { index: idx, future: future }
+          end
+
+          # Collect results in order
+          results = Array.new(items.length)
+          
+          futures.each do |entry|
+            begin
+              results[entry[:index]] = entry[:future].value!
+            rescue StandardError => e
+              case on_error
+              when :raise
+                pool.shutdown
+                raise
+              when :skip
+                next
+              when :return_nil
+                results[entry[:index]] = nil
+              else
+                raise
+              end
+            end
+          end
+
+          pool.shutdown
+          pool.wait_for_termination
+
+          results
+        end
+
+        # Execute block for each item in parallel, returning only successful results
+        #
+        # @param items [Array] Items to process
+        # @param concurrency [Integer] Max concurrent requests
+        #
+        # @yield [item] Block to execute for each item
+        # @return [Array] Successful results only
+        def map_compact(items, concurrency: DEFAULT_CONCURRENCY, &block)
+          map(items, concurrency: concurrency, on_error: :return_nil, &block).compact
+        end
+
+        # Execute block for each item in parallel, ignoring return values
+        #
+        # @param items [Array] Items to process
+        # @param concurrency [Integer] Max concurrent requests
+        #
+        # @yield [item] Block to execute for each item
+        # @return [void]
+        def each(items, concurrency: DEFAULT_CONCURRENCY, &block)
+          map(items, concurrency: concurrency, on_error: :skip, &block)
+          nil
+        end
+
+        # Execute multiple different operations in parallel
+        #
+        # @param operations [Array<Proc>] Procs to execute
+        # @param concurrency [Integer] Max concurrent requests
+        #
+        # @return [Array] Results from each proc
+        #
+        # @example
+        #   results = Exa::Utils::Parallel.all(
+        #     -> { client.search("AI") },
+        #     -> { client.search("ML") },
+        #     -> { client.get_contents([url]) }
+        #   )
+        def all(*operations, concurrency: DEFAULT_CONCURRENCY)
+          operations = operations.first if operations.first.is_a?(Array)
+          
+          map(operations, concurrency: concurrency) { |op| op.call }
+        end
+      end
+    end
+  end
+end

data/lib/exa/utils/sse_client.rb

@@ -0,0 +1,279 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+require "json"
+require "net/http"
+require "uri"
+
+module Exa
+  module Utils
+    # Server-Sent Events (SSE) streaming client for Exa.ai API
+    #
+    # Provides real-time streaming for Answer and Research endpoints that support
+    # the stream=true parameter. Tokens are yielded as they're generated.
+    #
+    # @example Stream an answer
+    #   Exa::Utils::SSEClient.stream_answer(
+    #     api_key: ENV["EXA_API_KEY"],
+    #     query: "What is quantum computing?"
+    #   ) do |event|
+    #     case event[:type]
+    #     when :token
+    #       print event[:data]  # Print each token as it arrives
+    #     when :citation
+    #       puts "\nSource: #{event[:data][:url]}"
+    #     when :done
+    #       puts "\n\nComplete!"
+    #     when :error
+    #       puts "Error: #{event[:data]}"
+    #     end
+    #   end
+    #
+    # @example Stream research progress
+    #   Exa::Utils::SSEClient.stream_research(
+    #     api_key: ENV["EXA_API_KEY"],
+    #     instructions: "Research latest AI developments"
+    #   ) do |event|
+    #     case event[:type]
+    #     when :progress
+    #       puts "Progress: #{event[:data][:percent]}%"
+    #     when :output
+    #       puts event[:data]
+    #     end
+    #   end
+    class SSEClient
+      DEFAULT_BASE_URL = "https://api.exa.ai"
+
+      # Event types that can be yielded
+      EVENT_TYPES = %i[token citation progress output done error ping].freeze
+
+      class << self
+        # Stream an answer with real-time token output
+        #
+        # @param api_key [String] Exa API key
+        # @param query [String] Question to answer
+        # @param base_url [String] API base URL
+        # @param options [Hash] Additional options (text, num_results, etc.)
+        #
+        # @yield [Hash] Event hash with :type and :data keys
+        # @yieldparam event [Hash] Event data
+        # @yieldparam event[:type] [Symbol] One of :token, :citation, :done, :error
+        # @yieldparam event[:data] [String, Hash] Event payload
+        #
+        # @return [void]
+        def stream_answer(api_key:, query:, base_url: DEFAULT_BASE_URL, **options, &block)
+          raise ArgumentError, "Block required for streaming" unless block_given?
+          raise InvalidRequestError, "query is required" if query.nil? || query.empty?
+
+          body = { query: query, stream: true }
+          body.merge!(options)
+
+          stream_request(
+            api_key: api_key,
+            url: "#{base_url}/answer",
+            body: body,
+            &block
+          )
+        end
+
+        # Stream research task output in real-time
+        #
+        # @param api_key [String] Exa API key
+        # @param instructions [String] Research instructions
+        # @param model [String] Model to use
+        # @param base_url [String] API base URL
+        # @param options [Hash] Additional options
+        #
+        # @yield [Hash] Event hash with :type and :data keys
+        # @yieldparam event [Hash] Event data
+        #
+        # @return [void]
+        def stream_research(api_key:, instructions:, model: "exa-research", base_url: DEFAULT_BASE_URL, **options, &block)
+          raise ArgumentError, "Block required for streaming" unless block_given?
+          raise InvalidRequestError, "instructions required" if instructions.nil? || instructions.empty?
+
+          body = {
+            instructions: instructions,
+            model: model,
+            stream: true
+          }
+          body.merge!(options)
+
+          stream_request(
+            api_key: api_key,
+            url: "#{base_url}/research/v1",
+            body: body,
+            &block
+          )
+        end
+
+        private
+
+        # Perform SSE streaming request
+        #
+        # @param api_key [String] API key
+        # @param url [String] Full endpoint URL
+        # @param body [Hash] Request body
+        #
+        # @yield [Hash] Parsed SSE events
+        def stream_request(api_key:, url:, body:)
+          uri = URI.parse(url)
+
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = uri.scheme == "https"
+          http.read_timeout = 300  # 5 minutes for long streams
+          http.open_timeout = 30
+
+          request = Net::HTTP::Post.new(uri.request_uri)
+          request["Content-Type"] = "application/json"
+          request["Accept"] = "text/event-stream"
+          request["Cache-Control"] = "no-cache"
+          request["x-api-key"] = api_key
+          request.body = JSON.generate(body)
+
+          buffer = String.new
+
+          http.request(request) do |response|
+            unless response.is_a?(Net::HTTPSuccess)
+              yield({ type: :error, data: "HTTP #{response.code}: #{response.message}" })
+              return
+            end
+
+            response.read_body do |chunk|
+              buffer << chunk
+              events = parse_sse_buffer(buffer)
+
+              events.each do |event|
+                yield(event)
+              end
+            end
+          end
+
+          yield({ type: :done, data: nil })
+        rescue Net::OpenTimeout, Net::ReadTimeout => e
+          yield({ type: :error, data: "Timeout: #{e.message}" })
+        rescue IOError, Errno::ECONNRESET => e
+          yield({ type: :error, data: "Connection error: #{e.message}" })
+        rescue StandardError => e
+          yield({ type: :error, data: "Error: #{e.message}" })
+        end
+
+        # Parse SSE buffer and extract complete events
+        #
+        # @param buffer [String] Buffer to parse (modified in place)
+        # @return [Array<Hash>] Parsed events
+        def parse_sse_buffer(buffer)
+          events = []
+          event_data = {}
+
+          # Split on double newlines (event boundaries)
+          while (idx = buffer.index("\n\n"))
+            raw_event = buffer.slice!(0, idx + 2)
+
+            raw_event.each_line do |line|
+              line = line.strip
+              next if line.empty?
+
+              if line.start_with?("event:")
+                event_data[:event] = line[6..].strip
+              elsif line.start_with?("data:")
+                data_content = line[5..].strip
+                event_data[:data] = data_content
+              elsif line.start_with?("id:")
+                event_data[:id] = line[3..].strip
+              elsif line.start_with?("retry:")
+                event_data[:retry] = line[6..].strip.to_i
+              end
+            end
+
+            if event_data.any?
+              parsed = parse_event(event_data)
+              events << parsed if parsed
+              event_data = {}
+            end
+          end
+
+          events
+        end
+
+        # Parse a single SSE event into our format
+        #
+        # @param event_data [Hash] Raw event data
+        # @return [Hash, nil] Parsed event or nil
+        def parse_event(event_data)
+          event_type = event_data[:event]&.to_sym || :message
+          raw_data = event_data[:data]
+
+          return nil unless raw_data
+
+          # Try to parse as JSON
+          data = begin
+            JSON.parse(raw_data, symbolize_names: true)
+          rescue JSON::ParserError
+            raw_data
+          end
+
+          case event_type
+          when :message, :token, :delta
+            # Token/delta events contain partial answer text
+            if data.is_a?(Hash)
+              text = data[:delta] || data[:content] || data[:text] || data[:token]
+              { type: :token, data: text } if text
+            else
+              { type: :token, data: data }
+            end
+          when :citation, :source
+            { type: :citation, data: data }
+          when :progress
+            { type: :progress, data: data }
+          when :output, :result
+            { type: :output, data: data }
+          when :done, :complete, :end
+            { type: :done, data: data }
+          when :error
+            { type: :error, data: data }
+          when :ping, :heartbeat
+            { type: :ping, data: nil }
+          else
+            # Return raw data for unknown event types
+            { type: event_type, data: data }
+          end
+        end
+      end
+
+      # Instance-based streaming for more control
+      #
+      # @param api_key [String] Exa API key
+      # @param base_url [String] API base URL
+      def initialize(api_key:, base_url: DEFAULT_BASE_URL)
+        @api_key = api_key
+        @base_url = base_url
+      end
+
+      # Stream an answer
+      # @see SSEClient.stream_answer
+      def answer(query, **options, &block)
+        self.class.stream_answer(
+          api_key: @api_key,
+          query: query,
+          base_url: @base_url,
+          **options,
+          &block
+        )
+      end
+
+      # Stream research
+      # @see SSEClient.stream_research
+      def research(instructions, **options, &block)
+        self.class.stream_research(
+          api_key: @api_key,
+          instructions: instructions,
+          base_url: @base_url,
+          **options,
+          &block
+        )
+      end
+    end
+  end
+end

data/lib/exa/version.rb

@@ -3,5 +3,5 @@
 # typed: strict
 
 module Exa
-  VERSION = "1.0.0"
+  VERSION = "1.2.0"
 end

data/lib/exa.rb

@@ -8,6 +8,31 @@ require_relative "exa/configuration"
 
 require_relative "exa/utils/parameter_converter"
 require_relative "exa/utils/webhook_handler"
+require_relative "exa/utils/sse_client"
+require_relative "exa/utils/parallel"
+
+# Middleware (loaded after Faraday is available in Client)
+require_relative "exa/middleware/request_logger"
+require_relative "exa/middleware/rate_limiter"
+require_relative "exa/middleware/response_cache"
+require_relative "exa/middleware/instrumentation"
+
+# Optional: Sorbet types (only loaded if sorbet-runtime is available)
+begin
+  require "sorbet-runtime"
+  require_relative "exa/types"
+rescue LoadError
+  # sorbet-runtime not installed, types module not available
+end
+
+# Optional: Rails integration (only loaded if Rails is available)
+begin
+  if defined?(::Rails::Railtie)
+    require_relative "exa/rails"
+  end
+rescue LoadError
+  # Rails not available
+end
 
 require_relative "exa/resources/base"
 require_relative "exa/resources/search_result"

data/lib/generators/exa/install_generator.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "rails/generators/base"
+
+module Exa
+  module Generators
+    # Rails generator for Exa gem installation
+    #
+    # Usage:
+    #   rails generate exa:install
+    #
+    # This creates:
+    #   - config/initializers/exa.rb
+    #   - Adds EXA_API_KEY to .env if using dotenv
+    class InstallGenerator < ::Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates an Exa initializer and credentials setup"
+
+      def create_initializer
+        create_file "config/initializers/exa.rb", <<~RUBY
+          # frozen_string_literal: true
+
+          # Exa API Configuration
+          # Documentation: https://docs.exa.ai
+          # Gem: https://github.com/tigel-agm/exaonruby
+
+          Exa.configure do |config|
+            # API Key (from Rails credentials or environment)
+            # Run: rails credentials:edit
+            # Add: exa: { api_key: "your-key" }
+            config.api_key = Rails.application.credentials.dig(:exa, :api_key) || ENV["EXA_API_KEY"]
+
+            # Request timeout in seconds (default: 60)
+            config.timeout = 60
+
+            # Maximum retry attempts for failed requests
+            config.max_retries = 3
+
+            # Optional: Enable request logging in development
+            if Rails.env.development?
+              config.logger = Rails.logger
+            end
+
+            # Optional: Enable caching to reduce API costs
+            # config.cache = Exa::Cache::MemoryStore.new
+            # config.cache_ttl = 300  # 5 minutes
+
+            # Optional: Rate limiting (requests per second)
+            # config.rate_limit = 10
+            # config.rate_limit_burst = 20
+          end
+        RUBY
+        say "Created config/initializers/exa.rb", :green
+      end
+
+      def add_to_env
+        if File.exist?(".env")
+          append_to_file ".env" do
+            "\n# Exa API Key\nEXA_API_KEY=your-api-key-here\n"
+          end
+          say "Added EXA_API_KEY to .env", :green
+        elsif File.exist?(".env.example")
+          append_to_file ".env.example" do
+            "\n# Exa API Key\nEXA_API_KEY=\n"
+          end
+          say "Added EXA_API_KEY to .env.example", :green
+        end
+      end
+
+      def show_instructions
+        say ""
+        say "Exa gem installed successfully!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Add your API key to Rails credentials:"
+        say "     rails credentials:edit"
+        say ""
+        say "     exa:"
+        say "       api_key: your-api-key-here"
+        say ""
+        say "  2. Or set the EXA_API_KEY environment variable"
+        say ""
+        say "  3. Test the connection:"
+        say "     rails exa:verify"
+        say ""
+        say "Usage examples:"
+        say "  client = Exa::Client.new"
+        say "  results = client.search('AI research', num_results: 10)"
+        say ""
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exaonruby
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - tigel-agm
@@ -69,16 +69,39 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 description: A production-ready Ruby gem wrapper for the Exa.ai Search and Websets
   APIs. Features neural search, LLM-powered answers, async research tasks, Websets
-  management (monitors, imports, webhooks), and a beautiful CLI. Includes n8n/Zapier
-  webhook signature verification utilities.
+  management (monitors, imports, webhooks), SSE streaming, request logging, rate limiting,
+  response caching, OpenTelemetry instrumentation, parallel requests, Rails integration,
+  Sorbet types, and a beautiful CLI. Includes n8n/Zapier webhook signature verification
+  utilities.
 email: []
 executables:
 - exa
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - LICENSE.txt
 - README.md
 - exaonruby.gemspec
@@ -101,6 +124,11 @@ files:
 - lib/exa/endpoints/webset_searches.rb
 - lib/exa/endpoints/websets.rb
 - lib/exa/errors.rb
+- lib/exa/middleware/instrumentation.rb
+- lib/exa/middleware/rate_limiter.rb
+- lib/exa/middleware/request_logger.rb
+- lib/exa/middleware/response_cache.rb
+- lib/exa/rails.rb
 - lib/exa/resources/answer_response.rb
 - lib/exa/resources/base.rb
 - lib/exa/resources/contents_response.rb
@@ -114,9 +142,13 @@ files:
 - lib/exa/resources/webhook.rb
 - lib/exa/resources/webset.rb
 - lib/exa/resources/webset_item.rb
+- lib/exa/types.rb
+- lib/exa/utils/parallel.rb
 - lib/exa/utils/parameter_converter.rb
+- lib/exa/utils/sse_client.rb
 - lib/exa/utils/webhook_handler.rb
 - lib/exa/version.rb
+- lib/generators/exa/install_generator.rb
 homepage: https://github.com/tigel-agm/exaonruby
 licenses:
 - MIT
@@ -142,5 +174,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.2
 specification_version: 4
-summary: Complete Ruby client for the Exa.ai API with beautiful CLI
+summary: Complete Ruby client for the Exa.ai API with CLI, middleware, and Rails integration
 test_files: []