dspy-deep_research 1.0.0

data/lib/dspy/deep_research/signatures.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module DSPy
+  module DeepResearch
+    module Signatures
+      extend T::Sig
+
+      class BuildOutline < DSPy::Signature
+        description "Generate an outline of sections to investigate for the research brief"
+
+        class Mode < T::Enum
+          enums do
+            Light = new("light")
+            Medium = new("medium")
+            Hard = new("hard")
+            Ultra = new("ultra")
+          end
+        end
+
+        class SectionSpec < T::Struct
+          const :identifier, String
+          const :title, String
+          const :prompt, String
+          const :token_budget, Integer
+          const :attempt, Integer, default: 0
+          const :parent_identifier, T.nilable(String), default: nil
+        end
+
+        input do
+          const :brief, String, description: "Research brief or question to investigate"
+          const :mode, Mode, description: "Desired research intensity mode", default: Mode::Medium
+        end
+
+        output do
+          const :sections, T::Array[SectionSpec], description: "Ordered section specifications to investigate"
+        end
+      end
+
+      class SynthesizeSection < DSPy::Signature
+        description "Transform DeepSearch results into a coherent section draft"
+
+        input do
+          const :brief, String, description: "Original research brief"
+          const :section, BuildOutline::SectionSpec, description: "Section specification being drafted"
+          const :answer, String, description: "Candidate answer from DeepSearch"
+          const :notes, T::Array[String], description: "Supporting notes collected during DeepSearch"
+          const :citations, T::Array[String], description: "Citations gathered for this section"
+        end
+
+        output do
+          const :draft, String, description: "Section draft ready for aggregation"
+          const :citations, T::Array[String], description: "Filtered citations supporting the draft"
+        end
+      end
+
+      class QAReview < DSPy::Signature
+        description "Decide whether a section draft is ready or requires more evidence"
+
+        class Status < T::Enum
+          enums do
+            Approved          = new("approved")
+            NeedsMoreEvidence = new("needs_more_evidence")
+          end
+        end
+
+        input do
+          const :brief, String, description: "Research brief"
+          const :section, BuildOutline::SectionSpec, description: "Section specification with metadata"
+          const :draft, String, description: "Draft content for the section"
+          const :notes, T::Array[String], description: "Supporting evidence notes"
+          const :citations, T::Array[String], description: "Citations backing the draft"
+          const :attempt, Integer, description: "Number of attempts made for this section"
+        end
+
+        output do
+          const :status, Status, description: "QA decision for the section"
+          const :follow_up_prompt, T.nilable(String), description: "Additional prompt if more evidence is required"
+        end
+      end
+
+      class AssembleReport < DSPy::Signature
+        description "Aggregate accepted sections into the final deliverable"
+
+        class SectionDraft < T::Struct
+          const :identifier, String
+          const :title, String
+          const :draft, String
+          const :citations, T::Array[String]
+        end
+
+        input do
+          const :brief, String, description: "Research brief"
+          const :sections, T::Array[SectionDraft], description: "Accepted section drafts"
+        end
+
+        output do
+          const :report, String, description: "Final synthesized report"
+          const :citations, T::Array[String], description: "Consolidated citations for the report"
+        end
+      end
+    end
+  end
+end

data/lib/dspy/deep_research/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module DSPy
+  module DeepResearch
+    VERSION = "1.0.0"
+  end
+end

data/lib/dspy/deep_research.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require_relative "deep_search"
+
+require_relative "deep_research/version"
+require_relative "deep_research/errors"
+require_relative "deep_research/signatures"
+require_relative "deep_research/section_queue"
+require_relative "deep_research/module"
+require_relative "deep_research_with_memory"
+
+module DSPy
+  module DeepResearch
+    extend T::Sig
+  end
+end

data/lib/dspy/deep_search/clients/exa_client.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module DSPy
+  module DeepSearch
+    module Clients
+      class ExaClient
+        extend T::Sig
+
+        class Error < StandardError; end
+        class ConfigurationError < Error; end
+        class ApiError < Error; end
+
+        class Result < T::Struct
+          const :url, String
+          const :title, T.nilable(String)
+          const :summary, T.nilable(String)
+          const :highlights, T::Array[String]
+          const :score, T.nilable(Float)
+        end
+
+        class Content < T::Struct
+          const :url, String
+          const :text, T.nilable(String)
+          const :summary, T.nilable(String)
+          const :highlights, T::Array[String]
+        end
+
+        sig { params(client: T.nilable(::Exa::Client)).void }
+        def initialize(client: nil)
+          @client = T.let(client || build_client, ::Exa::Client)
+        end
+
+        sig do
+          params(
+            query: String,
+            num_results: Integer,
+            autoprompt: T::Boolean
+          ).returns(T::Array[Result])
+        end
+        def search(query:, num_results: 5, autoprompt: true)
+          response = with_api_errors do
+            client.search.search(
+              query: query,
+              num_results: num_results,
+              use_autoprompt: autoprompt,
+              summary: true
+            )
+          end
+
+          response.results.filter_map do |result|
+            next if result.url.nil?
+
+            Result.new(
+              url: result.url,
+              title: result.title,
+              summary: result.summary,
+              highlights: normalize_highlights(result.highlights),
+              score: result.score
+            )
+          end
+        end
+
+        sig do
+          params(
+            urls: T::Array[String],
+            options: T::Hash[Symbol, T.untyped]
+          ).returns(T::Array[Content])
+        end
+        def contents(urls:, **options)
+          raise ArgumentError, "urls must not be empty" if urls.empty?
+
+          defaults = {
+            text: true,
+            summary: true,
+            highlights: true,
+            filter_empty_results: true
+          }
+
+          payload = Exa::Types::ContentsRequest.new(**defaults.merge(options).merge(urls: urls)).to_payload
+
+          raw_response = with_api_errors do
+            client.request(
+              method: :post,
+              path: "contents",
+              body: payload,
+              response_model: nil
+            )
+          end
+
+          symbolized = symbolize_keys(raw_response)
+
+          check_content_statuses!(symbolized)
+
+          Array(symbolized[:results]).each_with_index.filter_map do |result, index|
+            result = symbolize_keys(result)
+            url = result[:url] || urls[index]
+            next if url.nil?
+
+            Content.new(
+              url: url,
+              text: result[:text],
+              summary: result[:summary],
+              highlights: normalize_highlights(result[:highlights])
+            )
+          end
+        end
+
+        private
+
+        sig { returns(::Exa::Client) }
+        def build_client
+          ::Exa::Client.new
+        rescue ::Exa::Errors::ConfigurationError => e
+          raise ConfigurationError, e.message
+        end
+
+        sig { params(response: T::Hash[Symbol, T.untyped]).void }
+        def check_content_statuses!(response)
+          statuses = response[:statuses]
+          return if statuses.nil?
+
+          failure = Array(statuses).map { |status| symbolize_keys(status) }.find { |status| status[:status] != "success" }
+          return if failure.nil?
+
+          error_details = failure[:error] ? failure[:error].inspect : nil
+          message = [
+            "Exa contents request failed for #{failure[:id]}",
+            failure[:status],
+            error_details
+          ].compact.join(" - ")
+
+          raise ApiError, message
+        end
+
+        sig { params(hash: T.untyped).returns(T::Hash[Symbol, T.untyped]) }
+        def symbolize_keys(hash)
+          case hash
+          when Hash
+            hash.each_with_object({}) do |(key, value), acc|
+              acc[(key.is_a?(String) ? key.to_sym : key)] = value
+            end
+          else
+            {}
+          end
+        end
+
+        sig { params(highlights: T.nilable(T::Array[T.nilable(String)])).returns(T::Array[String]) }
+        def normalize_highlights(highlights)
+          Array(highlights).compact.map(&:to_s)
+        end
+
+        sig { params(block: T.proc.returns(T.untyped)).returns(T.untyped) }
+        def with_api_errors(&block)
+          block.call
+        rescue ::Exa::Errors::ConfigurationError => e
+          raise ConfigurationError, e.message
+        rescue ::Exa::Errors::APIError => e
+          raise ApiError, e.message
+        rescue ::Exa::Error => e
+          raise ApiError, e.message
+        end
+
+        sig { returns(::Exa::Client) }
+        attr_reader :client
+      end
+    end
+  end
+end

data/lib/dspy/deep_search/gap_queue.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "set"
+
+module DSPy
+  module DeepSearch
+    class GapQueue
+      extend T::Sig
+
+      class Empty < StandardError; end
+
+      sig { void }
+      def initialize
+        @queue = T.let([], T::Array[T.untyped])
+        @seen = T.let(Set.new, T::Set[T.untyped])
+      end
+
+      sig { params(item: T.untyped).void }
+      def enqueue(item)
+        return if @seen.include?(item)
+
+        @queue << item
+        @seen << item
+      end
+
+      sig { returns(T.untyped) }
+      def dequeue
+        raise Empty, "No items remaining in gap queue" if @queue.empty?
+
+        item = @queue.shift
+        @seen.delete(item)
+        item
+      end
+
+      sig { returns(Integer) }
+      def size
+        @queue.length
+      end
+
+      sig { returns(T::Boolean) }
+      def empty?
+        @queue.empty?
+      end
+    end
+  end
+end