bx_builder_chain 0.1.0

data/lib/bx_builder_chain/llm/open_ai.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+module BxBuilderChain::Llm
+  # LLM interface for OpenAI APIs: https://platform.openai.com/overview
+  #
+  # Gem requirements:
+  #    gem "ruby-openai", "~> 4.0.0"
+  #
+  # Usage:
+  #    openai = BxBuilderChain::LLM::OpenAI.new(api_key:, llm_options: {})
+  #
+  class OpenAi < Base
+    DEFAULTS = {
+      temperature: 0.0,
+      completion_model_name: "text-davinci-003",
+      chat_completion_model_name: "gpt-3.5-turbo",
+      embeddings_model_name: "text-embedding-ada-002",
+      dimension: 1536
+    }.freeze
+    LENGTH_VALIDATOR = BxBuilderChain::Utils::TokenLength::OpenAiValidator
+    ROLE_MAPPING = {
+      "ai" => "assistant",
+      "human" => "user"
+    }
+
+    attr_accessor :functions
+
+    def initialize(api_key: BxBuilderChain.configuration.openai_api_key, llm_options: {}, default_options: {})
+      depends_on "ruby-openai"
+      require "openai"
+
+      @client = ::OpenAI::Client.new(access_token: api_key, **llm_options)
+      @defaults = DEFAULTS.merge(default_options)
+    end
+
+    #
+    # Generate an embedding for a given text
+    #
+    # @param text [String] The text to generate an embedding for
+    # @param params extra parameters passed to OpenAI::Client#embeddings
+    # @return [Array] The embedding
+    #
+    def embed(text:, **params)
+      parameters = {model: @defaults[:embeddings_model_name], input: text}
+
+      validate_max_tokens(text, parameters[:model])
+
+      response = client.embeddings(parameters: parameters.merge(params))
+      response.dig("data").first.dig("embedding")
+    end
+
+    #
+    # Generate a completion for a given prompt
+    #
+    # @param prompt [String] The prompt to generate a completion for
+    # @param params  extra parameters passed to OpenAI::Client#complete
+    # @return [String] The completion
+    #
+    def complete(prompt:, **params)
+      parameters = compose_parameters @defaults[:completion_model_name], params
+
+      parameters[:prompt] = prompt
+      parameters[:max_tokens] = validate_max_tokens(prompt, parameters[:model])
+
+      response = client.completions(parameters: parameters)
+      response.dig("choices", 0, "text")
+    end
+
+    #
+    # Generate a chat completion for a given prompt or messages.
+    #
+    # == Examples
+    #
+    #     # simplest case, just give a prompt
+    #     openai.chat prompt: "When was Ruby first released?"
+    #
+    #     # prompt plus some context about how to respond
+    #     openai.chat context: "You are RubyGPT, a helpful chat bot for helping people learn Ruby", prompt: "Does Ruby have a REPL like IPython?"
+    #
+    #     # full control over messages that get sent, equivilent to the above
+    #     openai.chat messages: [
+    #       {
+    #         role: "system",
+    #         content: "You are RubyGPT, a helpful chat bot for helping people learn Ruby", prompt: "Does Ruby have a REPL like IPython?"
+    #       },
+    #       {
+    #         role: "user",
+    #         content: "When was Ruby first released?"
+    #       }
+    #     ]
+    #
+    #     # few-short prompting with examples
+    #     openai.chat prompt: "When was factory_bot released?",
+    #       examples: [
+    #         {
+    #           role: "user",
+    #           content: "When was Ruby on Rails released?"
+    #         }
+    #         {
+    #           role: "assistant",
+    #           content: "2004"
+    #         },
+    #       ]
+    #
+    # @param prompt [HumanMessage] The prompt to generate a chat completion for
+    # @param messages [Array<AIMessage|HumanMessage>] The messages that have been sent in the conversation
+    # @param context [SystemMessage] An initial context to provide as a system message, ie "You are RubyGPT, a helpful chat bot for helping people learn Ruby"
+    # @param examples [Array<AIMessage|HumanMessage>] Examples of messages to provide to the model. Useful for Few-Shot Prompting
+    # @param options [Hash] extra parameters passed to OpenAI::Client#chat
+    # @yield [AIMessage] Stream responses back one String at a time
+    # @return [AIMessage] The chat completion
+    #
+    def chat(prompt: "", messages: [], context: "", examples: [], **options)
+      raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
+
+      parameters = compose_parameters @defaults[:chat_completion_model_name], options
+      parameters[:messages] = compose_chat_messages(prompt: prompt, messages: messages, context: context, examples: examples)
+
+      if functions
+        parameters[:functions] = functions
+      else
+        parameters[:max_tokens] = validate_max_tokens(parameters[:messages], parameters[:model])
+      end
+
+      response = client.chat(parameters: parameters)
+      
+      response.dig("choices", 0, "message", "content")
+    end
+
+    #
+    # Generate a summary for a given text
+    #
+    # @param text [String] The text to generate a summary for
+    # @return [String] The summary
+    #
+    # def summarize(text:)
+    #   prompt_template = BxBuilderChain::Prompt.load_from_path(
+    #     file_path: BxBuilderChain.root.join("langchain/llm/prompts/summarize_template.yaml")
+    #   )
+    #   prompt = prompt_template.format(text: text)
+
+    #   complete(prompt: prompt, temperature: @defaults[:temperature])
+    # end
+
+    private
+
+    def compose_parameters(model, params)
+      default_params = {model: model, temperature: @defaults[:temperature]}
+
+      default_params[:stop] = params.delete(:stop_sequences) if params[:stop_sequences]
+
+      default_params.merge(params)
+    end
+
+    def compose_chat_messages(prompt:, messages:, context:, examples:)
+      history = []
+
+      history.concat transform_messages(examples) unless examples.empty?
+
+      history.concat transform_messages(messages) unless messages.empty?
+
+      unless context.nil? || context.to_s.empty?
+        history.reject! { |message| message[:role] == "system" }
+        history.prepend({role: "system", content: context.content})
+      end
+
+      unless prompt.empty?
+        if history.last && history.last[:role] == "user"
+          history.last[:content] += "\n#{prompt}"
+        else
+          history.append({role: "user", content: prompt})
+        end
+      end
+
+      history
+    end
+
+    def transform_messages(messages)
+      messages.map do |message|
+        {
+          role: ROLE_MAPPING.fetch(message.type, message.type),
+          content: message.content
+        }
+      end
+    end
+
+    def validate_max_tokens(messages, model)
+      LENGTH_VALIDATOR.validate_max_tokens!(messages, model)
+    end
+  end
+end

data/lib/bx_builder_chain/loader.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "open-uri"
+
+module BxBuilderChain
+  class Loader
+    class FileNotFound < StandardError; end
+
+    class UnknownFormatError < StandardError; end
+
+    URI_REGEX = %r{\A[A-Za-z][A-Za-z0-9+\-.]*://}
+
+    # Load data from a file or URL. Shorthand for  `BxBuilderChain::Loader.new(path).load`
+    #
+    # == Examples
+    #
+    #     # load a URL
+    #     data = BxBuilderChain::Loader.load("https://example.com/docs/README.md")
+    #
+    #     # load a file
+    #     data = BxBuilderChain::Loader.load("README.md")
+    #
+    #    # Load data using a custom processor
+    #    data = BxBuilderChain::Loader.load("README.md") do |raw_data, options|
+    #      # your processing code goes here
+    #      # return data at the end here
+    #    end
+    #
+    # @param path [String | Pathname] path to file or URL
+    # @param options [Hash] options passed to the processor class used to process the data
+    # @return [Data] data loaded from path
+    def self.load(path, options = {}, &block)
+      new(path, options).load(&block)
+    end
+
+    # Initialize BxBuilderChain::Loader
+    # @param path [String | Pathname] path to file or URL
+    # @param options [Hash] options passed to the processor class used to process the data
+    # @return [BxBuilderChain::Loader] loader instance
+    def initialize(path, options = {})
+      @options = options
+      @path = path
+    end
+
+    # Is the path a URL?
+    #
+    # @return [Boolean] true if path is URL
+    def url?
+      return false if @path.is_a?(Pathname)
+
+      !!(@path =~ URI_REGEX)
+    end
+
+    # Is the path a directory
+    #
+    # @return [Boolean] true if path is a directory
+    def directory?
+      File.directory?(@path)
+    end
+
+    # Load data from a file or URL
+    #
+    #    loader = BxBuilderChain::Loader.new("README.md")
+    #    # Load data using default processor for the file
+    #    loader.load
+    #
+    #    # Load data using a custom processor
+    #    loader.load do |raw_data, options|
+    #      # your processing code goes here
+    #      # return data at the end here
+    #    end
+    #
+    # @yield [String, Hash] handle parsing raw output into string directly
+    # @yieldparam [String] raw_data from the loaded URL or file
+    # @yieldreturn [String] parsed data, as a String
+    #
+    # @return [Data] data that was loaded
+    def load(&block)
+      return process_data(load_from_url, &block) if url?
+      return load_from_directory(&block) if directory?
+
+      process_data(load_from_path, &block)
+    end
+
+    private
+
+    def load_from_url
+      URI.parse(@path).open
+    end
+
+    def load_from_path
+      return File.open(@path) if File.exist?(@path)
+
+      raise FileNotFound, "File #{@path} does not exist"
+    end
+
+    def load_from_directory(&block)
+      Dir.glob(File.join(@path, "**/*")).map do |file|
+        # Only load and add to result files with supported extensions
+        BxBuilderChain::Loader.new(file, @options).load(&block)
+      rescue
+        UnknownFormatError nil
+      end.flatten.compact
+    end
+
+    def process_data(data, &block)
+      @raw_data = data
+
+      result = if block
+        yield @raw_data.read, @options
+      else
+        processor_klass.new(@options).parse(@raw_data)
+      end
+
+      BxBuilderChain::Data.new(result)
+    end
+
+    def processor_klass
+      raise UnknownFormatError unless (kind = find_processor)
+
+      BxBuilderChain::Processors.const_get(kind)
+    end
+
+    def find_processor
+      processors.find { |klass| processor_matches? "#{klass}::#{lookup_constant}", source_type }
+    end
+
+    def processor_matches?(constant, value)
+      BxBuilderChain::Processors.const_get(constant).include?(value)
+    end
+
+    def processors
+      BxBuilderChain::Processors.constants
+    end
+
+    def source_type
+      url? ? @raw_data.content_type : File.extname(@path)
+    end
+
+    def lookup_constant
+      url? ? :CONTENT_TYPES : :EXTENSIONS
+    end
+  end
+end

data/lib/bx_builder_chain/processors/base.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Processors
+    # Processors load and parse/process various data types such as CSVs, PDFs, Word documents, HTML pages, and others.
+    class Base
+      include BxBuilderChain::DependencyHelper
+
+      EXTENSIONS = []
+      CONTENT_TYPES = []
+
+      def initialize(options = {})
+        @options = options
+      end
+
+      def parse(data)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/processors/csv.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module BxBuilderChain
+  module Processors
+    class Csv < Base
+      EXTENSIONS = [".csv"]
+      CONTENT_TYPES = ["text/csv"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Array of Hash]
+      def parse(data)
+        ::CSV.new(data.read, col_sep: separator).map do |row|
+          row.map(&:strip)
+        end
+      end
+
+      private
+
+      def separator
+        @options[:col_sep] || ","
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/processors/docx.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Processors
+    class Docx < Base
+      EXTENSIONS = [".docx"]
+      CONTENT_TYPES = ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]
+
+      def initialize(*)
+        depends_on "docx"
+        require "docx"
+      end
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        ::Docx::Document
+          .open(StringIO.new(data.read))
+          .text
+          .strip
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/processors/html.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Processors
+    class Html < Base
+      EXTENSIONS = [".html", ".htm"]
+      CONTENT_TYPES = ["text/html"]
+
+      # We only look for headings and paragraphs
+      TEXT_CONTENT_TAGS = %w[h1 h2 h3 h4 h5 h6 p]
+
+      def initialize(*)
+        depends_on "nokogiri"
+        require "nokogiri"
+      end
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        Nokogiri::HTML(data.read)
+          .css(TEXT_CONTENT_TAGS.join(","))
+          .map(&:inner_text)
+          .join("\n\n")
+          .strip
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/processors/json.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Processors
+    class Json < Base
+      EXTENSIONS = [".json"]
+      CONTENT_TYPES = ["application/json"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Hash]
+      def parse(data)
+        ::JSON.parse(data.read)
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/processors/pdf.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Processors
+    class Pdf < Base
+      EXTENSIONS = [".pdf"]
+      CONTENT_TYPES = ["application/pdf"]
+
+      def initialize(*)
+        depends_on "pdf-reader"
+        require "pdf-reader"
+      end
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        ::PDF::Reader
+          .new(StringIO.new(data.read))
+          .pages
+          .map(&:text)
+          .join("\n\n")
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/processors/text.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Processors
+    class Text < Base
+      EXTENSIONS = [".txt"]
+      CONTENT_TYPES = ["text/plain"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        data.read
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/processors/xlsx.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Processors
+    class Xlsx < Base
+      EXTENSIONS = [".xlsx", ".xlsm"].freeze
+      CONTENT_TYPES = ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"].freeze
+
+      def initialize(*)
+        depends_on "roo"
+        require "roo"
+      end
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Array<Array<String>>] Array of rows, each row is an array of cells
+      def parse(data)
+        output = []
+        
+        xlsx_file = Roo::Spreadsheet.open(data)
+        xlsx_file.each_with_pagename do |sheet, rows|
+          output << rows.map do |row|
+            row.map { |i| i.to_s.strip }
+          end
+        end
+
+        output
+      end
+    end
+  end
+end