bx_builder_chain 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 550493404afe4cddc9999dc3071a0bc507052438409d320ee143a2711b6711a0
+  data.tar.gz: 75238a1638cc5970a110c14b3cdc914a9628c5c338d50317c698910a167f1a6f
+SHA512:
+  metadata.gz: 819f64ad09e864b4f008dd60b90857140a2d853b3adc145477f0e22da235ebfc558ad8f42c32345be8621c9ea1a7d6b00b3bedf06ca4b5dac8fb28fbb2a5e279
+  data.tar.gz: 40903aae567fc836e12ae05f0120ff44a3bbd36b213612abf35a20e7169236c5aff58655ccd65a452297d450798bf2e8d8187e54efac58784558e359e71e1699

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,13 @@
+AllCops:
+  TargetRubyVersion: 2.6
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-08-17
+
+- Initial release

data/Gemfile

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in bx_builder_chain.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.0"
+gem "rubocop", "~> 1.21"
+gem "sequel", "~> 5.71"
+gem "pg"
+gem 'dotenv'
+# gem "pgvector", path: '../pgvector-ruby'
+gem "ruby-openai"
+gem "pdf-reader"
+gem "nokogiri"
+gem "docx"
+gem "roo"
+gem "baran"
+
+gem "pry"

data/Gemfile.lock

@@ -0,0 +1,120 @@
+PATH
+  remote: .
+  specs:
+    bx_builder_chain (0.1.0)
+      baran (= 0.1.7)
+      docx
+      dotenv
+      nokogiri
+      pdf-reader
+      pg
+      roo
+      ruby-openai
+      sequel (~> 5.71)
+      zeitwerk (= 2.6.11)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    Ascii85 (1.1.0)
+    afm (0.2.2)
+    ast (2.4.2)
+    baran (0.1.7)
+    coderay (1.1.3)
+    diff-lcs (1.5.0)
+    docx (0.8.0)
+      nokogiri (~> 1.13, >= 1.13.0)
+      rubyzip (~> 2.0)
+    dotenv (2.8.1)
+    faraday (2.7.10)
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
+    faraday-net_http (3.0.2)
+    hashery (2.1.2)
+    json (2.6.3)
+    method_source (1.0.0)
+    multipart-post (2.3.0)
+    nokogiri (1.13.10-arm64-darwin)
+      racc (~> 1.4)
+    parallel (1.23.0)
+    parser (3.2.2.3)
+      ast (~> 2.4.1)
+      racc
+    pdf-reader (2.11.0)
+      Ascii85 (~> 1.0)
+      afm (~> 0.2.1)
+      hashery (~> 2.0)
+      ruby-rc4
+      ttfunk
+    pg (1.5.3)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    racc (1.7.1)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.8.1)
+    rexml (3.2.6)
+    roo (2.8.3)
+      nokogiri (~> 1)
+      rubyzip (>= 1.3.0, < 3.0.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+    rubocop (1.50.2)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.2.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.29.0)
+      parser (>= 3.2.1.0)
+    ruby-openai (5.0.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
+    ruby-progressbar (1.13.0)
+    ruby-rc4 (0.1.5)
+    ruby2_keywords (0.0.5)
+    rubyzip (2.3.2)
+    sequel (5.71.0)
+    ttfunk (1.7.0)
+    unicode-display_width (2.4.2)
+    zeitwerk (2.6.11)
+
+PLATFORMS
+  arm64-darwin-22
+
+DEPENDENCIES
+  baran
+  bx_builder_chain!
+  docx
+  dotenv
+  nokogiri
+  pdf-reader
+  pg
+  pry
+  rake (~> 13.0)
+  roo
+  rspec (~> 3.0)
+  rubocop (~> 1.21)
+  ruby-openai
+  sequel (~> 5.71)
+
+BUNDLED WITH
+   2.4.7

data/README.md

@@ -0,0 +1,74 @@
+# BxBuilderChain
+
+This gem / building block allows builder apps to use open ai.
+The following features are included:
+  - OpenAi completion with GPT 3.5 and GPT 4
+  - Q&A / additional prompt context with a users private documents
+  - Documents can be stored on a global and user/group level
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add bx_builder_chain
+
+followed by
+
+    $ bundle install
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install bx_builder_chain
+
+TODO: add rake task to create db structure
+### Optional
+
+generate the endpoint & Active admin contollers for Builder Chain
+
+    $ rails generate builder_chain:endpoints
+
+this will add the following endpoint controllers
+ - File upload
+ - OpenAi ask / completion
+ - ActiveAdmin documents controller
+
+## Usage
+```ruby
+require "bx_builder_chain"
+```
+
+create the llm and client
+```ruby
+llm = BxBuilderChain::Llm::OpenAi.new(api_key: 'open-ai-api-key')
+client = BxBuilderChain::Vectorsearch::Pgvector.new(
+      url: 'postgres://postgres:password@localhost:5432/test', # postgres db url
+      table_name: "embeddings", # table name for the documents to be stored
+      llm: llm,
+      namespace: user_id # default is nil, nil is used for global public documents
+    )
+```
+
+Add documents to the Vector Store
+
+```ruby
+# Store plain texts in your vector search database
+client.add_texts(
+    texts: [
+        "Begin by preheating your oven to 375°F (190°C). Prepare four boneless, skinless chicken breasts by cutting a pocket into the side of each breast, being careful not to cut all the way through. Season the chicken with salt and pepper to taste. In a large skillet, melt 2 tablespoons of unsalted butter over medium heat. Add 1 small diced onion and 2 minced garlic cloves, and cook until softened, about 3-4 minutes. Add 8 ounces of fresh spinach and cook until wilted, about 3 minutes. Remove the skillet from heat and let the mixture cool slightly.",
+        "In a bowl, combine the spinach mixture with 4 ounces of softened cream cheese, 1/4 cup of grated Parmesan cheese, 1/4 cup of shredded mozzarella cheese, and 1/4 teaspoon of red pepper flakes. Mix until well combined. Stuff each chicken breast pocket with an equal amount of the spinach mixture. Seal the pocket with a toothpick if necessary. In the same skillet, heat 1 tablespoon of olive oil over medium-high heat. Add the stuffed chicken breasts and sear on each side for 3-4 minutes, or until golden brown."
+    ]
+)
+```
+```ruby
+# Store the contents of your files in your vector search database
+my_pdf = "path/to/my.pdf"
+my_text = "path/to/my.txt"
+my_docx = "path/to/my.docx"
+
+client.add_data(paths: [my_pdf, my_text, my_docx])
+```
+
+Then ask the question
+```ruby
+client.ask(question: "What is Frogger?")
+```

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/bx_builder_chain.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "lib/bx_builder_chain/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "bx_builder_chain"
+  spec.version = BxBuilderChain::VERSION
+  spec.authors = ["Paul Ketelle"]
+  spec.email = ["paul.ketelle@builder.ai"]
+
+  spec.summary = "Write a short summary, because RubyGems requires one."
+  spec.description = "Write a longer description or delete this line."
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "zeitwerk", "2.6.11"
+  spec.add_dependency "baran", "0.1.7"
+  spec.add_dependency "sequel", "~> 5.71"
+  spec.add_dependency "pg", "~> 1.5.3"
+  spec.add_dependency 'dotenv', "~> 2.8"
+  spec.add_dependency "ruby-openai", "~> 5.1.0"
+  spec.add_dependency "pdf-reader", "~> 2.11.0"
+  spec.add_dependency "nokogiri", "~> 1.8" 
+  spec.add_dependency "docx", "~> 0.8.0"
+  spec.add_dependency "roo", "~> 2.8.3"
+end

data/lib/bx_builder_chain/chunker/recursive_text.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "baran"
+
+module BxBuilderChain
+  module Chunker
+    #
+    # Recursive text chunker. Preferentially splits on separators.
+    #
+    # Usage:
+    #     BxBuilderChain::Chunker::RecursiveText.new(text).chunks
+    #
+    class RecursiveText
+      attr_reader :text, :chunk_size, :chunk_overlap, :separators
+
+      # @param [String] text
+      # @param [Integer] chunk_size
+      # @param [Integer] chunk_overlap
+      # @param [Array<String>] separators
+      def initialize(text, chunk_size: 1000, chunk_overlap: 200, separators: ["\n\n", "\n", ".", " ", ""])
+        @text = text
+        @chunk_size = chunk_size
+        @chunk_overlap = chunk_overlap
+        @separators = separators
+      end
+
+      # @return [Array<String>]
+      def chunks
+        splitter = Baran::RecursiveCharacterTextSplitter.new(
+          chunk_size: chunk_size,
+          chunk_overlap: chunk_overlap,
+          separators: separators
+        )
+        splitter.chunks(text)
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/chunker/text.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "baran"
+
+module BxBuilderChain
+  module Chunker
+    #
+    # Simple text chunker
+    #
+    # Usage:
+    #     BxBuilderChain::Chunker::Text.new(text).chunks
+    #
+    class Text
+      attr_reader :text, :chunk_size, :chunk_overlap, :separator
+
+      # @param [String] text
+      # @param [Integer] chunk_size
+      # @param [Integer] chunk_overlap
+      # @param [String] separator
+      def initialize(text, chunk_size: 1024, chunk_overlap: 64, separator: "\n\n")
+        @text = text
+        @chunk_size = chunk_size
+        @chunk_overlap = chunk_overlap
+        @separator = separator
+      end
+
+      # @return [Array<String>]
+      def chunks
+        splitter = Baran::CharacterTextSplitter.new(
+          chunk_size: chunk_size,
+          chunk_overlap: chunk_overlap,
+          separator: separator
+        )
+        splitter.chunks(text)
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/configuration.rb

@@ -0,0 +1,21 @@
+# Inside your gem (e.g., in `lib/bx_builder_chain/vectorsearch/configuration.rb`)
+
+module BxBuilderChain
+    class Configuration
+      attr_accessor :pg_url, :openai_api_key, :public_namespace, :threshold, :default_prompt_template
+  
+      def initialize
+        @pg_url             = ENV['DB_URL']
+        @openai_api_key     = ENV['OPENAI_API_KEY']
+        @public_namespace   = 'public'
+        @threshold          = 0.25
+        @default_prompt_template = "Context information is below
+--------------------
+%{context}
+--------------------
+Given the context information and not prior knowledge
+answer the question: %{question}"
+      end
+    end
+  end
+  

data/lib/bx_builder_chain/data.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  # Abstraction for data loaded by a {BxBuilderChain::Loader}
+  class Data
+    # URL or Path of the data source
+    # @return [String]
+    attr_reader :source
+
+    # @param data [String] data that was loaded
+    # @option options [String] :source URL or Path of the data source
+    def initialize(data, options = {})
+      @source = options[:source]
+      @data = data
+    end
+
+    # @return [String]
+    def value
+      @data
+    end
+
+    # @param opts [Hash] options passed to the chunker
+    # @return [Array<String>]
+    def chunks(opts = {})
+      BxBuilderChain::Chunker::RecursiveText.new(@data, **opts).chunks
+    end
+  end
+end

data/lib/bx_builder_chain/dependency_helper.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module DependencyHelper
+    class VersionError < ScriptError; end
+
+    # This method requires and loads the given gem, and then checks to see if the version of the gem meets the requirements listed in `langchain.gemspec`
+    # This solution was built to avoid auto-loading every single gem in the Gemfile when the developer will mostly likely be only using a few of them.
+    #
+    # @param gem_name [String] The name of the gem to load
+    # @return [Boolean] Whether or not the gem was loaded successfully
+    # @raise [LoadError] If the gem is not installed
+    # @raise [VersionError] If the gem is installed, but the version does not meet the requirements
+    #
+    def depends_on(gem_name)
+      Gem::Specification.find_by_name(gem_name)
+      true
+    rescue LoadError
+      raise LoadError, "!Could not load #{gem_name}. Please ensure that the #{gem_name} gem is installed."
+    end
+  end
+end

data/lib/bx_builder_chain/llm/base.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module BxBuilderChain::Llm
+  class ApiError < StandardError; end
+
+  # A LLM is a language model consisting of a neural network with many parameters (typically billions of weights or more), trained on large quantities of unlabeled text using self-supervised learning or semi-supervised learning.
+  #
+  # BxBuilderChain.rb provides a common interface to interact with all supported LLMs:
+  #
+  # - {BxBuilderChain::Llm::OpenAI}
+  #
+  # @abstract
+  class Base
+    include BxBuilderChain::DependencyHelper
+
+    # A client for communicating with the LLM
+    attr_reader :client
+
+    def default_dimension
+      self.class.const_get(:DEFAULTS).dig(:dimension)
+    end
+
+    #
+    # Generate a chat completion for a given prompt. Parameters will depend on the LLM
+    #
+    # @raise NotImplementedError if not supported by the LLM
+    def chat(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support chat"
+    end
+
+    #
+    # Generate a completion for a given prompt. Parameters will depend on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
+    def complete(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support completion"
+    end
+
+    #
+    # Generate an embedding for a given text. Parameters depends on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
+    #
+    def embed(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support generating embeddings"
+    end
+
+    #
+    # Generate a summary for a given text. Parameters depends on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
+    #
+    def summarize(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support summarization"
+    end
+
+    def count_tokens(string)
+      tokens = string.scan(/[\w]+|[\W]/)
+      tokens = tokens.flat_map { |token| token.split(/(?<=[\W])$/) }
+
+      return (tokens.length*1.07).to_i
+    end
+  end
+end