bx_builder_chain 0.1.0

data/lib/sequel/plugins/pgvector/class_methods.rb

@@ -0,0 +1,47 @@
+module Sequel
+    module Plugins
+        module Pgvector
+            module ClassMethods
+                attr_accessor :vector_columns
+
+                def nearest_neighbors(column, value, distance:, threshold: nil)
+                    value = ::Pgvector.encode(value) unless value.is_a?(String)
+                    quoted_column = dataset.quote_identifier(column)
+                    distance = distance.to_s
+                
+                    operator =
+                        case distance
+                        when "inner_product"
+                        "<#>"
+                        when "cosine"
+                        "<=>"
+                        when "euclidean"
+                        "<->"
+                        else
+                        raise ArgumentError, "Invalid distance: #{distance}"
+                        end
+                
+                    order = "#{quoted_column} #{operator} ?"
+                
+                    neighbor_distance =
+                        if distance == "inner_product"
+                        "(#{order}) * -1"
+                        else
+                        order
+                        end
+                
+                    query = select_append(Sequel.lit("#{neighbor_distance} AS neighbor_distance", value))
+                        .exclude(column => nil)
+                        .order(Sequel.lit(order, value))
+                
+                    # Apply the WHERE condition only if threshold is provided
+                    query = query.where(Sequel.lit("(#{neighbor_distance}) < ?", value, threshold)) if threshold
+                
+                    query
+                end                
+
+                Plugins.inherited_instance_variables(self, :@vector_columns => :dup)
+            end
+        end
+    end
+end

data/lib/sequel/plugins/pgvector/instance_methods.rb

@@ -0,0 +1,34 @@
+module Sequel
+    module Plugins
+      module Pgvector
+        module InstanceMethods
+          def nearest_neighbors(column, **options)
+            column = column.to_sym
+            # important! check if neighbor attribute before calling send
+            raise ArgumentError, "Invalid column" unless self.class.vector_columns[column]
+  
+            self.class
+              .nearest_neighbors(column, self[column], **options)
+              .exclude(primary_key => self[primary_key])
+          end
+  
+          def []=(k, v)
+            if self.class.vector_columns.key?(k.to_sym) && !v.is_a?(String)
+              super(k, ::Pgvector.encode(v))
+            else
+              super
+            end
+          end
+  
+          def [](k)
+            if self.class.vector_columns.key?(k.to_sym)
+              ::Pgvector.decode(super)
+            else
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+  

data/lib/sequel/plugins/pgvector.rb

@@ -0,0 +1,12 @@
+module Sequel
+  module Plugins
+    module Pgvector
+      def self.configure(model, *columns)
+        model.vector_columns ||= {}
+        columns.each do |column|
+          model.vector_columns[column.to_sym] = {}
+        end
+      end
+    end
+  end
+end

data/sig/bx_langchain_chat.rbs

@@ -0,0 +1,4 @@
+module BxBuilderChain
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,238 @@
+--- !ruby/object:Gem::Specification
+name: bx_builder_chain
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Paul Ketelle
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-08-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.6.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.6.11
+- !ruby/object:Gem::Dependency
+  name: baran
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.7
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.71'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.71'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.0
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: docx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+description: Write a longer description or delete this line.
+email:
+- paul.ketelle@builder.ai
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bx_builder_chain.gemspec
+- lib/bx_builder_chain.rb
+- lib/bx_builder_chain/chunker/recursive_text.rb
+- lib/bx_builder_chain/chunker/text.rb
+- lib/bx_builder_chain/configuration.rb
+- lib/bx_builder_chain/data.rb
+- lib/bx_builder_chain/dependency_helper.rb
+- lib/bx_builder_chain/llm/base.rb
+- lib/bx_builder_chain/llm/open_ai.rb
+- lib/bx_builder_chain/loader.rb
+- lib/bx_builder_chain/processors/base.rb
+- lib/bx_builder_chain/processors/csv.rb
+- lib/bx_builder_chain/processors/docx.rb
+- lib/bx_builder_chain/processors/html.rb
+- lib/bx_builder_chain/processors/json.rb
+- lib/bx_builder_chain/processors/pdf.rb
+- lib/bx_builder_chain/processors/text.rb
+- lib/bx_builder_chain/processors/xlsx.rb
+- lib/bx_builder_chain/utils/token_data/cl100k_base.tiktoken
+- lib/bx_builder_chain/utils/token_length/base_validator.rb
+- lib/bx_builder_chain/utils/token_length/open_ai_validator.rb
+- lib/bx_builder_chain/utils/tokenization/byte_pair_encoding.rb
+- lib/bx_builder_chain/utils/tokenization/open_ai_encodings.rb
+- lib/bx_builder_chain/vectorsearch/base.rb
+- lib/bx_builder_chain/vectorsearch/pgvector.rb
+- lib/bx_builder_chain/version.rb
+- lib/generators/bx_builder_chain/install_generator.rb
+- lib/generators/bx_builder_chain/templates/app/admin/bx_builder_chain_document.rb
+- lib/generators/bx_builder_chain/templates/app/controllers/bx_builder_chain/documents_controller.rb
+- lib/generators/bx_builder_chain/templates/app/controllers/bx_builder_chain/questions_controller.rb
+- lib/generators/bx_builder_chain/templates/app/controllers/bx_builder_chain/test_controller.rb
+- lib/generators/bx_builder_chain/templates/app/models/bx_builder_chain/document.rb
+- lib/generators/bx_builder_chain/templates/app/models/bx_builder_chain/document_chunk.rb
+- lib/generators/bx_builder_chain/templates/app/models/bx_builder_chain/embedding.rb
+- lib/generators/bx_builder_chain/templates/app/services/bx_builder_chain/document_upload_service.rb
+- lib/generators/bx_builder_chain/templates/app/services/bx_builder_chain/question_asking_service.rb
+- lib/generators/bx_builder_chain/templates/app/views/bx_builder_chain/test/form.html.erb
+- lib/generators/bx_builder_chain/templates/app/workers/bx_builder_chain/document_processor_worker.rb
+- lib/generators/bx_builder_chain/templates/initializer.rb
+- lib/generators/bx_builder_chain/templates/migration.rb
+- lib/pgvector.rb
+- lib/pgvector/pg.rb
+- lib/pgvector/pg/binary_decoder/vector.rb
+- lib/pgvector/pg/text_decoder/vector.rb
+- lib/sequel/plugins/pgvector.rb
+- lib/sequel/plugins/pgvector/class_methods.rb
+- lib/sequel/plugins/pgvector/instance_methods.rb
+- sig/bx_langchain_chat.rbs
+homepage:
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.0
+signing_key:
+specification_version: 4
+summary: Write a short summary, because RubyGems requires one.
+test_files: []