bx_builder_chain 0.1.0

data/lib/generators/bx_builder_chain/install_generator.rb

@@ -0,0 +1,42 @@
+require 'rails/generators'
+require 'rails/generators/active_record'
+
+module BxBuilderChain
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path('templates', __dir__)
+
+      desc "Creates BxBuilderChain initializer, migration, and copies app templates for your application"
+
+      def copy_initializer
+        template "initializer.rb", "config/initializers/bx_builder_chain.rb"
+      end
+
+      def copy_migration
+        timestamp_number = Time.now.strftime("%Y%m%d%H%M%S")
+        template "migration.rb", "db/migrate/#{timestamp_number}_create_bx_builder_chain_schema.rb"
+      end
+
+      def copy_app_templates
+        directory "app", Rails.root.join("app")
+      end
+
+      def add_routes
+        inject_into_file 'config/routes.rb', after: "Rails.application.routes.draw do\n" do
+          <<~ROUTES
+  namespace :bx_builder_chain do
+    get 'test_form', to: 'test#form' # remove before production
+
+    post 'documents/upload', to: 'documents#upload_and_process'
+    post 'documents/upload_for_later', to: 'documents#upload_and_process_later'
+    get 'documents/list', to: 'documents#namespace_documents'
+    delete 'documents/delete', to: 'documents#delete_documents'
+
+    post 'ask', to: 'questions#ask'
+  end
+          ROUTES
+        end
+      end
+    end
+  end
+end

data/lib/generators/bx_builder_chain/templates/app/admin/bx_builder_chain_document.rb

@@ -0,0 +1,65 @@
+ActiveAdmin.register BxBuilderChain::Document, as: "Document Embeddings" do
+  permit_params :name, :namespace
+
+  controller do
+    def scoped_collection
+      super.includes(:document_chunks)
+    end
+
+    def create
+
+      uploaded_files = params[:bx_builder_chain_document][:files]
+      namespace = params[:bx_builder_chain_document][:namespace]
+
+      user_groups = namespace.to_s.split(',').reject(&:blank?)
+
+      service = BxBuilderChain::DocumentUploadService.new(
+        files: uploaded_files,
+        current_user_groups: user_groups,
+        client_class_name: 'BxBuilderChain::Vectorsearch::Pgvector',
+        llm_class_name: 'BxBuilderChain::Llm::OpenAi'
+      )
+      result = service.upload_and_process
+
+      if result[:error]
+        flash.now[:error] = JSON.parse(response.body)["error"]
+        render :new
+      else
+        redirect_to admin_bx_builder_chain_documents_path, notice: 'Document uploaded and processed successfully.'
+      end
+    end
+  end
+
+  index do
+    selectable_column
+    id_column
+    column :name
+    column :namespace
+    column "Document Chunks" do |document|
+      document.document_chunks.size
+    end
+    column :created_at
+
+    # # Custom actions
+    # actions defaults: false do |document|
+    #   link_to 'Delete', admin_bx_builder_chain_document_path(document), method: :delete, data: { confirm: 'Are you sure?' }
+    # end
+    actions
+  end
+
+  filter :name
+  filter :namespace
+  filter :created_at
+
+  form html: { multipart: true } do |f|
+    f.inputs do
+      if f.object.new_record?
+        f.input :files, as: :file, input_html: { multiple: true }
+      else
+        f.input :name, as: :string, input_html: { disabled: true }
+      end
+      f.input :namespace, as: :string
+    end
+    f.actions
+  end
+end

data/lib/generators/bx_builder_chain/templates/app/controllers/bx_builder_chain/documents_controller.rb

@@ -0,0 +1,65 @@
+module BxBuilderChain
+  class DocumentsController < ::ApplicationController
+    skip_before_action :verify_authenticity_token
+
+    LLM_CLASS_NAME = 'BxBuilderChain::Llm::OpenAi'
+    CLIENT_CLASS_NAME = 'BxBuilderChain::Vectorsearch::Pgvector'
+    
+    def namespace_documents
+      documents = BxBuilderChain::Document.where(namespace: current_user_document_groups.first)
+      render json: documents
+    end
+
+    def delete_documents
+      return render json: { error: 'No document IDs provided' }, status: :bad_request unless params[:ids].present?
+    
+      # Filter documents by the provided IDs and the namespace from the current user groups
+      documents_to_delete = BxBuilderChain::Document.where(id: params[:ids], namespace: current_user_document_groups.first)
+    
+      # Destroy the documents
+      documents_to_delete.destroy_all
+    
+      render json: { success: 'Documents deleted successfully' }
+    end
+
+    def upload_and_process
+      service = DocumentUploadService.new(
+        files: params[:files],
+        user_groups: current_user_document_groups, # optional defaults to ['public']
+        client_class_name: CLIENT_CLASS_NAME, # optional defaults to 'BxBuilderChain::Vectorsearch::Pgvector'
+        llm_class_name: LLM_CLASS_NAME # optional defaults to 'BxBuilderChain::Llm::OpenAi'
+      )
+
+      result = service.upload_and_process
+
+      if result[:error]
+        render json: { error: result[:error] }, status: :bad_request
+      else
+        render json: result
+      end
+    end
+
+    def upload_and_process_later
+      service = DocumentUploadService.new(
+        files: params[:files],
+        current_user_groups: current_user_document_groups,
+        client_class_name: CLIENT_CLASS_NAME,
+        llm_class_name: LLM_CLASS_NAME
+      )
+      result = service.upload_and_process_later
+
+      if result[:error]
+        render json: { error: result[:error] }, status: :bad_request
+      else
+        render json: result
+      end
+    end
+
+    private
+
+    def current_user_document_groups
+      params[:current_user_groups].to_s.split(',').reject(&:blank?) 
+      # replace this with the actual user document permission groups
+    end   
+  end
+end

data/lib/generators/bx_builder_chain/templates/app/controllers/bx_builder_chain/questions_controller.rb

@@ -0,0 +1,33 @@
+module BxBuilderChain
+  class QuestionsController < ::ApplicationController
+    skip_before_action :verify_authenticity_token
+
+    LLM_CLASS_NAME = 'BxBuilderChain::Llm::OpenAi'
+    CLIENT_CLASS_NAME = 'BxBuilderChain::Vectorsearch::Pgvector'
+
+    # POST /bx_builder_chain/ask
+    def ask
+      service = QuestionAskingService.new(
+        question: params[:question],
+        user_groups: current_user_document_groups, # optional defaults to ['public']
+        client_class_name: CLIENT_CLASS_NAME, # optional defaults to 'BxBuilderChain::Vectorsearch::Pgvector'
+        llm_class_name: LLM_CLASS_NAME, # optional defaults to 'BxBuilderChain::Llm::OpenAi'
+        context_results: 10 # optional defaults to 6
+      )
+      response = service.ask
+
+      if response[:error]
+        render json: response, status: :bad_request
+      else
+        render json: response
+      end
+    end
+
+    private
+
+    def current_user_document_groups 
+      params[:current_user_groups].to_s.split(',').reject(&:blank?) 
+      # replace this with the actual user document permission groups
+    end
+  end
+end

data/lib/generators/bx_builder_chain/templates/app/controllers/bx_builder_chain/test_controller.rb

@@ -0,0 +1,10 @@
+module BxBuilderChain
+    class TestController < ::ApplicationController
+      skip_before_action :verify_authenticity_token
+  
+      def form
+
+      end
+    end
+  end
+  

data/lib/generators/bx_builder_chain/templates/app/models/bx_builder_chain/document.rb

@@ -0,0 +1,26 @@
+module BxBuilderChain
+  class Document < ::ApplicationRecord
+    self.table_name = "bx_builder_chain_documents"
+    
+    has_many :document_chunks, class_name: 'BxBuilderChain::DocumentChunk'
+    has_many :embeddings, through: :document_chunks
+    
+    before_destroy :destroy_associated_data
+    after_update :update_associated_embedding_namespaces, if: :saved_change_to_namespace?
+
+    private
+
+    def update_associated_embedding_namespaces
+      embeddings.update_all(namespace: namespace)
+    end
+
+    def destroy_associated_data
+      # Delete all associated embeddings without loading them into memory
+      embedding_ids = BxBuilderChain::DocumentChunk.where(document_id: id).pluck(:embedding_id)
+      BxBuilderChain::Embedding.where(id: embedding_ids).delete_all
+
+      # Delete all chunks without loading them
+      BxBuilderChain::DocumentChunk.where(document_id: id).delete_all
+    end
+  end
+end

data/lib/generators/bx_builder_chain/templates/app/models/bx_builder_chain/document_chunk.rb

@@ -0,0 +1,9 @@
+module BxBuilderChain
+    class DocumentChunk < ::ApplicationRecord
+      self.table_name = "bx_builder_chain_document_chunks"
+      
+      belongs_to :document, class_name: 'BxBuilderChain::Document'
+      belongs_to :embedding, class_name: 'BxBuilderChain::Embedding'
+    end
+  end
+  

data/lib/generators/bx_builder_chain/templates/app/models/bx_builder_chain/embedding.rb

@@ -0,0 +1,9 @@
+module BxBuilderChain
+    class Embedding < ::ApplicationRecord
+      self.table_name = "bx_builder_chain_embeddings"
+      
+      has_many :document_chunks, class_name: 'BxBuilderChain::DocumentChunk', dependent: :destroy
+      has_many :documents, through: :document_chunks
+    end
+  end
+  

data/lib/generators/bx_builder_chain/templates/app/services/bx_builder_chain/document_upload_service.rb

@@ -0,0 +1,47 @@
+module BxBuilderChain
+    class DocumentUploadService
+      attr_reader :files, :user_groups, :llm_class_name, :client_class_name
+
+      WAITING_FOLDER = 'path/to/waiting_folder' # Update this path
+  
+      def initialize(files:, user_groups: ['public'], client_class_name: 'BxBuilderChain::Vectorsearch::Pgvector', llm_class_name: 'BxBuilderChain::Llm::OpenAi')
+        @files = files
+        @user_groups = user_groups
+        @client_class_name = client_class_name
+        @llm_class_name = llm_class_name
+      end
+  
+      def upload_and_process
+        return { error: 'No files provided' } unless @files&.any?
+  
+        files_n_paths = @files.map { |file| { path: file.tempfile.path, filename: file.original_filename } }
+        client.add_data(paths: files_n_paths)
+  
+        { success: 'Files added to document store' }
+      end
+  
+      def upload_and_process_later
+        return { error: 'No files provided' } unless @files&.any?
+  
+        @files.each do |file|
+          new_path = File.join(WAITING_FOLDER, file.original_filename)
+          FileUtils.mv(file.tempfile.path, new_path)
+          file_n_path = { path: new_path, filename: file.original_filename }
+          # Enqueue for processing with Sidekiq
+          BxBuilderChain::DocumentProcessorWorker.perform_async(file_n_path, llm_class: @llm_class_name, client_class: @client_class_name, namespaces: @user_groups)
+        end
+  
+        { success: 'Files queued for processing' }
+      end
+  
+      private
+  
+      def client
+        @client ||= @client_class_name.constantize.new(
+                          llm: @llm_class_name.constantize.new,
+                          namespaces: @user_groups
+                    )
+      end
+    end
+  end
+  

data/lib/generators/bx_builder_chain/templates/app/services/bx_builder_chain/question_asking_service.rb

@@ -0,0 +1,35 @@
+module BxBuilderChain
+    class QuestionAskingService
+      attr_reader :question, :user_groups, :llm_class_name, :client_class_name, :context_results
+
+      def initialize(question:, user_groups: ['public'], client_class_name: 'BxBuilderChain::Vectorsearch::Pgvector', llm_class_name: 'BxBuilderChain::Llm::OpenAi', context_results: 6)
+        @question = question
+        @user_groups = user_groups
+        @client_class_name = client_class_name
+        @llm_class_name = llm_class_name
+        @context_results = context_results
+      end
+  
+      def ask
+        return { error: 'No question provided' } unless question.present?
+  
+        response = client.ask(question: question, context_results: context_results)
+        { answer: response }
+      end
+  
+      private
+  
+      def client
+        @client ||= client_class_name.constantize.new(
+          llm: llm_class_name.constantize.new(
+            default_options: {
+              chat_completion_model_name: "gpt-3.5-turbo", 
+              temperature: 0.2
+            }
+          ),
+          namespaces: user_groups
+        )
+      end
+    end
+  end
+  

data/lib/generators/bx_builder_chain/templates/app/views/bx_builder_chain/test/form.html.erb

@@ -0,0 +1,164 @@
+<style>
+    body {
+        font-family: 'Arial', sans-serif;
+        background-color: #f5f5f5;
+    }
+
+    div.page_wrapper {
+        max-width: 1100px;
+        margin: 5% auto;
+        padding: 20px;
+        background-color: #fff;
+        box-shadow: 0 3px 6px rgba(0, 0, 0, 0.1);
+        border-radius: 5px;
+    }
+
+    div.form_wrapper {
+        width: 50%;
+        padding: 1%;
+        float: left;
+        background-color: #f9f9f9;
+        border-radius: 5px;
+        box-sizing: border-box;
+        height: 340px;
+    }
+
+    form {
+        width: 100%;
+    }
+
+    h2 {
+        font-size: 1.5em;
+        margin-bottom: 20px;
+        color: #333;
+    }
+
+    .form-group {
+        margin-bottom: 15px;
+    }
+
+    .form-control {
+        width: 93%;
+        padding: 8px 12px;
+        border: 1px solid #ccc;
+        border-radius: 4px;
+    }
+
+    .btn-primary {
+        background-color: #007bff;
+        color: #fff;
+        padding: 10px 15px;
+        border: none;
+        border-radius: 4px;
+        cursor: pointer;
+        transition: background-color 0.3s ease;
+    }
+
+    .btn-primary:hover {
+        background-color: #0056b3;
+    }
+    div.console_wrapper {
+        width: 100%;
+        margin-top: 20px;
+        padding: 15px;
+        background-color: #000;
+        color: #0f0;
+        font-family: 'Courier New', monospace;
+        border-radius: 5px;
+        min-height: 150px;
+        box-sizing: border-box;
+        overflow: auto;
+    }
+
+    .console_message {
+        white-space: pre-line; /* To respect new lines in the content */
+    }
+</style>
+
+<div class="page_wrapper">
+    <div class="form_wrapper">
+        <h2>Upload Files</h2>
+        <%= form_tag(bx_builder_chain_documents_upload_path, multipart: true, method: :post, id: 'upload_form') do %>
+            <div class="form-group">
+                <label for="files">Choose files to upload:</label>
+                <%= file_field_tag 'files[]', multiple: true, class: 'form-control' %>
+            </div>
+            <div class="form-group">
+                <label for="files">Choose files to upload:</label>
+                <%= file_field_tag 'files[]', multiple: true, class: 'form-control' %>
+            </div>
+
+            <div class="form-group">
+                <label for="current_user_groups">User ID:</label>
+                <%= text_field_tag 'current_user_groups', 1, class: 'form-control' %>
+            </div>
+
+            <div class="actions">
+                <%= submit_tag "Upload", class: 'btn btn-primary' %>
+            </div>
+        <% end %>
+    </div>
+
+    <div class="form_wrapper">
+        <h2>Ask Question</h2>
+
+        <%= form_tag(bx_builder_chain_ask_path, method: :post, id: 'ask_form') do %>
+            <div class="form-group">
+                <label for="question">Ask a question:</label>
+                <%= text_area_tag 'question', nil, class: 'form-control', rows: 3 %>
+            </div>
+
+            <div class="form-group">
+                <label for="current_user_groups">User Groups:</label>
+                <%= text_field_tag 'current_user_groups', 1, class: 'form-control' %>
+            </div>
+
+            <div class="actions">
+                <%= submit_tag "Submit Question", class: 'btn btn-primary' %>
+            </div>
+        <% end %>
+    </div>
+    <div style="clear: both;"></div>
+
+    <!-- Console box -->
+    <div class="console_wrapper">
+        <div class="console_message">
+            <%= flash[:console_message] || "Awaiting response..." %>
+        </div>
+    </div>
+</div>
+<script>
+    document.addEventListener('DOMContentLoaded', function () { // Selecting both forms
+        const uploadForm = document.querySelector('#upload_form');
+        const askForm = document.querySelector('#ask_form');
+        const consoleBox = document.querySelector('.console_message');
+
+        // Attach event listener to both forms
+        [uploadForm, askForm].forEach(form => {
+            form.addEventListener('submit', async function (event) {
+                event.preventDefault();
+                // Prevent default form submission
+
+                // Show loading message
+                consoleBox.textContent = "Processing...";
+
+                try {
+                    let response = await fetch(event.target.action, {
+                        method: 'POST',
+                        body: new FormData(event.target) // This gathers form data
+                    });
+
+                    if (response.ok) {
+                        let data = await response.json();
+                        consoleBox.textContent = data.answer || data.success || "Success!";
+                    } else {
+                        let errorData = await response.json();
+                        consoleBox.textContent = errorData.error || "An error occurred.";
+                    }
+                } catch (error) {
+                    consoleBox.textContent = "Failed to submit. Please try again.";
+                }
+            });
+        });
+    });
+</script>

data/lib/generators/bx_builder_chain/templates/app/workers/bx_builder_chain/document_processor_worker.rb

@@ -0,0 +1,32 @@
+require 'sidekiq'
+
+module BxBuilderChain
+  class DocumentProcessorWorker
+    include Sidekiq::Worker
+
+    sidekiq_options retry: 2 
+
+    def perform(file_data, llm_class:, client_class:, namespaces:)
+      # Create a new instance of the service class
+      service = DocumentUploadService.new(
+        files: [file_data],
+        user_groups: namespaces,
+        client_class_name: client_class,
+        llm_class_name: llm_class
+      )
+
+      # Use the service method to process the file
+      result = service.upload_and_process
+
+      # Log errors if they occur
+      if result[:error]
+        BxBuilderChain.logger.error("BxBuilderChain::DocumentProcessorWorker Error: #{result[:error]}\n File: #{file_data[:filename]}\nNameSpace: #{namespaces.join(', ')}")
+      end
+
+    rescue => e
+        BxBuilderChain.logger.error("BxBuilderChain::DocumentProcessorWorker Error: #{e.message}\n File: #{file_data[:filename]}\nNameSpace: #{namespaces.join(', ')}")
+        # Re-raise the exception to let Sidekiq handle retries
+        raise e
+    end
+  end
+end

data/lib/generators/bx_builder_chain/templates/initializer.rb

@@ -0,0 +1,12 @@
+BxBuilderChain.configure do |config|
+  config.pg_url = "your_database_url" # eg 'postgres://postgres:password@localhost:5432/my_db'
+  config.openai_api_key = "your_openai_api_key" # eg ENV['OPENAI_API_KEY']
+  config.public_namespace = "public"
+  config.threshold = 0.25
+  config.default_prompt_template = "Context information is below
+  --------------------
+  %{context}
+  --------------------
+  Given the context information and not prior knowledge
+  answer the question: %{question}"
+end

data/lib/generators/bx_builder_chain/templates/migration.rb

@@ -0,0 +1,33 @@
+class CreateBxBuilderChainSchema < ActiveRecord::Migration[6.1]
+  def change
+    enable_extension 'vector' unless extension_enabled?('vector')
+
+    # bx_builder_chain_embeddings table
+    create_table :bx_builder_chain_embeddings do |t|
+      t.text :content
+      t.column :vectors, "vector(1536)"
+      t.text :namespace, default: 'public'
+      
+      t.index :namespace
+    end
+
+    # bx_builder_chain_documents table
+    create_table :bx_builder_chain_documents do |t|
+      t.text :name
+      t.text :namespace, default: 'public'
+      
+      t.index [:name, :namespace], unique: true
+      t.index :namespace
+      t.timestamps
+    end
+
+    # bx_builder_chain_document_chunks table
+    create_table :bx_builder_chain_document_chunks do |t|
+      t.references :document, null: false, foreign_key: { to_table: :bx_builder_chain_documents, on_delete: :cascade }
+      t.references :embedding, null: false, foreign_key: { to_table: :bx_builder_chain_embeddings, on_delete: :cascade }
+    end
+
+    # Unique constraint for combination of document_id and embedding_id
+    add_index :bx_builder_chain_document_chunks, [:document_id, :embedding_id], unique: true, name: 'index_document_embedding_unique'
+  end
+end

data/lib/pgvector/pg/binary_decoder/vector.rb

@@ -0,0 +1,14 @@
+module Pgvector
+    module PG
+      module BinaryDecoder
+        class Vector < ::PG::SimpleDecoder
+          def decode(string, tuple = nil, field = nil)
+            dim, unused = string[0, 4].unpack("nn")
+            raise "expected unused to be 0" if unused != 0
+            string[4..-1].unpack("g#{dim}")
+          end
+        end
+      end
+    end
+  end
+  

data/lib/pgvector/pg/text_decoder/vector.rb

@@ -0,0 +1,12 @@
+module Pgvector
+    module PG
+      module TextDecoder
+        class Vector < ::PG::SimpleDecoder
+          def decode(string, tuple = nil, field = nil)
+            Pgvector.decode(string)
+          end
+        end
+      end
+    end
+  end
+  

data/lib/pgvector/pg.rb

@@ -0,0 +1,10 @@
+require "pg"
+
+module Pgvector
+  module PG
+    def self.register_vector(registry)
+      registry.register_type(0, "vector", nil, TextDecoder::Vector)
+      registry.register_type(1, "vector", nil, BinaryDecoder::Vector)
+    end
+  end
+end

data/lib/pgvector.rb

@@ -0,0 +1,11 @@
+module Pgvector
+  autoload :PG, "pgvector/pg"
+
+  def self.encode(data)
+    "[#{data.to_a.map(&:to_f).join(",")}]"
+  end
+
+  def self.decode(string)
+    string[1..-2].split(",").map(&:to_f)
+  end
+end