boxcars 0.2.11 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da3d8b9838602151837c0cc5bb9f3cae841ba24d1c338eade82c7807d913d4bb
-  data.tar.gz: f7be434c18f0ff2c95625fe32fae25f3a5df265331425d0c1f0430ab75761578
+  metadata.gz: 74f14f8575e4670d2be6196c5196d41dd9728b5a44a0d4e199dfb705dfc77ed5
+  data.tar.gz: 06f2e8178f9696831870b5d8d5ea40bda8ba74a2fcc27283849f49124c51a06b
 SHA512:
-  metadata.gz: 57dd238c56f13f63a4665d4469efdabfa5f3c54f82cb6832c4158858d4b307a80c57f619633cdad6934d64186d560dcab7a62efa9adc727edfa61afbc5acc188
-  data.tar.gz: d2c782acf20c6b6b13cbfadf8f5406363b347be90a058626ec1bb21fe32baf1acb57a4a72c4770a7ad820700b465c0474a498080604e23e5d0270001d5d4aec1
+  metadata.gz: 9ff6e759f3d942f859de85763ffe9bc0ccf5636914d894205f78cffc99abb1b27dea47463f0fc968eba6746c055f796c95884d5e421486e06374fb0519eb8c63
+  data.tar.gz: 03bf42b1fbd6dac1eff4734bd2443a4a2a9f7cb931c99a2fe3f9453f5a23f0853b4eb26c817e1757a16629617eff4704f276b99fb165689ef6016ace86c2fb56

data/.env_sample

@@ -1,2 +1,3 @@
 openai_access_token: ''
 serpapi_api_key: ''
+DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/boxcars_test'

data/.rubocop.yml

@@ -3,6 +3,7 @@ require:
   - rubocop-rake
 
 AllCops:
+  TargetRubyVersion: 3
   Exclude:
     - 'bin/{rails,rake}'
     - 'node_modules/**/*'
@@ -152,3 +153,18 @@ Style/SlicingWithRange:
 
 Bundler/OrderedGems:
   Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+RSpec/PendingWithoutReason:
+  Enabled: false
+
+RSpec/NestedGroups:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/MultipleExpectations:
+  Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [v0.2.11](https://github.com/BoxcarsAI/boxcars/tree/v0.2.11) (2023-05-05)
+
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.10...v0.2.11)
+
+**Closed issues:**
+
+- Chore: move vector store to top level [\#67](https://github.com/BoxcarsAI/boxcars/issues/67)
+
+**Merged pull requests:**
+
+- Move vector store [\#69](https://github.com/BoxcarsAI/boxcars/pull/69) ([francis](https://github.com/francis))
+
 ## [v0.2.10](https://github.com/BoxcarsAI/boxcars/tree/v0.2.10) (2023-05-05)
 
 [Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.9...v0.2.10)

data/Gemfile

@@ -5,19 +5,11 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in boxcars.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
-
-gem "rspec", "~> 3.2"
-
-gem "rubocop", "~> 1.21"
-
-gem "vcr", "~> 6.1.0"
+gem "debug", "~> 1.1"
 
-gem "webmock", "~> 3.18.1"
+gem "dotenv", "~> 2.8"
 
-gem "rubocop-rake", "~> 0.6.0"
-
-gem "rubocop-rspec", "~> 2.17"
+gem "rake", "~> 13.0"
 
 gem "sqlite3", "~> 1.6"
 
@@ -32,3 +24,15 @@ gem "activesupport", "~> 7.0"
 gem "rest-client", "~> 2.1"
 
 gem "hnswlib", "~> 0.8.1"
+
+gem "pg", "~> 1.5", ">= 1.5.3"
+gem "pgvector", "~> 0.2.0"
+
+group :development, :test do
+  gem "rspec", "~> 3.2"
+  gem "rubocop", "~> 1.21"
+  gem "vcr", "~> 6.1.0"
+  gem "webmock", "~> 3.18.1"
+  gem "rubocop-rake", "~> 0.6.0"
+  gem "rubocop-rspec", "~> 2.17"
+end

data/Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.11)
+    boxcars (0.2.13)
       google_search_results (~> 2.2)
       gpt4all (~> 0.0.4)
-      ruby-openai (~> 3.0)
+      hnswlib (~> 0.8)
+      pgvector (~> 0.2)
+      ruby-openai (~> 4.1)
 
 GEM
   remote: https://rubygems.org/
@@ -19,7 +21,7 @@ GEM
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.1)
+    addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     async (1.31.0)
@@ -34,7 +36,7 @@ GEM
       protocol-http1 (~> 0.15.0)
       protocol-http2 (~> 0.15.0)
       traces (>= 0.8.0)
-    async-http-faraday (0.11.0)
+    async-http-faraday (0.12.0)
       async-http (~> 0.42)
       faraday
     async-io (1.34.3)
@@ -46,7 +48,7 @@ GEM
       fiber-local
     crack (0.4.5)
       rexml
-    debug (1.7.2)
+    debug (1.8.0)
       irb (>= 1.5.0)
       reline (>= 0.3.1)
     diff-lcs (1.5.0)
@@ -56,8 +58,10 @@ GEM
     faraday (2.7.4)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
-    faraday-http-cache (2.4.1)
+    faraday-http-cache (2.5.0)
       faraday (>= 0.8)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
     faraday-net_http (3.0.2)
     faraday-retry (2.1.0)
       faraday (~> 2.0)
@@ -81,35 +85,33 @@ GEM
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
-    httparty (0.21.0)
-      mini_mime (>= 1.0.0)
-      multi_xml (>= 0.5.2)
-    i18n (1.12.0)
+    i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     io-console (0.6.0)
     io-console (0.6.0-java)
-    irb (1.6.3)
+    irb (1.6.4)
       reline (>= 0.3.0)
     json (2.6.3)
     json (2.6.3-java)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0218.1)
-    mini_mime (1.1.2)
-    mini_portile2 (2.8.1)
+    mini_portile2 (2.8.2)
     minitest (5.18.0)
     multi_json (1.15.0)
-    multi_xml (0.6.0)
+    multipart-post (2.3.0)
     netrc (0.11.0)
-    nio4r (2.5.8)
-    nio4r (2.5.8-java)
+    nio4r (2.5.9)
+    nio4r (2.5.9-java)
     octokit (4.25.1)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
     os (1.1.4)
-    parallel (1.22.1)
-    parser (3.2.1.1)
+    parallel (1.23.0)
+    parser (3.2.2.1)
       ast (~> 2.4.1)
+    pg (1.5.3)
+    pgvector (0.2.0)
     protocol-hpack (1.4.2)
     protocol-http (0.24.1)
     protocol-http1 (0.15.0)
@@ -120,7 +122,7 @@ GEM
     public_suffix (5.0.1)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.7.0)
+    regexp_parser (2.8.0)
     reline (0.3.3)
       io-console (~> 0.5)
     rest-client (2.1.0)
@@ -133,46 +135,50 @@ GEM
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.1)
+    rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.48.1)
+    rubocop (1.51.0)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.2.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.26.0, < 2.0)
+      rubocop-ast (>= 1.28.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.0)
+    rubocop-ast (1.28.1)
       parser (>= 3.2.1.0)
-    rubocop-capybara (2.17.1)
+    rubocop-capybara (2.18.0)
       rubocop (~> 1.41)
+    rubocop-factory_bot (2.23.1)
+      rubocop (~> 1.33)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.19.0)
+    rubocop-rspec (2.22.0)
       rubocop (~> 1.33)
       rubocop-capybara (~> 2.17)
-    ruby-openai (3.7.0)
-      httparty (>= 0.18.1)
+      rubocop-factory_bot (~> 2.22)
+    ruby-openai (4.1.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     sawyer (0.9.2)
       addressable (>= 2.3.5)
       faraday (>= 0.17.3, < 3)
-    sqlite3 (1.6.2)
+    sqlite3 (1.6.3)
       mini_portile2 (~> 2.8.0)
-    sqlite3 (1.6.2-arm64-darwin)
-    sqlite3 (1.6.2-x86_64-darwin)
-    sqlite3 (1.6.2-x86_64-linux)
+    sqlite3 (1.6.3-arm64-darwin)
+    sqlite3 (1.6.3-x86_64-darwin)
+    sqlite3 (1.6.3-x86_64-linux)
     strings-ansi (0.2.0)
     timers (4.3.5)
     traces (0.9.1)
@@ -212,6 +218,8 @@ DEPENDENCIES
   faraday-retry (~> 2.0)
   github_changelog_generator (~> 1.16)
   hnswlib (~> 0.8.1)
+  pg (~> 1.5, >= 1.5.3)
+  pgvector (~> 0.2.0)
   rake (~> 13.0)
   rest-client (~> 2.1)
   rspec (~> 3.2)

data/README.md

@@ -21,6 +21,7 @@ All of these concepts are in a module named Boxcars:
 - Train - Given a list of Boxcars and optionally an Engine, a Train breaks down a problem into pieces for individual Boxcars to solve. The individual results are then combined until a final answer is found. ZeroShot is the only current implementation of Train (but we are adding more soon), and you can either construct it directly or use `Boxcars::train` when you want to build a Train.
 - Prompt - used by an Engine to generate text results. Our Boxcars have built-in prompts, but you have the flexibility to change or augment them if you so desire.
 - Engine - an entity that generates text from a Prompt. OpenAI's LLM text generator is the default Engine if no other is specified, and you can override the default engine if so desired (`Boxcar.configuration.default_engine`).
+- VectorStore - a place to store and query vectors.
 
 ## Security
 Currently, our system is designed for individuals who already possess administrative privileges for their project. It is likely possible to manipulate the system's prompts to carry out malicious actions, but if you already have administrative access, you can perform such actions without requiring boxcars in the first place.
@@ -132,7 +133,9 @@ Next Actions:
 ### More Examples
 See [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/boxcars_examples.ipynb) Jupyter Notebook for more examples.
 
-For the new Swagger boxcar, see [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/swagger_examples.ipynb) Jupyter Notebook.
+For the Swagger boxcar, see [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/swagger_examples.ipynb) Jupyter Notebook.
+
+For simple vector storage and search, see [this](https://github.com/BoxcarsAI/boxcars/blob/main/notebooks/vector_store_examples.ipynb) Jupyter Notebook.
 
 Note, some folks that we talked to didn't know that you could run Ruby Jupyter notebooks. [You can](https://github.com/SciRuby/iruby).
 

data/boxcars.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.description = "You simply set an OpenAI key, give a number of Boxcars to a Train, and magic ensues when you run it."
   spec.homepage = "https://github.com/BoxcarsAI/boxcars"
   spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.6.0"
+  spec.required_ruby_version = ">= 3.0"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
@@ -30,15 +30,12 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # dev / test dependencies
-  spec.add_development_dependency "debug", "~> 1.1"
-  spec.add_development_dependency "dotenv", "~> 2.8"
-  spec.add_development_dependency "rspec", "~> 3.2"
-
   # runtime dependencies
   spec.add_dependency "google_search_results", "~> 2.2"
   spec.add_dependency "gpt4all", "~> 0.0.4"
-  spec.add_dependency "ruby-openai", "~> 3.0"
+  spec.add_dependency "hnswlib", "~> 0.8"
+  spec.add_dependency "ruby-openai", "~> 4.1"
+  spec.add_dependency "pgvector", "~> 0.2"
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/boxcars/boxcar/active_record.rb

@@ -29,7 +29,7 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { model_info: model_info }.merge super
     end
 
@@ -161,7 +161,7 @@ module Boxcars
       begin
         return true unless changes&.positive?
       rescue StandardError => e
-        Boscar.error "Error while computing change count: #{e.message}", :red
+        Boxcars.error "Error while computing change count: #{e.message}", :red
       end
 
       Boxcars.debug "#{name}(Pending Changes): #{changes}", :yellow

data/lib/boxcars/boxcar/engine_boxcar.rb

@@ -114,14 +114,14 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { stop: stop, top_k: top_k }
     end
 
     # @param inputs [Hash] The inputs to the boxcar.
     # @return Hash The variables for this boxcar.
     def prediction_variables(inputs)
-      prediction_additional.merge(inputs)
+      prediction_additional(inputs).merge(inputs)
     end
 
     # remove backticks or triple backticks from the code

data/lib/boxcars/boxcar/sql.rb

@@ -26,7 +26,7 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { schema: schema, dialect: dialect }.merge super
     end
 

data/lib/boxcars/boxcar/swagger.rb

@@ -25,7 +25,7 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { swagger_url: swagger_url, context: context }.merge super
     end
 

data/lib/boxcars/boxcar/vector_answer.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  # A Boxcar that interprets a prompt and executes ruby code to do math
+  class VectorAnswer < EngineBoxcar
+    # the description of this engine boxcar
+    DESC = "useful for when you need to answer questions from vector search results."
+
+    attr_reader :embeddings, :vector_documents, :search_content
+
+    # @param embeddings [Hash] The vector embeddings to use for this boxcar.
+    # @param vector_documents [Hash] The vector documents to use for this boxcar.
+    # @param engine [Boxcars::Engine] The engine to user for this boxcar. Can be inherited from a train if nil.
+    # @param prompt [Boxcars::Prompt] The prompt to use for this boxcar. Defaults to built-in prompt.
+    # @param kwargs [Hash] Any other keyword arguments to pass to the parent class.
+    def initialize(embeddings:, vector_documents:, engine: nil, prompt: nil, **kwargs)
+      the_prompt = prompt || my_prompt
+      @embeddings = embeddings
+      @vector_documents = vector_documents
+      kwargs[:stop] ||= ["```output"]
+      kwargs[:name] ||= "VectorAnswer"
+      kwargs[:description] ||= DESC
+      super(engine: engine, prompt: the_prompt, **kwargs)
+    end
+
+    # @param inputs [Hash] The inputs to use for the prediction.
+    # @return Hash The additional variables for this boxcar.
+    def prediction_additional(inputs)
+      { search_content: get_search_content(inputs[:question]) }.merge super
+    end
+
+    private
+
+    # @param results [Array] The results from the vector search.
+    # @return [String] The content of the search results.
+    def get_results_content(results)
+      results&.map do |result|
+        result[:document].content
+      end.to_a.join("\n\n")
+    end
+
+    # return the content of the search results for count results
+    # @params question [String] The question to search for.
+    # @params count [Integer] The number of results to return.
+    # @return [String] The content of the search results.
+    def get_search_content(question, count: 1)
+      search = Boxcars::VectorSearch.new(embeddings: embeddings, vector_documents: vector_documents)
+      results = search.call query: question, count: count
+      @search_content = get_search_content(results)
+    end
+
+    # our template
+    CTEMPLATE = [
+      syst("You are tasked with answering a question using these possibly relevant excerpts from a large volume of text:\n" \
+           "```text\n%<search_content>s\n```\n\n",
+           "Using the above, just answer the question as if you were answering directly."),
+      user("%<question>s")
+    ].freeze
+
+    # The prompt to use for the engine.
+    def my_prompt
+      @conversation ||= Conversation.new(lines: CTEMPLATE)
+      @my_prompt ||= ConversationPrompt.new(
+        conversation: @conversation,
+        input_variables: [:question],
+        other_inputs: [:search_content],
+        output_variables: [:answer])
+    end
+  end
+end

data/lib/boxcars/boxcar.rb

@@ -156,4 +156,6 @@ require "boxcars/boxcar/wikipedia_search"
 require "boxcars/boxcar/sql"
 require "boxcars/boxcar/swagger"
 require "boxcars/boxcar/active_record"
+require "boxcars/vector_store"
 require "boxcars/vector_search"
+require "boxcars/boxcar/vector_answer"

data/lib/boxcars/engine/openai.rb

@@ -43,6 +43,10 @@ module Boxcars
       ::OpenAI::Client.new(access_token: access_token, organization_id: organization_id)
     end
 
+    def conversation_model?(model)
+      ["gpt-3.5-turbo", "gpt-4"].include?(model)
+    end
+
     # Get an answer from the engine.
     # @param prompt [String] The prompt to use when asking the engine.
     # @param openai_access_token [String] The access token to use when asking the engine.
@@ -51,7 +55,7 @@ module Boxcars
     def client(prompt:, inputs: {}, openai_access_token: nil, **kwargs)
       clnt = Openai.open_ai_client(openai_access_token: openai_access_token)
       params = open_ai_params.merge(kwargs)
-      if params[:model] == "gpt-3.5-turbo"
+      if conversation_model?(params[:model])
         prompt = prompt.first if prompt.is_a?(Array)
         params = prompt.as_messages(inputs).merge(params)
         if Boxcars.configuration.log_prompts
@@ -71,6 +75,9 @@ module Boxcars
     def run(question, **kwargs)
       prompt = Prompt.new(template: question)
       response = client(prompt: prompt, **kwargs)
+      raise Error, "OpenAI: No response from API" unless response
+      raise Error, "OpenAI: #{response['error']}" if response["error"]
+
       answer = response["choices"].map { |c| c.dig("message", "content") || c["text"] }.join("\n").strip
       puts answer
       answer

data/lib/boxcars/train/zero_shot.rb

@@ -19,7 +19,7 @@ module Boxcars
     end
 
     # @return Hash The additional variables for this boxcar.
-    def prediction_additional
+    def prediction_additional(_inputs)
       { boxcar_names: boxcar_names, boxcar_descriptions: boxcar_descriptions }.merge super
     end
 

data/lib/boxcars/train.rb

@@ -69,7 +69,7 @@ module Boxcars
     # @return [Boxcars::Action] Action specifying what boxcar to use.
     def plan(intermediate_steps, **kwargs)
       thoughts = construct_scratchpad(intermediate_steps)
-      full_inputs = prediction_additional.merge(kwargs).merge(agent_scratchpad: thoughts)
+      full_inputs = prediction_additional(kwargs).merge(kwargs).merge(agent_scratchpad: thoughts)
       action = get_next_action(full_inputs)
       return TrainFinish.new({ output: action.boxcar_input }, log: action.log) if action.boxcar == finish_boxcar_name
 

data/lib/boxcars/vector_search.rb

@@ -3,8 +3,72 @@
 # Boxcars is a framework for running a series of tools to get an answer to a question.
 module Boxcars
   # For Boxcars that use an engine to do their work.
-  class VectorSearch < Boxcar
-    Error = Class.new(StandardError)
+  class VectorSearch
+    def initialize(params)
+      @vector_documents = params[:vector_documents]
+      @embedding_tool = params[:embedding_tool] || :openai
+      @vector_search_instance = vector_search_instance
+      @openai_connection = params[:openai_connection] || default_connection(openai_access_token: params[:openai_access_token])
+    end
+
+    def call(query:, count: 1)
+      validate_query(query)
+      query_vector = convert_query_to_vector(query)
+      @vector_search_instance.call(query_vector: query_vector, count: count)
+    end
+
+    private
+
+    attr_reader :vector_documents, :embedding_tool, :openai_connection
+
+    def vector_search_instance
+      case vector_documents[:type]
+      when :hnswlib
+        Boxcars::VectorStore::Hnswlib::Search.new(
+          vector_documents: vector_documents
+        )
+      when :in_memory
+        Boxcars::VectorStore::InMemory::Search.new(
+          vector_documents: vector_documents
+        )
+      when :pgvector
+        Boxcars::VectorStore::Pgvector::Search.new(
+          vector_documents: vector_documents
+        )
+      else
+        raise_argument_error('Unsupported vector store provided')
+      end
+    end
+
+    def default_connection(openai_access_token: nil)
+      Openai.open_ai_client(openai_access_token: openai_access_token)
+    end
+
+    def validate_query(query)
+      raise_argument_error('query must be a string') unless query.is_a?(String)
+      raise_argument_error('query must not be empty') if query.empty?
+    end
+
+    def convert_query_to_vector(query)
+      tool = embeddings_method(embedding_tool)
+      res = tool[:klass].call(
+        texts: [query], client: tool[:client]
+      ).first
+      res[:embedding]
+    end
+
+    def embeddings_method(embedding_tool)
+      case embedding_tool
+      when :openai
+        { klass: Boxcars::VectorStore::EmbedViaOpenAI, client: openai_connection }
+      when :tensorflow
+        { klass: Boxcars::VectorStore::EmbedViaTensorflow, client: nil }
+      end
+    end
+
+    def raise_argument_error(message)
+      raise ::Boxcars::ArgumentError, message
+    end
   end
 end
 

data/lib/boxcars/vector_store/document.rb

@@ -3,10 +3,11 @@
 module Boxcars
   module VectorStore
     class Document
-      attr_accessor :page_content, :metadata
+      attr_accessor :content, :metadata, :embedding
 
       def initialize(fields = {})
-        @page_content = fields[:page_content] || ""
+        @content = fields[:content] || ""
+        @embedding = fields[:embedding] || []
         @metadata = fields[:metadata] || {}
       end
     end

data/lib/boxcars/vector_store/embed_via_open_ai.rb

@@ -7,8 +7,6 @@ module Boxcars
     class EmbedViaOpenAI
       include VectorStore
 
-      attr_accessor :texts, :client, :model
-
       def initialize(texts:, client:, model: 'text-embedding-ada-002')
         validate_params(texts, client)
         @texts = texts
@@ -28,6 +26,8 @@ module Boxcars
 
       private
 
+      attr_accessor :texts, :client, :model
+
       def validate_params(texts, client)
         raise_error 'texts must be an array of strings' unless texts.is_a?(Array) && texts.all? { |text| text.is_a?(String) }
         raise_error 'openai_connection must be an OpenAI::Client' unless client.is_a?(OpenAI::Client)