roseflow-pinecone 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '026090287ba3476e5380c6c608deb67ddca3900e29ec73039095fcc8958622db'
+  data.tar.gz: 7e8b2e32985dca38c6ed694ab64f64996b640f437de9beec0c6df9dd37842f16
+SHA512:
+  metadata.gz: ea6af6e5198bb2dc34bef0cfa538eb188efb0873d2d3945502ba5d2773bce04743462ff9cbb6c072de0ae5d12b45d0aa5d154a5fab835a37717a8a4262a478c3
+  data.tar.gz: 821b6c710661e44eca7b2e3e5f0330afd006a55b38810bc71e9e02ba623b1ee8703582eb51aee8fca717c512de9959d6c43937a8199d1aea28838bfc5bfa178e

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.standard.yml

@@ -0,0 +1,3 @@
+# For available configuration options, see:
+#   https://github.com/testdouble/standard
+ruby_version: 2.6

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-05-10
+
+- Initial release

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,7 @@
+# Contributor Code of Conduct
+
+The Roseflow team is committed to fostering a welcoming community.
+
+**Our Code of Conduct can be found here**:
+
+https://roseflow.ai/conduct

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in roseflow-pinecone.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.5"
+gem "standard", "~> 1.3"
+
+eval File.read("Gemfile.local") if File.exist?("Gemfile.local")

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Lauri Jutila
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# Roseflow Pinecone integration
+
+This gem adds Pinecone vector database support and integration for Roseflow, a framework for interacting with AI in Ruby.
+
+## Prerequisites
+
+To use this gem effectively, you need the [core Roseflow gem](https://github.com/roseflow-ai/roseflow).
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add roseflow-pinecone
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install roseflow-pinecone
+
+## Usage
+
+See full documentation how to configure and use Roseflow with Pinecone at [docs.roseflow.ai](https://docs.roseflow.ai/pinecone).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/roseflow-ai/roseflow-pinecone.
+
+## Community
+
+### Discord
+
+Join us in our [Discord](https://discord.gg/roseflow).
+
+### Twitter
+
+Connect with the core team on Twitter.
+
+<a href="https://twitter.com/ljuti" target="_blank">
+  <img alt="Twitter Follow" src="https://img.shields.io/twitter/follow/ljuti?logo=twitter&style=social">
+</a>
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Roseflow OpenAI project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/roseflow-ai/roseflow-pinecone/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "standard/rake"
+
+task default: %i[spec standard]

data/config/pinecone.yml

@@ -0,0 +1,3 @@
+default: &default
+  environment: <YOUR PINECONE ENVIRONMENT>
+  api_key: <YOUR PINECONE API KEY>

data/lib/roseflow/pinecone/client.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "faraday/retry"
+
+require "roseflow/pinecone/index"
+require "roseflow/pinecone/vector"
+
+module Roseflow
+  module Pinecone
+    class Client
+      attr_accessor :index_url
+      
+      def initialize(config = Config.new)
+        @environment = config.environment
+        @api_key = config.api_key
+        @index_url = nil
+      end
+
+      def indices
+        Index.new(connection).list.body
+      end
+      
+      alias_method :indexes, :indices
+
+      def describe_index(name)
+        Index.new(connection).describe(name)
+      end
+
+      def create_index(name, options = {})
+        Index.new(connection).create(name, options)
+      end
+
+      def delete_index(name)
+        Index.new(connection).delete(name)
+      end
+
+      def collections
+        Collection.new(connection).list
+      end
+
+      def describe_collection(name)
+        Collection.new(connection).describe(name)
+      end
+
+      def create_collection(name, source)
+        Collection.new(connection).create(name, source)
+      end
+
+      def delete_collection(name)
+        Collection.new(connection).delete(name)
+      end
+
+      def index(name)
+        @index ||= get_vector_index(name)
+      end
+      
+      private
+
+      def get_vector_index(name)
+        return Vector.new(index_connection(index_url)) if index_url && index_url.match(/#{name}/)
+        @index_url = describe_index(name).object.status.host
+        return Vector.new(index_connection(@index_url))
+      end
+
+      def environment_url(env)
+        "https://controller.#{env}.pinecone.io"
+      end
+
+      def index_url_for(url)
+        "https://#{url}"
+      end
+
+      def connection
+        @connection ||= Faraday.new(
+          url: environment_url(@environment),
+          headers: {
+            "accept": "application/json; charset=utf-8",
+            "Api-Key": @api_key,
+          }
+        ) do |faraday|
+          faraday.request :json
+          faraday.adapter Faraday.default_adapter
+        end
+      end
+
+      def index_connection(index_url = nil)
+        raise IndexURLNotSetError, "You must provide index URL to access an index" unless index_url
+        @index_connection ||= Faraday.new(
+          url: index_url_for(index_url),
+          headers: {
+            "accept": "application/json; charset=utf-8",
+            "Api-Key": @api_key,
+          }
+        ) do |faraday|
+          faraday.request :json
+          faraday.adapter Faraday.default_adapter
+        end
+      end
+    end
+  end
+end

data/lib/roseflow/pinecone/collection.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "roseflow/pinecone/response"
+require "roseflow/pinecone/structs"
+
+module Roseflow
+  module Pinecone
+    class Collection
+      def initialize(connection)
+        @connection = connection
+      end
+
+      def list
+        CollectionResponse.new(
+          method: :list,
+          response: connection.get("/collections")
+        )
+      end
+
+      def describe(name)
+        CollectionResponse.new(
+          method: :describe,
+          response: connection.get("/collections/#{name}")
+        )
+      end
+
+      def create(name = "", source_index = "")
+        CollectionResponse.new(
+          method: :create,
+          response: connection.post("/collections", { name: name, source: source_index })
+        )
+      end
+
+      def delete(name)
+        CollectionResponse.new(
+          method: :delete,
+          response: connection.delete("/collections/#{name}")
+        )
+      end
+
+      private
+
+      attr_reader :connection
+    end
+  end
+end

data/lib/roseflow/pinecone/config.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "anyway_config"
+
+module Roseflow
+  module Pinecone
+    class Config < Anyway::Config
+      config_name :pinecone
+
+      attr_config :api_key, :environment
+
+      required :api_key
+      required :environment
+    end
+  end
+end

data/lib/roseflow/pinecone/index.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "roseflow/pinecone/response"
+require "roseflow/pinecone/structs"
+
+module Roseflow
+  module Pinecone
+    class Index
+      def initialize(connection)
+        @connection = connection
+      end
+
+      def list
+        IndexResponse.new(
+          method: :list,
+          response: connection.get("/databases")
+        )
+      end
+
+      def describe(name)
+        IndexResponse.new(
+          method: :describe,
+          response: connection.get("/databases/#{name}")
+        )
+      end
+
+      def create(name, options = {})
+        IndexResponse.new(
+          method: :create,
+          response: connection.post("/databases", create_payload(name, options))
+        )
+      end
+
+      def delete(name)
+        IndexResponse.new(
+          method: :delete,
+          response: connection.delete("/databases/#{name}")
+        )
+      end
+
+      private
+
+      attr_reader :connection
+
+      def create_payload(name, options)
+        options.merge({
+          name: name
+        })
+      end
+    end
+  end
+end

data/lib/roseflow/pinecone/object.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "dry-struct"
+
+module Roseflow
+  module Pinecone
+    class PineconeObject < Dry::Struct
+      defines :contract_object
+    end
+  end
+end

data/lib/roseflow/pinecone/response.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Roseflow
+  module Pinecone
+    class BaseResponse
+      attr_reader :body
+      attr_reader :status
+      attr_reader :object
+
+      def initialize(method:, response:)
+        @response = response
+        @method_ = method
+        handle_response
+      end
+
+      def success?
+        @response.success?
+      end
+
+      private
+
+      attr_reader :response
+    end
+
+    class CollectionResponse < BaseResponse
+      private
+
+      def handle_response
+        case @method_
+        when :list
+          @body = JSON.parse(response.body)
+        when :describe
+          if response.body.match(/{*.}/)
+            @body = JSON.parse(response.body)
+            @object = Collection::Description.new(@body)
+          end
+        when :create
+          if response.body.match(/{*.}/)
+            @body = JSON.parse(response.body)
+          else
+            @body = response.body
+          end
+        end
+        @status = response.status
+      end
+    end
+
+    class VectorResponse < BaseResponse
+      attr_reader :upsert_count, :vectors
+
+      def upsert_count
+        @body.fetch("upsertedCount")
+      end
+
+      def message
+        @body.fetch("message")
+      end
+
+      def success?
+        @response.success? && @status == 200
+      end
+
+      def failure?
+        !success?
+      end
+
+      def vectors
+        @vectors.map do |vector|
+          Vector::VectorObject.new(vector.last)
+        end
+      end
+      
+      private
+
+      def handle_response
+        case @method_
+        when :upsert
+          @body = JSON.parse(@response.body)
+        when :delete
+          @body = JSON.parse(@response.body)
+        when :update
+          @body = JSON.parse(@response.body)
+        end
+        @status = response.status
+      end
+    end
+
+    class VectorQueryResponse < VectorResponse
+      def vectors
+        case @method_
+        when :fetch
+          @vectors.map do |vector|
+            Vector::VectorObject.new(vector.last)
+          end
+        when :query
+          @vectors.map do |vector|
+            Vector::VectorObject.new(vector)
+          end
+        end
+      end
+
+      private
+
+      def handle_response
+        case @method_
+        when :fetch
+          @body = JSON.parse(@response.body)
+          @vectors = @body["vectors"]
+        when :query
+          @body = JSON.parse(@response.body)
+          @vectors = @body["matches"]
+        end
+        @status = @response.status
+      end
+    end
+
+    class IndexResponse < BaseResponse      
+      private
+
+      def handle_response
+        case @method_
+        when :list
+          @body = JSON.parse(@response.body)
+        when :describe
+          if @response.body.match(/{*.}/)
+            @body = JSON.parse(@response.body)
+            attrs = @body["database"].merge({ status: @body["status"] })
+            @object = Index::Description.new(attrs)
+          end
+        when :create
+          if @response.body.match(/{*.}/)
+            @body = JSON.parse(@response.body)
+          else
+            @body = @response.body
+          end
+        end
+
+        @status = @response.status
+      end
+    end
+  end
+end

data/lib/roseflow/pinecone/structs.rb

@@ -0,0 +1,44 @@
+require "dry-struct"
+
+module Types
+  include Dry.Types()
+end
+
+module Roseflow
+  module Pinecone
+    class Index
+      class Status < Dry::Struct
+        transform_keys(&:to_sym)
+
+        attribute :host, Types::Strict::String
+        attribute :port, Types::Strict::Integer
+        attribute :state, Types::Strict::String
+        attribute :ready, Types::Strict::Bool
+      end
+
+      class Description < Dry::Struct
+        transform_keys(&:to_sym)
+
+        attribute :name, Types::Strict::String
+        attribute :metric, Types::Strict::String
+        attribute :dimension, Types::Strict::Integer
+        attribute :replicas, Types::Strict::Integer
+        attribute :shards, Types::Strict::Integer
+        attribute :pods, Types::Strict::Integer
+        attribute :pod_type, Types::Strict::String
+
+        attribute :status, Status
+      end
+    end
+
+    class Collection
+      class Description < Dry::Struct
+        transform_keys(&:to_sym)
+
+        attribute :name, Types::Strict::String
+        attribute :status, Types::Strict::String
+        attribute :size, Types::Strict::Integer.default(0)
+      end
+    end
+  end
+end

data/lib/roseflow/pinecone/vector.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "roseflow/pinecone/vectors/deletion"
+require "roseflow/pinecone/vectors/query"
+require "roseflow/pinecone/vectors/upsert"
+require "roseflow/pinecone/vectors/update"
+require "roseflow/pinecone/response"
+
+module Roseflow
+  module Pinecone
+    class Vector
+      attr_reader :connection
+
+      def initialize(connection)
+        @connection = connection
+      end
+
+      def self.build(id:, values:, sparse_values: nil, metadata: nil)
+        VectorObject.new(
+          id: id,
+          values: values,
+          sparse_values: sparse_values,
+          metadata: metadata,
+        )
+      end
+
+      def empty?(namespace = "")
+        query(namespace: namespace).vectors.empty?
+      end
+
+      def query(query)
+        object = query.is_a?(Query) ? query : Query.new(query)
+        VectorQueryResponse.new(
+          method: :query,
+          response: @connection.post("/query", object.to_json),
+        )
+      end
+
+      def delete(query)
+        object = query.is_a?(Deletion) ? query : Deletion.new(query)
+        VectorResponse.new(
+          method: :delete,
+          response: @connection.post("/vectors/delete", object.to_json),
+        )
+      end
+
+      def update(vectors)
+        object = vectors.is_a?(Update) ? vectors : Update.new(vectors)
+        VectorResponse.new(
+          method: :update,
+          response: @connection.post("/vectors/update", object.to_json),
+        )
+      end
+
+      def upsert(data)
+        object = data.is_a?(Upsert) ? data : Upsert.from(data)
+        VectorResponse.new(
+          method: :upsert,
+          response: @connection.post("/vectors/upsert", object.to_json),
+        )
+      end
+
+      def fetch(options)
+        query_string = URI.encode_www_form(
+          {
+            namespace: options.fetch(:namespace, ""),
+            ids: options.fetch(:ids, []),
+          }
+        )
+        VectorQueryResponse.new(
+          method: :fetch,
+          response: @connection.get("/vectors/fetch?#{query_string}"),
+        )
+      end
+    end
+  end
+end

data/lib/roseflow/pinecone/vector_store.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "roseflow/vector_stores/base"
+require "roseflow/pinecone"
+require "roseflow/pinecone/vector"
+require "roseflow/pinecone/vectors/vector_object"
+
+module Roseflow
+  module Pinecone
+    class VectorStore < Roseflow::VectorStores::Base
+      attr_reader :client
+
+      def initialize
+        @client = Roseflow::Pinecone::Client.new
+      end
+
+      def index(name)
+        client.index(name)
+      end
+
+      def build_vector(name, content)
+        Vector.build(id: name, values: content)
+      end
+
+      def create_vector(name, vector, **options)
+        index(name).upsert(options.merge(vectors: [vector]))
+      end
+
+      def delete_vector(name, id)
+        index(name).delete(ids: [id])
+      end
+
+      def update_vector(name, vector)
+        index(name).update(vector)
+      end
+
+      def query(name, query)
+        index(name).query(query).vectors
+      end
+
+      alias_method :where, :query
+
+      def find(name, ids, **options)
+        index(name).fetch(options.merge(ids: ids)).vectors.first
+      end
+    end
+  end
+end