pinecone 0.1.71 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea82946a00b57e8d81af48478eee15aa1f0a17a22316d982878f1033aa57dc7f
-  data.tar.gz: 6db21ac3336d691bce0064f2c22a8e1f1f5ced9a725e5b76eec14a0752c5d488
+  metadata.gz: c9e5f89bdb821f056dde85c82168420d575599d116508b083fb987ea7b3b60da
+  data.tar.gz: b4d1da67d606505ca251db63cc63a9077d363e92425ddfeb4ea55263af30aac6
 SHA512:
-  metadata.gz: 6265199be8db129c2b428cebef001249cc6827819643e19421eea23727484c69bd7054eb0eb0e3a3fc9e332eecfadaed900a618664bdb8fc61f320b9e4833c6b
-  data.tar.gz: '0913f7fe938dddfc78a150847f73da703b375250d55150c1718ee408b0f8b2012321db7c10a4de2fbb5e6fdf8f15ec8a3e1c27067903b807a0574de85f2455da'
+  metadata.gz: c429ec85b521a797a73597a6cba93b77e4892e84176064079367605c8436b15709aa03fd7ab563240c96bc8406c6b805c3d43bb7cc7f91c8c7ad1b016724acfe
+  data.tar.gz: 749c157e0650086c2365023cf8a9626ef04fd4bce9b5eac886de5007480c64318aa0c3312ea1795713ca08475c5e2f5396130aaabe2bb84d5d853d52969c12d1

data/lib/pinecone/client.rb

@@ -39,8 +39,9 @@ module Pinecone
     end
 
     # This is a very confusing naming convention
+    # Pinecone's reference now delineates between 'control plane' and 'data plane' which we'll reflect eventually
     def index(index_name)
-      @index ||= Pinecone::Vector.new(index_name)
+      Pinecone::Vector.new(index_name)
     end
   end
-end
+end

data/lib/pinecone/collection.rb

@@ -1,19 +1,20 @@
 module Pinecone
   class Collection
     include HTTParty
+    parser Pinecone::ResponseParser
 
     def initialize
-      self.class.base_uri "https://controller.#{Pinecone.configuration.environment}.pinecone.io"
+      self.class.base_uri "https://api.pinecone.io"
 
       @headers = {
         "Content-Type" => "application/json",
         "Accept" => "application/json",
-        "Api-Key" => Pinecone.configuration.api_key,
+        "Api-Key" => Pinecone.configuration.api_key
       }
     end
 
     def list
-      self.class.get('/collections', options)
+      self.class.get("/collections", options)
     end
 
     def describe(collection_name)
@@ -22,7 +23,7 @@ module Pinecone
 
     def create(body)
       payload = options.merge(body: body.to_json)
-      self.class.post('/collections', payload)
+      self.class.post("/collections", payload)
     end
 
     def delete(collection_name)
@@ -31,8 +32,8 @@ module Pinecone
 
     def options
       {
-        headers: @headers,
+        headers: @headers
       }
     end
   end
-end
+end

data/lib/pinecone/index.rb

@@ -1,42 +1,43 @@
 module Pinecone
   class Index
     include HTTParty
+    parser Pinecone::ResponseParser
 
     def initialize
-      self.class.base_uri "https://controller.#{Pinecone.configuration.environment}.pinecone.io"
+      self.class.base_uri "https://api.pinecone.io"
       @headers = {
         "Content-Type" => "application/json",
         "Accept" => "application/json",
-        "Api-Key" => Pinecone.configuration.api_key,
+        "Api-Key" => Pinecone.configuration.api_key
       }
     end
 
     def list
-      self.class.get('/databases', options)
+      self.class.get("/indexes", options)
     end
 
     def describe(index_name)
-      self.class.get("/databases/#{index_name}", options)
+      self.class.get("/indexes/#{index_name}", options)
     end
-    
+
     def create(body)
       payload = options.merge(body: body.to_json)
-      self.class.post('/databases', payload)
+      self.class.post("/indexes", payload)
     end
 
     def delete(index_name)
-      self.class.delete("/databases/#{index_name}", options)
+      self.class.delete("/indexes/#{index_name}", options)
     end
 
     def configure(index_name, body)
       payload = options.merge(body: body.to_json)
-      self.class.patch("/databases/#{index_name}", payload)
+      self.class.patch("/indexes/#{index_name}", payload)
     end
 
     def options
       {
-        headers: @headers,
+        headers: @headers
       }
     end
   end
-end
+end

data/lib/pinecone/response_parser.rb

@@ -0,0 +1,16 @@
+module Pinecone
+  class ResponseParser < HTTParty::Parser
+    # standard:disable Naming/ConstantName
+    SupportedFormats = {
+      "application/json" => :json,
+      "text/plain" => :json
+    }
+    # standard:enable Naming/ConstantName
+
+    def json
+      JSON.parse(body)
+    rescue
+      body
+    end
+  end
+end

data/lib/pinecone/vector/filter.rb

@@ -6,9 +6,9 @@ require "dry-validation"
 module Types
   include Dry.Types()
 
-  StringOrNumberOrBoolean = Dry::Types['string'] | Dry::Types['integer'] | Dry::Types['float'] | Dry::Types['bool']
-  StringOrNumber = Dry::Types['string'] | Dry::Types['integer'] | Dry::Types['float']
-  Number = Dry::Types['integer'] | Dry::Types['float']
+  StringOrNumberOrBoolean = Dry::Types["string"] | Dry::Types["integer"] | Dry::Types["float"] | Dry::Types["bool"]
+  StringOrNumber = Dry::Types["string"] | Dry::Types["integer"] | Dry::Types["float"]
+  Number = Dry::Types["integer"] | Dry::Types["float"]
 end
 
 module Pinecone
@@ -50,14 +50,14 @@ module Pinecone
 
         def to_filter(input)
           return false unless input.is_a?(Hash)
-          return Filter.new(input)
+          Filter.new(input)
         end
       end
 
       def self.new(input)
         validation = FilterContract.new.call(input)
         if validation.success?
-          super(input)
+          super
         else
           raise ArgumentError.new(validation.errors.to_h.inspect)
         end
@@ -68,4 +68,4 @@ module Pinecone
       end
     end
   end
-end
+end

data/lib/pinecone/vector/query.rb

@@ -29,22 +29,22 @@ module Pinecone
 
       schema schema.strict
 
-      attribute :namespace, Dry::Types['string'].default("")
-      attribute :include_values, Dry::Types['bool'].default(false)
-      attribute :include_metadata, Dry::Types['bool'].default(true)
-      attribute :top_k, Dry::Types['integer'].default(10)
-      attribute? :vector, Dry::Types['array'].of(Dry::Types['float'] | Dry::Types['integer'])
+      attribute :namespace, Dry::Types["string"].default("")
+      attribute :include_values, Dry::Types["bool"].default(false)
+      attribute :include_metadata, Dry::Types["bool"].default(true)
+      attribute :top_k, Dry::Types["integer"].default(10)
+      attribute? :vector, Dry::Types["array"].of(Dry::Types["float"] | Dry::Types["integer"])
       # Disabled contract since it wasn't carrying forward attributes to to_json
       # See failing test in query_spec.rb
-      # attribute? :filter, Filter 
-      attribute? :filter, Dry::Types['hash']
+      # attribute? :filter, Filter
+      attribute? :filter, Dry::Types["hash"]
       attribute? :sparse_vector, SparseVector
-      attribute? :id, Dry::Types['string']
+      attribute? :id, Dry::Types["string"]
 
       def self.new(input)
         validation = QueryContract.new.call(input)
         if validation.success?
-          super(input)
+          super
         else
           raise ArgumentError.new(validation.errors.to_h.inspect)
         end
@@ -53,10 +53,10 @@ module Pinecone
       def to_json
         to_h.map do |key, value|
           [key.to_s.split("_").map.with_index do |word, index|
-            index == 0 ? word : word.capitalize
+            (index == 0) ? word : word.capitalize
           end.join.to_sym, value]
         end.to_h.to_json
       end
     end
   end
-end
+end

data/lib/pinecone/vector/sparse_vector.rb

@@ -24,17 +24,17 @@ module Pinecone
         end
       end
 
-      attribute :indices, Dry::Types['array'].of(Dry::Types['integer'])
-      attribute :values, Dry::Types['array'].of(Dry::Types['float'] | Dry::Types['integer'])
+      attribute :indices, Dry::Types["array"].of(Dry::Types["integer"])
+      attribute :values, Dry::Types["array"].of(Dry::Types["float"] | Dry::Types["integer"])
 
       def self.new(input)
         validation = SparseVectorContract.new.call(input)
         if validation.success?
-          super(input)
+          super
         else
           raise ArgumentError.new(validation.errors.to_h.inspect)
         end
       end
     end
   end
-end
+end

data/lib/pinecone/vector.rb

@@ -5,46 +5,103 @@ require "pinecone/vector/sparse_vector"
 module Pinecone
   class Vector
     include HTTParty
+    parser Pinecone::ResponseParser
+
+    attr_reader :base_uri
 
     def initialize(index_name)
-      self.class.base_uri set_base_uri(index_name)
+      @base_uri = set_base_uri(index_name)
 
       @headers = {
         "Content-Type" => "application/json",
         "Accept" => "application/json",
-        "Api-Key" => Pinecone.configuration.api_key,
+        "Api-Key" => Pinecone.configuration.api_key
       }
     end
 
-    def delete(namespace: "", ids: [], delete_all: false)
+    def delete(namespace: "", ids: [], delete_all: false, filter: {})
       inputs = {
-        "namespace": namespace,
-        "ids": ids,
-        "deleteAll": delete_all,
-      }.to_json
-      payload = options.merge(body: inputs)
-      self.class.post('/vectors/delete', payload)
+        namespace: namespace,
+        ids: ids,
+        deleteAll: delete_all,
+        filter: filter
+      }
+      inputs.delete(:filter) if delete_all || ids.any?
+      payload = options.merge(body: inputs.to_json)
+      self.class.post("#{@base_uri}/vectors/delete", payload)
     end
 
     def fetch(namespace: "", ids: [])
-      query_string = URI.encode_www_form({ namespace: namespace, ids: ids})
-      self.class.get("/vectors/fetch?#{query_string}", options)
+      query_string = URI.encode_www_form({namespace: namespace, ids: ids})
+      self.class.get("#{@base_uri}/vectors/fetch?#{query_string}", options)
+    end
+
+    def list(prefix: nil, limit: nil, namespace: nil, &block)
+      all_ids = []
+      pagination_token = nil
+      total_fetched = 0
+
+      loop do
+        current_limit = limit.nil? ? 100 : [limit - total_fetched, 100].min
+
+        response = list_paginated(
+          prefix: prefix,
+          limit: current_limit,
+          pagination_token: pagination_token,
+          namespace: namespace
+        )
+
+        break unless response.success?
+
+        results = response.parsed_response
+        ids = results["vectors"]&.map { |v| v["id"] } || []
+
+        if block
+          yield ids
+        else
+          all_ids.concat(ids)
+        end
+
+        total_fetched += ids.length
+        pagination_token = results.dig("pagination", "next")
+
+        break if ids.empty?
+        break if limit && total_fetched >= limit
+        break if pagination_token.nil? || pagination_token.empty?
+      end
+
+      if block
+        nil
+      else
+        limit ? all_ids.first(limit) : all_ids
+      end
+    end
+
+    def list_paginated(prefix: nil, limit: nil, pagination_token: nil, namespace: nil)
+      query_params = {}
+      query_params["prefix"] = prefix if prefix
+      query_params["limit"] = limit if limit
+      query_params["paginationToken"] = pagination_token if pagination_token
+      query_params["namespace"] = namespace if namespace
+
+      query_string = URI.encode_www_form(query_params)
+      self.class.get("#{@base_uri}/vectors/list?#{query_string}", options)
     end
 
     def upsert(body)
       payload = options.merge(body: body.to_json)
-      self.class.post('/vectors/upsert', payload)
+      self.class.post("#{@base_uri}/vectors/upsert", payload)
     end
 
     def query(query)
       object = query.is_a?(Pinecone::Vector::Query) ? query : Pinecone::Vector::Query.new(query)
       payload = options.merge(body: object.to_json)
-      self.class.post('/query', payload)
+      self.class.post("#{@base_uri}/query", payload)
     end
 
     def update(id:, values: [], sparse_values: {indices: [], values: []}, set_metadata: {}, namespace: "")
       inputs = {
-        "id": id
+        id: id
       }
       inputs["values"] = values unless values.empty?
       inputs["sparseValues"] = sparse_values unless sparse_values[:indices].empty? || sparse_values[:values].empty?
@@ -52,7 +109,7 @@ module Pinecone
       inputs["namespace"] = namespace unless namespace.empty?
 
       payload = options.merge(body: inputs.to_json)
-      self.class.post('/vectors/update', payload)
+      self.class.post("#{@base_uri}/vectors/update", payload)
     end
 
     def describe_index_stats(filter: {})
@@ -61,13 +118,12 @@ module Pinecone
       else
         options.merge(body: {filter: filter}.to_json)
       end
-      self.class.post('/describe_index_stats', payload)
+      self.class.post("#{@base_uri}/describe_index_stats", payload)
     end
 
-
     def options
       {
-        headers: @headers,
+        headers: @headers
       }
     end
 
@@ -77,8 +133,8 @@ module Pinecone
     def set_base_uri(index_name)
       index_description = Pinecone::Index.new.describe(index_name)
       raise Pinecone::IndexNotFoundError, "Index #{index_name} does not exist" if index_description.code != 200
-      uri = index_description.parsed_response["status"]["host"]
+      uri = index_description.parsed_response["host"]
       "https://#{uri}"
     end
   end
-end
+end

data/lib/pinecone/version.rb

@@ -1,3 +1,3 @@
 module Pinecone
-  VERSION = "0.1.71".freeze
-end
+  VERSION = "1.1.0".freeze
+end

data/lib/pinecone.rb

@@ -1,5 +1,6 @@
 require "httparty"
 
+require "pinecone/response_parser"
 require "pinecone/client"
 require "pinecone/index"
 require "pinecone/vector"
@@ -8,14 +9,16 @@ require "pinecone/version"
 
 module Pinecone
   class ConfigurationError < StandardError; end
+
   class IndexNotFoundError < StandardError; end
+
   class Configuration
     attr_writer :api_key, :base_uri, :environment
 
     def initialize
-      @api_key     = nil
+      @api_key = nil
       @environment = nil
-      @base_uri    = nil
+      @base_uri = nil
     end
 
     def api_key
@@ -48,4 +51,4 @@ module Pinecone
   def self.configure
     yield(configuration)
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pinecone
 version: !ruby/object:Gem::Version
-  version: 0.1.71
+  version: 1.1.0
 platform: ruby
 authors:
 - Scott Carleton
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-28 00:00:00.000000000 Z
+date: 2024-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.21.0
+        version: 0.22.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.21.0
+        version: 0.22.0
 - !ruby/object:Gem::Dependency
   name: dry-struct
   requirement: !ruby/object:Gem::Requirement
@@ -52,104 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.10.0
-- !ruby/object:Gem::Dependency
-  name: dotenv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-- !ruby/object:Gem::Dependency
-  name: awesome_print
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.9'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.9'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: debug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.12'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.12'
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.14'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.14'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.1'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.1'
 description: Ruby client library which includes index and vector operations to upload
   embeddings into Pinecone and do similarity searches on them.
 email: scott@extrayarn.com
@@ -161,6 +63,7 @@ files:
 - lib/pinecone/client.rb
 - lib/pinecone/collection.rb
 - lib/pinecone/index.rb
+- lib/pinecone/response_parser.rb
 - lib/pinecone/vector.rb
 - lib/pinecone/vector/filter.rb
 - lib/pinecone/vector/query.rb
@@ -179,14 +82,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.3
+rubygems_version: 3.5.15
 signing_key:
 specification_version: 4
 summary: Ruby client library for Pinecone Vector DB