activerecord-turbopuffer-adapter 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 735feec18b7fd1ae5ded62e4ed1be93fc11c72c7f26e4297cdad2959e364f5ae
-  data.tar.gz: 463e7962478cac322980777e1af86edd02ccf6109e4114afcbdf0738cb1ec635
+  metadata.gz: f7e4579b0235c712aef39dedae94abdb3e8053ef5b04fff2e46ee38c0fd5710a
+  data.tar.gz: dadc48af421f105ed95d7964b7929ced1dd613217fdda1b88b00a58c291867cf
 SHA512:
-  metadata.gz: 961a01ae0f07741adf56396ef8df18ddb94896627d9ccee153b3f191ff34fda091136133ecc52e3c8e09b3726e9a45d70cde91957f1d9241f2cf5c122caabd38
-  data.tar.gz: ed62e9af80fd247549aecfe079f9d45f05037411dc631e310ef1faae981ba2581682335be96bfee6b3877cad4b8ae2d59936637a9179525349c375e9596671df
+  metadata.gz: 6037d2898aa914ed3990f238ac00b86c4b710d630fd3aa7c9dd3af9200f768736b9207cf8cedf079c01d60fd8fce614cdafdaa5873241b1c7a47fa1f6bb4b9df
+  data.tar.gz: 8b6c7caf5e1b82fbed33da7e5e06e1232cf9603aab035b6a9ef8540223047020429927721928911c80512c778fbdef5a52b1628ca0110ac4414d0419f18a9aed

data/README.md

@@ -56,6 +56,17 @@ class Document < ApplicationRecord
 end
 ```
 
+Alternatively, turbopuffer can generate embeddings for you. Declare `embed` on a string attribute and the vector is computed on write:
+
+```ruby
+class Document < ApplicationRecord
+  turbopuffer_attribute "id",   "uuid",   not_null: 1
+  turbopuffer_attribute "body", "string", embed: "openai/text-embedding-3-small"
+end
+```
+
+turbopuffer stores the vector in a computed `embed_body` attribute. `embed` also accepts a hash (`model:`, `attribute:`, `dims:`, `dtype:`) to name the vector attribute or set its size. Embedding is billed per token by model, see https://turbopuffer.com/docs/embedding for the supported models.
+
 ### Creating records
 
 ```ruby
@@ -104,6 +115,7 @@ Document.find("018f...")
 
 Document.rank_by("vector", "ANN", query_vector).limit(10)
 Document.rank_by("text", "BM25", "quick walrus").limit(10)
+Document.rank_by("body", "ANN", ["Embed", "sea mammals"]).limit(10)
 
 Document.consistency(:eventual).where(published: true)
 
@@ -120,6 +132,20 @@ Document
 
 > Eventual reads are cheaper but may not include writes from the last few seconds. Set a default with `turbopuffer_consistency "eventual"` in a model or `consistency: eventual` in `database.yml`. Strong is the default.
 
+### Namespaces per tenant
+
+turbopuffer recommends one namespace per tenant: a query can only ever see that tenant's documents, and search covers just the documents that matter rather than everyone's. `namespace` scopes a model to any namespace, and records remember where they were loaded from, so `update!`, `reload` and `destroy` go back to the same place.
+
+```ruby
+Document.namespace("documents-#{tenant.id}").create!(title: "Hello")
+Document.namespace("documents-#{tenant.id}").where(published: true)
+
+doc = Document.namespace("documents-#{tenant.id}").find(id)
+doc.update!(published: true)
+```
+
+The name is used as given (under `namespace_prefix` if one is set). Without `namespace`, a model's table name is its namespace.
+
 ### Errors
 
 Turbopuffer errors are raised as the ActiveRecord exceptions a Rails app already handles.

data/lib/active_record/connection_adapters/turbopuffer_adapter.rb

@@ -151,7 +151,7 @@ module ActiveRecord
           schema: model.turbopuffer_schema_hash
         }
 
-        if model.turbopuffer_attributes.any?(&:ann)
+        if model.turbopuffer_attributes.any? { |attribute| attribute.ann || attribute.embed }
           tpuf_insert_args[:distance_metric] = model.turbopuffer_distance_metric
         end
 
@@ -299,8 +299,7 @@ module ActiveRecord
       def perform_query(raw_connection, sql, binds, type_casted_binds, prepare:, notification_payload:, batch:)
         tpuf_query = sql.is_a?(Array) ? sql.first : sql
 
-        namespace_name = [ @config[:namespace_prefix], tpuf_query.namespace ].compact_blank.join("-")
-        namespace = raw_connection.namespace(namespace_name)
+        namespace = raw_connection.namespace(namespace_name_for(tpuf_query))
 
         result = case tpuf_query.op
         when :insert then turbopuffer_insert(namespace, tpuf_query)
@@ -318,6 +317,12 @@ module ActiveRecord
         result
       end
 
+      def namespace_name_for(query)
+        override = query.namespace_override || ::Turbopuffer::ActiveRecord.namespace_override
+
+        [ @config[:namespace_prefix], override || query.namespace ].compact_blank.join("-")
+      end
+
       def arel_visitor
         Arel::Visitors::Turbopuffer.new
       end

data/lib/arel/visitors/turbopuffer.rb

@@ -11,7 +11,8 @@ module Arel::Visitors
         include_attributes: @include_attributes,
         aggregate_by: @aggregate_by,
         upsert_rows: @upsert_rows,
-        consistency: @consistency
+        consistency: @consistency,
+        namespace_override: @namespace_override
       }
     end
     def to_s = JSON.generate(to_h)
@@ -21,10 +22,10 @@ module Arel::Visitors
     alias == eql?
 
     attr_reader :op, :namespace, :filters, :top_k, :rank_by, :include_attributes, :upsert_rows, :aggregate_by, :group_by,
-                :consistency
+                :consistency, :namespace_override
 
     def initialize(op:, namespace:, filters: nil, top_k: nil, rank_by: nil, include_attributes: nil,
-                   upsert_rows: nil, aggregate_by: nil, group_by: nil, consistency: nil)
+                   upsert_rows: nil, aggregate_by: nil, group_by: nil, consistency: nil, namespace_override: nil)
       @op = op
       @namespace = namespace
       @filters = filters
@@ -35,6 +36,7 @@ module Arel::Visitors
       @group_by = group_by
       @upsert_rows = upsert_rows
       @consistency = consistency
+      @namespace_override = namespace_override
     end
   end
 
@@ -124,10 +126,19 @@ module Arel::Visitors
         include_attributes: attributes.flat_map { |p| visit(p) },
         group_by:           core.groups.map { |g| visit(g) },
         aggregate_by:       aggregate && visit(aggregate),
-        consistency:        consistency_hint(core.optimizer_hints)
+        consistency:        consistency_hint(core.optimizer_hints),
+        namespace_override: namespace_annotation(core.comment)
       )
     end
 
+    def namespace_annotation(comment)
+      return unless comment
+
+      prefix = ::Turbopuffer::ActiveRecord::NAMESPACE_ANNOTATION
+      annotation = comment.values.reverse.find { |value| value.start_with?(prefix) }
+      annotation&.delete_prefix(prefix)
+    end
+
     def consistency_hint(hints)
       return unless hints
 
@@ -152,18 +163,20 @@ module Arel::Visitors
       rows = o.values ? visit(o.values) : [ [] ]
 
       TurbopufferQuery.new(
-        op:          :update,
-        namespace:   visit(o.relation),
-        filters:     conjoin(o.wheres.map { |w| visit(w) }),
-        upsert_rows: rows
+        op:                 :update,
+        namespace:          visit(o.relation),
+        filters:            conjoin(o.wheres.map { |w| visit(w) }),
+        upsert_rows:        rows,
+        namespace_override: namespace_annotation(o.comment)
       )
     end
 
     def visit_Arel_Nodes_DeleteStatement(o)
       TurbopufferQuery.new(
-        op:        :delete,
-        namespace: visit(o.relation),
-        filters:   conjoin(o.wheres.map { |w| visit(w) })
+        op:                 :delete,
+        namespace:          visit(o.relation),
+        filters:            conjoin(o.wheres.map { |w| visit(w) }),
+        namespace_override: namespace_annotation(o.comment)
       )
     end
 

data/lib/turbopuffer/active_record/namespace_scope.rb

@@ -0,0 +1,15 @@
+module Turbopuffer
+  module ActiveRecord
+    module NamespaceScope
+      def turbopuffer_namespace
+        annotation = annotate_values.reverse.find { |value| value.start_with?(NAMESPACE_ANNOTATION) }
+        annotation&.delete_prefix(NAMESPACE_ANNOTATION)
+      end
+
+      def exec_queries(...) = ::Turbopuffer::ActiveRecord.with_namespace(turbopuffer_namespace) { super }
+      def insert_all(...)   = ::Turbopuffer::ActiveRecord.with_namespace(turbopuffer_namespace) { super }
+      def insert_all!(...)  = ::Turbopuffer::ActiveRecord.with_namespace(turbopuffer_namespace) { super }
+      def upsert_all(...)   = ::Turbopuffer::ActiveRecord.with_namespace(turbopuffer_namespace) { super }
+    end
+  end
+end

data/lib/turbopuffer/active_record/schema/attribute.rb

@@ -5,16 +5,20 @@ module Turbopuffer
         TYPES = %r{\A(?:\[\])?(?:string|uuid|int|uint|float|bool|datetime)\z|\Abytes\z|\A\[\d+\](?:f16|f32|i8)\z|\A\{\}f16\z}
         VECTOR_TYPES = %r{\A\[\d+\]|\A\{\}}
 
-        attr_reader :name, :type, :filterable, :full_text_search, :ann, :glob, :regex, :notnull
+        attr_reader :name, :type, :filterable, :full_text_search, :ann, :glob, :regex, :embed, :notnull
 
         def initialize(name, type, not_null: 0, filterable: false, full_text_search: false,
-                       ann: false, glob: false, regex: false)
+                       ann: false, glob: false, regex: false, embed: nil)
           type = type.to_s
 
           unless TYPES.match?(type)
             raise ArgumentError, "unknown turbopuffer type #{type.inspect} for attribute #{name.inspect}"
           end
 
+          if embed && type != "string"
+            raise ArgumentError, "embed is only supported on string attributes, #{name.inspect} is #{type.inspect}"
+          end
+
           @name = name
           @type = type
           @filterable = filterable
@@ -22,6 +26,7 @@ module Turbopuffer
           @ann = ann
           @glob = glob
           @regex = regex
+          @embed = embed
 
           @notnull = not_null
         end

data/lib/turbopuffer/active_record/schema/model.rb

@@ -13,6 +13,27 @@ module Turbopuffer
           class_attribute :_turbopuffer_consistency,      instance_accessor: false, default: nil
         end
 
+        def turbopuffer_namespace = @turbopuffer_namespace
+
+        def update_columns(...) = with_turbopuffer_namespace { super }
+
+        private
+
+        def init_internals
+          @turbopuffer_namespace = ::Turbopuffer::ActiveRecord.namespace_override ||
+            self.class.current_scope.try(:turbopuffer_namespace)
+          super
+        end
+
+        def _create_record(...) = with_turbopuffer_namespace { super }
+        def _update_row(...)    = with_turbopuffer_namespace { super }
+        def _delete_row(...)    = with_turbopuffer_namespace { super }
+        def _find_record(...)   = with_turbopuffer_namespace { super }
+
+        def with_turbopuffer_namespace(&block)
+          ::Turbopuffer::ActiveRecord.with_namespace(@turbopuffer_namespace, &block)
+        end
+
         class << self
           def namespaces
             ::ActiveRecord::Base.descendants.select do |model|
@@ -48,10 +69,10 @@ module Turbopuffer
         end
 
         class_methods do
-          def turbopuffer_attribute(name, type, not_null: 0, filterable: false,
-                                    full_text_search: false, ann: false, glob: false, regex: false)
+          def turbopuffer_attribute(name, type, not_null: 0, filterable: false, full_text_search: false,
+                                    ann: false, glob: false, regex: false, embed: nil)
             attribute = ::Turbopuffer::ActiveRecord::Schema::Attribute.new(
-              name.to_s, type, not_null:, filterable:, full_text_search:, ann:, glob:, regex:
+              name.to_s, type, not_null:, filterable:, full_text_search:, ann:, glob:, regex:, embed:
             )
 
             self.turbopuffer_attributes =
@@ -82,6 +103,10 @@ module Turbopuffer
             select(*turbopuffer_attributes.map(&:name))
           end
 
+          def namespace(name)
+            annotate("#{NAMESPACE_ANNOTATION}#{name}").extending(::Turbopuffer::ActiveRecord::NamespaceScope)
+          end
+
           def glob(pattern, case_sensitive: true)
             ::Turbopuffer::ActiveRecord::Glob.new(pattern, case_sensitive:)
           end
@@ -119,6 +144,7 @@ module Turbopuffer
               attrs[:ann] = true if attribute.ann
               attrs[:glob] = true if attribute.glob
               attrs[:regex] = true if attribute.regex
+              attrs[:embed] = attribute.embed if attribute.embed
               attrs[:sparse_knn] = { distance_metric: "dot_product" } if attribute.type == "{}f16"
 
               schema[attribute.name] = attrs

data/lib/turbopuffer/active_record/schema.rb

@@ -1,6 +1,7 @@
 require "arel/nodes/glob"
 require "arel/nodes/rank_by_node"
 require "turbopuffer/active_record/glob"
+require "turbopuffer/active_record/namespace_scope"
 require "turbopuffer/active_record/schema/attribute"
 require "turbopuffer/active_record/schema/model"
 

data/lib/turbopuffer/active_record/version.rb

@@ -1,5 +1,5 @@
 module Turbopuffer
   module ActiveRecord
-    VERSION = "0.1.6"
+    VERSION = "0.1.8"
   end
 end

data/lib/turbopuffer/active_record.rb

@@ -4,6 +4,20 @@ module Turbopuffer
   module ActiveRecord
     class Error < StandardError; end
 
+    NAMESPACE_ANNOTATION = "turbopuffer_namespace="
+
+    def self.namespace_override
+      ActiveSupport::IsolatedExecutionState[:turbopuffer_namespace]
+    end
+
+    def self.with_namespace(name)
+      previous = namespace_override
+      ActiveSupport::IsolatedExecutionState[:turbopuffer_namespace] = name
+      yield
+    ensure
+      ActiveSupport::IsolatedExecutionState[:turbopuffer_namespace] = previous
+    end
+
     # Teaches Active Record that `adapter: turbopuffer` in database.yml maps to
     # our adapter. The railtie calls this during boot; test harnesses and
     # non-Rails consumers call it themselves.

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-turbopuffer-adapter
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Richard Monette
 bindir: bin
 cert_chain: []
-date: 2026-09-24 00:00:00.000000000 Z
+date: 2026-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -27,16 +27,16 @@ dependencies:
   name: base64
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -89,6 +89,7 @@ files:
 - lib/arel/visitors/turbopuffer.rb
 - lib/turbopuffer/active_record.rb
 - lib/turbopuffer/active_record/glob.rb
+- lib/turbopuffer/active_record/namespace_scope.rb
 - lib/turbopuffer/active_record/railtie.rb
 - lib/turbopuffer/active_record/schema.rb
 - lib/turbopuffer/active_record/schema/attribute.rb