woods 1.6.1 → 2.0.0.beta2

data/lib/woods/storage/qdrant.rb

@@ -3,12 +3,19 @@
 require 'ipaddr'
 require 'net/http'
 require 'json'
+require 'openssl'
 require 'socket'
 require 'uri'
 require_relative 'vector_store'
 require_relative '../util/host_guard'
+require_relative '../util/uuid5'
 
 module Woods
+  # Same conditional-define pattern used elsewhere in the gem (pgvector,
+  # metadata_store) so this file can be required in isolation without
+  # tripping NameError on the RequestError superclass below.
+  class Error < StandardError; end unless defined?(Woods::Error)
+
   module Storage
     module VectorStore
       # Qdrant adapter for vector storage and similarity search via HTTP API.
@@ -25,6 +32,26 @@ module Woods
       class Qdrant # rubocop:disable Metrics/ClassLength
         include Interface
 
+        class RequestError < Woods::Error
+          attr_reader :http_status, :retry_after
+
+          def initialize(message, http_status: nil, retry_after: nil, retryable: false, ambiguous: false)
+            super(message)
+            @http_status = http_status
+            @retry_after = retry_after
+            @retryable = retryable
+            @ambiguous = ambiguous
+          end
+
+          def retryable?
+            @retryable
+          end
+
+          def ambiguous?
+            @ambiguous
+          end
+        end
+
         # URL schemes allowed for the Qdrant endpoint. `file://`, `gopher://`,
         # and anything else would let a misconfigured or attacker-controlled
         # config value turn the adapter into an SSRF vector against the host
@@ -67,6 +94,54 @@ module Woods
         # Hostnames that always map to loopback regardless of DNS.
         PRIVATE_HOSTNAMES = %w[localhost localhost. ip6-localhost ip6-loopback].freeze
 
+        # Fixed UUIDv5 namespace for Woods → Qdrant point ids.
+        #
+        # Qdrant only accepts an unsigned integer or a UUID as a point id;
+        # a Woods identifier ("User", "Api::V1::UsersController",
+        # "User#chunk_0") is neither, and sending one back is a 400 on
+        # every upsert. Point ids are therefore UUIDv5 values derived from
+        # the identifier, with the identifier itself carried in the payload
+        # under {IDENTIFIER_KEY} so search and delete can round-trip.
+        #
+        # Derived once as
+        #   Util::UUID5.generate(Util::UUID5::NAMESPACE_DNS, 'woods.qdrant.point-id')
+        # and pinned as a literal here. `spec/util/uuid5_spec.rb` asserts
+        # the literal still equals that derivation.
+        #
+        # **This value must never change.** The whole point of a v5 id is
+        # that re-embedding an unchanged unit lands on the same point and
+        # *replaces* it. A new namespace makes every existing point
+        # unreachable — orphaned vectors that no delete can name and a
+        # silently doubled collection.
+        POINT_ID_NAMESPACE = '7eb8ae2b-670b-55ee-a474-36bd1a8dc6b4'
+
+        # Query string appended to every mutating point operation.
+        #
+        # Qdrant defaults these endpoints to `wait=false`: the API returns
+        # `status: "acknowledged"` as soon as the change is queued, before it is
+        # readable. Every caller in this gem assumes otherwise — the embed
+        # pipeline writes vectors and then dumps/verifies, and the prune paths
+        # delete and then re-count — so an un-awaited mutation reads back as
+        # stale data (a deleted unit still answering searches, a fresh count
+        # showing the pre-write total). Correctness beats the throughput the
+        # async default buys.
+        WAIT_FOR_WRITE = '?wait=true'
+
+        # Points per page when scrolling ids in {#each_id}. Large enough that a
+        # sizable index costs few round trips, small enough that one response
+        # stays comfortably in memory (ids and one payload key only).
+        SCROLL_PAGE_SIZE = 1_000
+        DISTANCES = %w[Cosine Dot Euclid Manhattan].freeze
+
+        # Payload key holding the original Woods identifier for a point.
+        #
+        # Deliberately NOT `identifier`: the embedding Indexer already
+        # writes an `identifier` key holding the unit's *base* identifier,
+        # while a point id is derived from the possibly chunk-suffixed
+        # embed id ("User#chunk_0"). Reusing the key would clobber one
+        # with the other.
+        IDENTIFIER_KEY = 'woods_identifier'
+
         # @param url [String] Qdrant server URL
         # @param collection [String] Collection name
         # @param api_key [String, nil] Optional API key for authentication
@@ -80,12 +155,13 @@ module Woods
         #   by default to block the common SSRF footgun. Set to true when the
         #   operator intentionally runs Qdrant on `localhost:6333` or inside
         #   a private network.
-        def initialize(url:, collection:, api_key: nil, dimensions: nil, allow_private_hosts: false)
+        def initialize(url:, collection:, api_key: nil, dimensions: nil, distance: 'Cosine', allow_private_hosts: false) # rubocop:disable Metrics/ParameterLists
           @uri = self.class.validate_url!(url, allow_private_hosts: allow_private_hosts)
           @url = url
           @collection = collection
           @api_key = api_key
           @dimensions = dimensions
+          @distance = normalize_distance(distance)
         end
 
         # Validate a Qdrant endpoint URL — scheme in {ALLOWED_SCHEMES} and,
@@ -185,34 +261,49 @@ module Woods
         #
         # @param dimensions [Integer] Vector dimensionality
         def ensure_collection!(dimensions:)
-          @dimensions ||= dimensions
-          body = {
-            vectors: {
-              size: dimensions,
-              distance: 'Cosine'
-            }
-          }
-          request(:put, "/collections/#{@collection}", body)
+          dimensions = normalize_dimensions(dimensions)
+          validate_configured_dimensions!(dimensions)
+          @dimensions = dimensions
+          existing = request(:get, "/collections/#{@collection}")
+          verify_collection_dimensions!(existing, dimensions)
+          true
+        rescue RequestError => e
+          raise unless e.http_status == 404
+
+          create_collection!(dimensions)
+          true
+        end
+
+        # Deterministic Qdrant point id for a Woods identifier.
+        #
+        # Integers and canonical UUIDs pass through untouched — those are
+        # already native Qdrant point ids, so a caller holding one (from a
+        # raw scroll, say) can address the point directly. Everything else
+        # is a Woods identifier and becomes its UUIDv5.
+        #
+        # @param identifier [String, Integer]
+        # @return [String, Integer]
+        def self.point_id(identifier)
+          return identifier if identifier.is_a?(Integer)
+          return identifier if Util::UUID5.uuid?(identifier)
+
+          Util::UUID5.generate(POINT_ID_NAMESPACE, identifier)
         end
 
         # Store or update a vector with metadata payload.
         #
+        # The point id sent to Qdrant is {.point_id}(id), not +id+ itself;
+        # the original identifier travels in the payload under
+        # {IDENTIFIER_KEY} so {#search} can map results back.
+        #
         # @param id [String] Unique identifier
         # @param vector [Array<Float>] The embedding vector
         # @param metadata [Hash] Optional payload metadata
         # @see Interface#store
         def store(id, vector, metadata = {})
           validate_dimensions!(vector) if @dimensions
-          body = {
-            points: [
-              {
-                id: id,
-                vector: vector,
-                payload: metadata
-              }
-            ]
-          }
-          request(:put, "/collections/#{@collection}/points", body)
+          body = { points: [build_point(id, vector, metadata)] }
+          request(:put, "/collections/#{@collection}/points#{WAIT_FOR_WRITE}", body)
         end
 
         # Store multiple vectors in a single batch upsert request.
@@ -236,21 +327,29 @@ module Woods
           end
 
           body = {
-            points: entries.map do |entry|
-              { id: entry[:id], vector: entry[:vector], payload: entry[:metadata] || {} }
-            end
+            points: entries.map { |entry| build_point(entry[:id], entry[:vector], entry[:metadata] || {}) }
           }
-          request(:put, "/collections/#{@collection}/points", body)
+          request(:put, "/collections/#{@collection}/points#{WAIT_FOR_WRITE}", body)
         end
 
         # Search for similar vectors.
         #
+        # The query vector is dimension-checked before the request, mirroring
+        # the upsert path: otherwise a wrong-dimension query surfaces as a
+        # Qdrant 400 instead of the typed Woods::Error callers already handle
+        # from {#store}/{#store_batch}.
+        #
         # @param query_vector [Array<Float>] The query embedding
         # @param limit [Integer] Maximum results to return
         # @param filters [Hash] Metadata key-value filters
-        # @return [Array<SearchResult>] Results sorted by descending similarity
+        # @return [Array<SearchResult>] Results sorted by descending similarity,
+        #   with +id+ carrying the Woods identifier (not the UUID point id)
+        #   so the adapter is interchangeable with pgvector downstream.
+        # @raise [Woods::Error] if the query vector's length disagrees with the
+        #   configured dimension
         # @see Interface#search
         def search(query_vector, limit: 10, filters: {})
+          validate_dimensions!(query_vector) if @dimensions
           body = {
             vector: query_vector,
             limit: limit,
@@ -262,24 +361,88 @@ module Woods
           results = response['result'] || []
 
           results.map do |hit|
+            payload = hit['payload'] || {}
             SearchResult.new(
-              id: hit['id'],
+              # Reverse-map the UUID point id back to the Woods identifier.
+              # Falls back to the raw point id for points written by
+              # something other than this adapter (or before the UUID
+              # mapping existed) — a hit with no reverse mapping is still
+              # more useful than nil.
+              id: payload[IDENTIFIER_KEY] || hit['id'],
               score: hit['score'],
-              metadata: hit['payload']
+              metadata: payload
             )
           end
         end
 
+        # The vector width the collection was actually created with.
+        #
+        # `ensure_collection!` is a PUT that Qdrant treats as idempotent, so an
+        # existing collection keeps whatever width it was built with; a
+        # dimension change then surfaces as a 400 on every upsert. Reading it
+        # back lets the pipeline refuse up front with a re-index remedy (#214).
+        #
+        # @return [Integer, nil] the collection's vector size, or nil when the
+        #   collection does not exist or the shape is unrecognized (named
+        #   vectors, for instance, which this adapter does not write)
+        def stored_dimensions
+          extract_dimensions(request(:get, "/collections/#{@collection}"))
+        rescue RequestError => e
+          raise unless e.http_status == 404
+
+          nil
+        end
+
+        # Iterate over every stored id, yielding the Woods identifier.
+        #
+        # Uses the scroll API with `with_vector: false` and only the identifier
+        # key requested, so reconciliation costs payload-sized pages rather
+        # than whole vectors. Yields the same value {#search} returns as `id`
+        # (the Woods identifier, reverse-mapped from the payload) so callers
+        # can compare against extraction output directly, and so any id yielded
+        # here can be handed straight back to {#delete}.
+        #
+        # Points written by something else — no {IDENTIFIER_KEY} in their
+        # payload — are **skipped**, not yielded (STO-2). A collection may be
+        # shared with another writer; an unattributable point can never appear
+        # in extraction output, so yielding it would make the embed pipeline's
+        # reconciliation sweep read it as a vanished unit and delete another
+        # system's vector on every run. Enumerating only what Woods wrote is
+        # the property callers depend on. {#search} still falls back to the raw
+        # id, because there the point was matched, not swept.
+        #
+        # @see Interface#each_id
+        def each_id(&block)
+          return enum_for(:each_id) unless block
+
+          offset = nil
+          loop do
+            points, offset = scroll_page(offset)
+            points.each do |point|
+              identifier = woods_identifier_for(point)
+              yield(identifier) if identifier
+            end
+            break if offset.nil?
+          end
+        end
+
+        # Delete a single point by Woods identifier.
+        #
+        # Translates through {.point_id}, so it necessarily computes the
+        # same id {#store}/{#store_batch} wrote. Deleting by the raw Woods
+        # string would 400 (or, worse, succeed against nothing) and leave
+        # the vector live — silent data retention.
+        #
         # @see Interface#delete
         def delete(id)
-          body = { points: [id] }
-          request(:post, "/collections/#{@collection}/points/delete", body)
+          body = { points: [self.class.point_id(id)] }
+          request(:post, "/collections/#{@collection}/points/delete#{WAIT_FOR_WRITE}", body)
         end
 
         # @see Interface#delete_by_filter
         def delete_by_filter(filters)
           body = { filter: build_filter(filters) }
-          request(:post, "/collections/#{@collection}/points/delete", body)
+          request(:post, "/collections/#{@collection}/points/delete#{WAIT_FOR_WRITE}", body)
         end
 
         # @see Interface#count
@@ -290,6 +453,105 @@ module Woods
 
         private
 
+        def normalize_dimensions(value)
+          dimensions = Integer(value)
+          raise ArgumentError, 'dimensions must be positive' unless dimensions.positive?
+
+          dimensions
+        end
+
+        def normalize_distance(value)
+          distance = DISTANCES.find { |candidate| candidate.casecmp?(value.to_s) }
+          return distance if distance
+
+          raise ArgumentError, "distance must be one of #{DISTANCES.join(', ')}"
+        end
+
+        def validate_configured_dimensions!(dimensions)
+          return unless @dimensions && Integer(@dimensions) != dimensions
+
+          raise Woods::ConfigurationError,
+                "Qdrant dimension mismatch: Builder requested #{dimensions}, " \
+                "but vector_store_options configured #{@dimensions}"
+        end
+
+        def verify_collection_dimensions!(response, dimensions)
+          vectors = response.dig('result', 'config', 'params', 'vectors')
+          unless vectors.is_a?(Hash) && vectors['size']
+            raise Woods::ConfigurationError,
+                  "Qdrant collection #{@collection.inspect} uses named vectors; named vectors are not supported " \
+                  'by this unnamed-vector adapter.'
+          end
+          existing_dimensions = vectors['size']
+          if existing_dimensions != dimensions
+            raise Woods::ConfigurationError,
+                  "Qdrant collection dimension mismatch: configured #{dimensions}, existing #{existing_dimensions}. " \
+                  'Use a new collection or rebuild the existing collection.'
+          end
+          return if vectors['distance'] == @distance
+
+          raise Woods::ConfigurationError,
+                "Qdrant collection distance mismatch: configured #{@distance}, existing #{vectors['distance']}. " \
+                'Use a new collection or rebuild the existing collection.'
+        end
+
+        def create_collection!(dimensions)
+          request(
+            :put,
+            "/collections/#{@collection}",
+            vectors: { size: dimensions, distance: @distance }
+          )
+        end
+
+        def extract_dimensions(response)
+          config = response.dig('result', 'config', 'params', 'vectors')
+          return unless config.is_a?(Hash)
+
+          size = config['size']
+          size if size.is_a?(Integer) && size.positive?
+        end
+
+        # Fetch one page of the scroll cursor.
+        #
+        # @param offset [Object, nil] the cursor from the previous page
+        # @return [Array(Array<Hash>, Object)] the page's points and the next
+        #   offset (nil when the scroll is exhausted)
+        def scroll_page(offset)
+          body = { limit: SCROLL_PAGE_SIZE, with_payload: [IDENTIFIER_KEY], with_vector: false }
+          body[:offset] = offset if offset
+
+          result = request(:post, "/collections/#{@collection}/points/scroll", body)['result'] || {}
+          [result['points'] || [], result['next_page_offset']]
+        end
+
+        # The Woods identifier a scrolled point carries, or nil when the point
+        # was not written by this adapter.
+        #
+        # @param point [Hash] a scroll-result point
+        # @return [String, nil]
+        def woods_identifier_for(point)
+          (point['payload'] || {})[IDENTIFIER_KEY]
+        end
+
+        # Build one Qdrant point: UUIDv5 id, vector, and a payload carrying
+        # the original Woods identifier alongside the caller's metadata.
+        #
+        # The identifier is merged in rather than replacing the payload, and
+        # written last so a metadata hash that already carries the key can't
+        # break the reverse mapping. Symbol and string metadata keys both
+        # serialize to JSON strings, so the string key here is what Qdrant
+        # stores either way.
+        #
+        # @param id [String, Integer] Woods identifier (or native point id)
+        # @param vector [Array<Float>]
+        # @param metadata [Hash]
+        # @return [Hash]
+        def build_point(id, vector, metadata)
+          { id: self.class.point_id(id),
+            vector: vector,
+            payload: (metadata || {}).merge(IDENTIFIER_KEY => id) }
+        end
+
         # Cap interpolated response bodies so misconfigured Qdrant responses
         # (e.g. proxied HTML error pages) don't unbounded-leak into logs or
         # re-raised error messages.
@@ -341,22 +603,70 @@ module Woods
         # @raise [Woods::Error] if the API returns a non-success status
         def request(method, path, body = nil)
           req = build_request(method, path, body)
-          response = http_client.request(req)
-
-          unless response.is_a?(Net::HTTPSuccess)
-            raise Woods::Error, "Qdrant API error: #{response.code} #{truncate_response_body(response.body)}"
+          attempt = 0
+
+          begin
+            attempt += 1
+            response = http_client.request(req)
+            parse_response(response, path)
+          rescue OpenSSL::SSL::SSLError => e
+            discard_http_client
+            raise RequestError, "Qdrant TLS error: #{e.message}"
+          rescue Errno::ECONNREFUSED, SocketError, Net::OpenTimeout => e
+            discard_http_client
+            retry if attempt == 1
+
+            raise transport_error(e, ambiguous: false)
+          rescue Errno::ECONNRESET, Net::ReadTimeout, Net::WriteTimeout, IOError => e
+            discard_http_client
+            retry if attempt == 1 && !write_request?(method, path)
+
+            raise transport_error(e, ambiguous: write_request?(method, path))
           end
+        end
 
-          JSON.parse(response.body)
-        rescue Errno::ECONNRESET, Net::OpenTimeout, IOError
-          # Connection dropped — reset and retry once
-          @http_client = nil
-          response = http_client.request(req)
-          unless response.is_a?(Net::HTTPSuccess)
-            raise Woods::Error, "Qdrant API error: #{response.code} #{truncate_response_body(response.body)}"
-          end
+        def parse_response(response, path)
+          raise response_error(response) unless response.is_a?(Net::HTTPSuccess)
 
           JSON.parse(response.body)
+        rescue JSON::ParserError => e
+          raise RequestError.new(
+            "Qdrant API returned malformed JSON for #{path}: #{e.message}",
+            http_status: response.code.to_i
+          )
+        end
+
+        def response_error(response)
+          status = response.code.to_i
+          RequestError.new(
+            "Qdrant API error: #{status} #{truncate_response_body(response.body)}",
+            http_status: status,
+            retry_after: response['Retry-After'],
+            retryable: status == 408 || status == 429 || status >= 500
+          )
+        end
+
+        def transport_error(error, ambiguous:)
+          RequestError.new(
+            "Qdrant transport error: #{error.class}: #{error.message}",
+            retryable: true,
+            ambiguous: ambiguous
+          )
+        end
+
+        # Only GET and read-only POST (search/count/scroll) retry a
+        # mid-exchange network failure — every other verb, and every other
+        # POST (upsert, delete), may already have committed server-side, so
+        # a blind retry risks double-applying it.
+        def write_request?(method, path)
+          return false if method == :get
+          return false if method == :post && read_only_post?(path)
+
+          true
+        end
+
+        def read_only_post?(path)
+          %w[/search /count /scroll].any? { |suffix| path.split('?').first.end_with?(suffix) }
         end
 
         # Return a reusable, started HTTP client for the Qdrant server.
@@ -376,6 +686,22 @@ module Woods
           @http_client = http
         end
 
+        # Close a persistent connection before dropping the reference. Nil-ing
+        # {#http_client} alone abandons the open socket — the descriptor stays
+        # allocated until GC finalizes the object — so finish it first. Finish
+        # raises IOError on a session that was never started, and a connection
+        # that died mid-request can refuse to close cleanly; the rescue keeps
+        # the discard best-effort and only guarantees the reference is dropped.
+        #
+        # @return [void]
+        def discard_http_client
+          @http_client&.finish
+        rescue StandardError
+          nil
+        ensure
+          @http_client = nil
+        end
+
         # Build an HTTP request with headers and body.
         #
         # @param method [Symbol] HTTP method