woods 1.1.0 → 1.3.0

data/lib/woods/storage/vector_store.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'set'
+
 module Woods
   module Storage
     # VectorStore provides an interface for storing and searching embedding vectors.
@@ -36,11 +38,39 @@ module Woods
           entries.each { |e| store(e[:id], e[:vector], e[:metadata] || {}) }
         end
 
+        # Iterate over every live entry, yielding `(id, vector, metadata)`.
+        #
+        # Persistence seam for Snapshotter and similar consumers. Default
+        # implementation falls through to `NotImplementedError`; adapters
+        # that need to support dumping must implement it. Persistent
+        # backends (pgvector, Qdrant) aren't expected to implement this —
+        # the Snapshotter only touches non-persistent stores.
+        #
+        # @yield [id, vector, metadata]
+        # @return [Enumerator] when no block given
+        def each_entry
+          raise NotImplementedError
+        end
+
+        # Bulk-load pre-computed entries. Dual of {#each_entry} — the
+        # Snapshotter hydrates a store by feeding this the dump contents.
+        #
+        # @param entries [Enumerable<Hash>] Each entry has :id, :vector, :metadata keys
+        def bulk_load(entries)
+          store_batch(entries.to_a)
+        end
+
         # Search for similar vectors using cosine similarity.
         #
+        # Filter values may be scalars (exact match) or Arrays (membership
+        # match — "value ∈ array"). Adapters implement the membership
+        # semantics natively: in-memory loops, pgvector IN (...), Qdrant
+        # `match: { any: [...] }`.
+        #
         # @param query_vector [Array<Float>] The query embedding vector
         # @param limit [Integer] Maximum number of results to return
-        # @param filters [Hash] Optional metadata filters to apply
+        # @param filters [Hash] Optional metadata filters — values may be
+        #   scalars or Arrays
         # @return [Array<SearchResult>] Results sorted by descending similarity
         # @raise [NotImplementedError] if not implemented by adapter
         def search(query_vector, limit: 10, filters: {})
@@ -87,79 +117,198 @@ module Woods
       #   store.search([1.0, 0.0], limit: 1)
       #   # => [#<SearchResult id="doc1", score=1.0, metadata={type: "model"}>]
       #
-      class InMemory
+      class InMemory # rubocop:disable Metrics/ClassLength
         include Interface
 
+        # Flat-buffer backing. One Array<Float> of length count*dim holds
+        # every vector contiguously; two parallel Arrays hold the ids and
+        # metadata at matching positions. Deleted entries are tombstoned
+        # (their index is added to @tombstones) rather than removed, so
+        # stored vector positions stay stable under concurrent iteration
+        # and dumps. Tombstones are compacted at next full-embed run.
+        #
+        # The flat buffer exists both for cache friendliness during the
+        # cosine kernel (all vectors live in one contiguous allocation)
+        # and to make dump/load via `pack("e*")` a single call rather
+        # than a per-vector concatenation.
         def initialize
-          @entries = {} # id => { vector:, metadata: }
+          @dim = nil
+          @ids = [] # Array<String> (frozen)
+          @vectors_flat = [] # flat Array<Float>, length @ids.size * @dim
+          @metadata = []     # Array<Hash>, index-aligned with @ids
+          @id_to_index = {}  # id => Integer for O(1) delete/overwrite
+          @tombstones = Set.new
         end
 
+        # @return [Integer, nil] dimension of stored vectors, nil if empty
+        attr_reader :dim
+
         # @see Interface#store
         def store(id, vector, metadata = {})
-          @entries[id] = { vector: vector, metadata: metadata }
+          @dim ||= vector.length
+          unless vector.length == @dim
+            raise ArgumentError,
+                  "Vector dimension mismatch (#{vector.length} vs #{@dim})"
+          end
+
+          frozen_id = id.frozen? ? id : id.dup.freeze
+          existing = @id_to_index[frozen_id]
+          if existing
+            overwrite(existing, vector, metadata)
+          else
+            append(frozen_id, vector, metadata)
+          end
+        end
+
+        # @see Interface#bulk_load
+        # Single-pass hydrate — more efficient than N store calls when
+        # the Snapshotter feeds a large dump at boot time.
+        def bulk_load(entries)
+          entries.each { |entry| store(entry[:id], entry[:vector], entry[:metadata] || {}) }
+        end
+
+        # Drop every stored entry, restoring the store to its post-+new+ state.
+        #
+        # Used by the MCP +reload+ tool to pick up a fresh embed run without
+        # restarting the process. A subsequent +#bulk_load+ then repopulates
+        # from disk. Safe on an already-empty store.
+        def clear!
+          @dim = nil
+          @ids = []
+          @vectors_flat = []
+          @metadata = []
+          @id_to_index = {}
+          @tombstones = Set.new
+        end
+
+        # @see Interface#each_entry
+        def each_entry(&block)
+          return enum_for(:each_entry) unless block
+
+          @ids.each_with_index do |id, idx|
+            next if @tombstones.include?(idx)
+
+            base = idx * @dim
+            yield(id, @vectors_flat[base, @dim], @metadata[idx])
+          end
         end
 
         # @see Interface#search
         def search(query_vector, limit: 10, filters: {})
-          candidates = filter_entries(filters)
+          return [] if @dim.nil?
 
-          scored = candidates.map do |id, entry|
-            score = cosine_similarity(query_vector, entry[:vector])
-            SearchResult.new(id: id, score: score, metadata: entry[:metadata])
+          unless query_vector.length == @dim
+            raise ArgumentError,
+                  "Vector dimension mismatch (#{query_vector.length} vs #{@dim})"
           end
-          scored.sort_by { |r| -r.score }.first(limit)
+
+          scored = gather_candidates(query_vector, filters)
+          scored.sort_by! { |r| -r.score }
+          scored.first(limit)
         end
 
         # @see Interface#delete
         def delete(id)
-          @entries.delete(id)
+          idx = @id_to_index.delete(id)
+          @tombstones << idx if idx
         end
 
         # @see Interface#delete_by_filter
         def delete_by_filter(filters)
-          @entries.reject! do |_id, entry|
-            filters.all? { |key, value| entry[:metadata][key] == value }
+          @ids.each_with_index do |id, idx|
+            next if @tombstones.include?(idx)
+            next unless filters.all? { |key, value| @metadata[idx][key] == value }
+
+            @tombstones << idx
+            @id_to_index.delete(id)
           end
         end
 
         # @see Interface#count
         def count
-          @entries.size
+          @ids.size - @tombstones.size
         end
 
         private
 
-        # Filter entries by metadata key-value pairs.
-        #
-        # @param filters [Hash] Metadata filters
-        # @return [Hash] Filtered entries
-        def filter_entries(filters)
-          return @entries if filters.empty?
+        # Match a filter value against a metadata value. Arrays are
+        # membership filters ("any of"); scalars are equality.
+        def filter_match?(filter_value, meta_value)
+          filter_value.is_a?(Array) ? filter_value.include?(meta_value) : filter_value == meta_value
+        end
+
+        # Append a new entry to the flat buffer.
+        def append(id, vector, metadata)
+          idx = @ids.size
+          @ids << id
+          @vectors_flat.concat(vector)
+          @metadata << metadata
+          @id_to_index[id] = idx
+        end
 
-          @entries.select do |_id, entry|
-            filters.all? { |key, value| entry[:metadata][key] == value }
+        # Overwrite an existing entry in place. Tombstones the old slot's
+        # deletion marker (if any) so the new vector is live again.
+        def overwrite(idx, vector, metadata)
+          base = idx * @dim
+          i = 0
+          while i < @dim
+            @vectors_flat[base + i] = vector[i]
+            i += 1
           end
+          @metadata[idx] = metadata
+          @tombstones.delete(idx)
         end
 
-        # Compute cosine similarity between two vectors.
-        #
-        # @param vec_a [Array<Float>] First vector
-        # @param vec_b [Array<Float>] Second vector
-        # @return [Float] Cosine similarity between -1.0 and 1.0
-        # @raise [ArgumentError] if vectors have different dimensions
-        def cosine_similarity(vec_a, vec_b)
-          unless vec_a.length == vec_b.length
-            raise ArgumentError,
-                  "Vector dimension mismatch (#{vec_a.length} vs #{vec_b.length})"
+        # Walk every non-tombstoned index, apply filters, score survivors.
+        # Filter check runs BEFORE the cosine kernel — avoids computing
+        # 12k dot products only to discard most of them.
+        def gather_candidates(query_vector, filters)
+          scored = []
+          len = @ids.size
+          idx = 0
+          while idx < len
+            if @tombstones.include?(idx)
+              idx += 1
+              next
+            end
+            meta = @metadata[idx]
+            unless filters.empty? || filters.all? { |k, v| filter_match?(v, meta[k]) }
+              idx += 1
+              next
+            end
+
+            score = cosine_similarity_strided(query_vector, idx * @dim)
+            scored << SearchResult.new(id: @ids[idx], score: score, metadata: meta)
+            idx += 1
           end
+          scored
+        end
 
-          dot = vec_a.zip(vec_b).sum { |x, y| x * y }
-          mag_a = Math.sqrt(vec_a.sum { |x| x**2 })
-          mag_b = Math.sqrt(vec_b.sum { |x| x**2 })
+        # Cosine similarity between a query Array<Float> and a vector
+        # that lives at @vectors_flat[base, @dim]. Strided access avoids
+        # allocating a copy of the stored vector on every comparison.
+        #
+        # See bench/vector_query_and_serialization.rb for the allocation
+        # story — the old Enumerable path allocated ~770 objects per pair;
+        # this loop allocates none inside the hot path.
+        def cosine_similarity_strided(query, base)
+          len = @dim
+          i = 0
+          dot = 0.0
+          mag_a = 0.0
+          mag_b = 0.0
+          while i < len
+            a = query[i]
+            b = @vectors_flat[base + i]
+            dot += a * b
+            mag_a += a * a
+            mag_b += b * b
+            i += 1
+          end
 
           return 0.0 if mag_a.zero? || mag_b.zero?
 
-          dot / (mag_a * mag_b)
+          dot / (Math.sqrt(mag_a) * Math.sqrt(mag_b))
         end
       end
     end

data/lib/woods/tasks.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative 'builder'
+require_relative 'embedding/indexer'
+require_relative 'embedding/text_preparer'
+require_relative 'resolved_config'
+
+module Woods
+  # Small helpers invoked from `lib/tasks/woods.rake`.
+  #
+  # Keeps rake task bodies to a couple of lines each so the real work lives in
+  # plain Ruby that can be unit-tested without Rake's global state.
+  module Tasks
+    module_function
+
+    # Build an {Embedding::Indexer} wired to the provider and stores described
+    # by {Woods.configuration}. Uses {Builder} so `config.embedding_provider`,
+    # `config.embedding_options`, and `config.vector_store(_options)` are all
+    # honoured — prior to this the rake tasks hardcoded Ollama + InMemory and
+    # silently ignored configuration, which was invisible until the provider
+    # tried to reach an unreachable default host.
+    #
+    # The TextPreparer and SemanticChunker are tuned to the selected
+    # provider so oversize units are split into chunks that fit the
+    # provider's input budget (e.g. Ollama's num_ctx, OpenAI's 8k cap).
+    #
+    # @return [Embedding::Indexer]
+    def build_embed_indexer
+      config = Woods.configuration
+      builder = Builder.new(config)
+      provider = builder.build_embedding_provider
+
+      # Wire the persistence-arc pieces (resolved_config, metadata_store,
+      # dump_retention_count) so Indexer#persist_snapshot can write
+      # woods.json, dump metadata, and honour the user's retention setting.
+      # Without these kwargs, embed writes vectors.bin + latest pointer but
+      # never writes woods.json — which breaks the standalone woods-mcp
+      # Shape-2 boot path entirely.
+      #
+      # metadata_store and resolved_config are nil-safe — hosts that don't
+      # configure metadata or that pre-date the persistence arc still work.
+      Embedding::Indexer.new(
+        provider: provider,
+        text_preparer: builder.build_text_preparer(provider),
+        vector_store: builder.build_vector_store,
+        metadata_store: config.metadata_store ? builder.build_metadata_store : nil,
+        resolved_config: build_resolved_config(config, provider: provider),
+        chunker: builder.build_chunker(provider),
+        dump_retention_count: config.dump_retention_count,
+        output_dir: ENV.fetch('WOODS_OUTPUT', config.output_dir)
+      )
+    end
+
+    # Build a ResolvedConfig snapshot from the live Woods::Configuration.
+    # Returns nil if the configuration doesn't have enough to produce one
+    # (pre-persistence-arc hosts) so the Indexer falls back to the legacy
+    # dump-without-woods.json behaviour.
+    #
+    # Passes the live +provider+ so {ResolvedConfig.from_configuration} can
+    # probe +provider.dimensions+ — without this, Ollama snapshots record
+    # +dimension: 0+ and every subsequent MCP boot fails a spurious
+    # dimension-mismatch check against the real stored vectors.
+    def build_resolved_config(config, provider: nil)
+      return nil unless config.embedding_provider
+
+      ResolvedConfig.from_configuration(config, provider: provider)
+    rescue StandardError
+      nil
+    end
+
+    # Print an indexer stats hash in the format the rake tasks have historically
+    # used. `mode:` only affects the header line.
+    #
+    # @param stats [Hash]
+    # @param mode [Symbol] :full or :incremental
+    def print_embed_stats(stats, mode:)
+      header = mode == :incremental ? 'Incremental embedding complete!' : 'Embedding complete!'
+      puts
+      puts header
+      puts "  Processed: #{stats[:processed]}"
+      puts "  Skipped:   #{stats[:skipped]}"
+      puts "  Errors:    #{stats[:errors]}"
+    end
+  end
+end

data/lib/woods/temporal/snapshot_store.rb

@@ -23,10 +23,58 @@ module Woods
     #
     class SnapshotStore # rubocop:disable Metrics/ClassLength
       # @param connection [Object] Database connection supporting #execute and #get_first_row
-      def initialize(connection:)
+      # @param validate_schema [Boolean] If true (default), probe both required
+      #   tables at construction time and raise a descriptive error pointing at
+      #   migrations 004+005 when they are missing. Set false in tests that
+      #   construct the store with a bare mock.
+      def initialize(connection:, validate_schema: true)
         @db = connection
+        validate_schema! if validate_schema
       end
 
+      REQUIRED_TABLES = %w[woods_snapshots woods_snapshot_units].freeze
+
+      # Probe that `woods_snapshots` and `woods_snapshot_units` exist. If
+      # they don't, raise with guidance to run migrations 004 + 005 —
+      # without this, the first call to {#capture}/{#find} raises a generic
+      # adapter error that doesn't tell operators why.
+      #
+      # When the connection responds to `#columns` (ActiveRecord-shaped) or
+      # `#table_exists?`, use that — these are hard to spoof from a test
+      # mock, so a partial mock can no longer silently pass. Falls back to
+      # the `SELECT 1 FROM t LIMIT 1` probe for minimal connections.
+      #
+      # @raise [Woods::Error]
+      def validate_schema!
+        REQUIRED_TABLES.each { |t| probe_table!(t) }
+      rescue Woods::Error
+        raise
+      rescue StandardError => e
+        raise Woods::Error, schema_error_message(e)
+      end
+
+      private
+
+      def probe_table!(table)
+        if @db.respond_to?(:table_exists?)
+          raise Woods::Error, schema_error_message("table `#{table}` does not exist") unless @db.table_exists?(table)
+        elsif @db.respond_to?(:columns)
+          cols = @db.columns(table)
+          raise Woods::Error, schema_error_message("no columns for `#{table}`") if cols.nil? || cols.empty?
+        else
+          @db.execute("SELECT 1 FROM #{table} LIMIT 1")
+        end
+      end
+
+      def schema_error_message(detail)
+        'SnapshotStore requires the `woods_snapshots` and ' \
+          '`woods_snapshot_units` tables (migrations 004 + 005 under ' \
+          '`lib/woods/db/migrations/`). Run `rake woods:migrate` on the ' \
+          "metadata DB and retry. Underlying error: #{detail}"
+      end
+
+      public
+
       # Capture a snapshot after extraction completes.
       #
       # Stores the manifest metadata and per-unit content hashes.

data/lib/woods/token_utils.rb

@@ -1,19 +1,58 @@
 # frozen_string_literal: true
 
 module Woods
-  # Shared token estimation utility.
+  # Shared token estimation utility — the single source of truth for the
+  # chars-per-token ratio used across cost estimation, context assembly,
+  # and embedding budgeting.
   #
-  # Uses project convention: (string.length / 4.0).ceil
-  # See docs/TOKEN_BENCHMARK.md — conservative floor (~10.6% overestimate).
+  # Ratios:
+  # - `:openai` / default — 4.0 chars/token. Benchmarked against tiktoken
+  #   (cl100k_base) on 19 Ruby source files (mean 4.41 chars/token). We use
+  #   4.0 as a conservative floor (~10.6 % overestimate) so truncation never
+  #   hands the model more tokens than it budgeted for. See
+  #   `docs/TOKEN_BENCHMARK.md`.
+  # - `:ollama` — 1.5 chars/token. Matches the BERT WordPiece tokenizers
+  #   used by nomic-embed-text and mxbai-embed-large. See
+  #   `docs/EMBEDDING_MODELS.md` and `Woods::Builder#chars_per_token_for`.
+  #
+  # Callers should prefer {.chars_per_token_for} over hardcoding a divisor
+  # so future tokenizer changes propagate in one place instead of drifting
+  # between {ContextAssembler}, {Builder}, and cost-model components.
   module TokenUtils
+    CHARS_PER_TOKEN_BY_PROVIDER = {
+      openai: 4.0,
+      ollama: 1.5
+    }.freeze
+
+    DEFAULT_CHARS_PER_TOKEN = CHARS_PER_TOKEN_BY_PROVIDER[:openai]
+
     module_function
 
-    # Estimate token count for a string.
+    # Chars-per-token ratio for the given embedding provider.
+    #
+    # @param provider [Symbol, String, nil] Provider identifier. Unknown or
+    #   nil providers fall back to {DEFAULT_CHARS_PER_TOKEN}.
+    # @return [Float]
+    def chars_per_token_for(provider)
+      CHARS_PER_TOKEN_BY_PROVIDER.fetch(provider&.to_sym, DEFAULT_CHARS_PER_TOKEN)
+    end
+
+    # Estimate token count for a string using the default (OpenAI) ratio.
+    # Use {.estimate_tokens_for} when a specific provider is in play.
     #
     # @param text [String] Text to estimate
     # @return [Integer] Estimated token count
     def estimate_tokens(text)
-      (text.length / 4.0).ceil
+      estimate_tokens_for(text, provider: nil)
+    end
+
+    # Estimate token count for a string using the provider's native ratio.
+    #
+    # @param text [String] Text to estimate
+    # @param provider [Symbol, String, nil] `:openai`, `:ollama`, or nil.
+    # @return [Integer] Estimated token count
+    def estimate_tokens_for(text, provider:)
+      (text.length / chars_per_token_for(provider)).ceil
     end
   end
 end

data/lib/woods/unblocked/client.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+require_relative 'rate_limiter'
+
+module Woods
+  module Unblocked
+    # REST client for the Unblocked API v1.
+    #
+    # Handles document and collection CRUD with rate limiting, retries,
+    # and error handling. Uses Net::HTTP for zero external dependencies.
+    #
+    # @example
+    #   client = Client.new(api_token: "ubk_...")
+    #   client.put_document(
+    #     collection_id: "uuid",
+    #     title: "Order (model)",
+    #     body: "# Order\n...",
+    #     uri: "https://github.com/org/repo/blob/main/app/models/order.rb"
+    #   )
+    #
+    class Client
+      BASE_URL = 'https://getunblocked.com/api/v1'
+      MAX_RETRIES = 3
+      DEFAULT_TIMEOUT = 30
+
+      # @param api_token [String] Unblocked API token (Personal or Team)
+      # @param rate_limiter [RateLimiter] Rate limiter instance
+      # @raise [ArgumentError] if api_token is nil or empty
+      def initialize(api_token:, rate_limiter: RateLimiter.new)
+        raise ArgumentError, 'api_token is required' if api_token.nil? || api_token.to_s.strip.empty?
+
+        @api_token = api_token
+        @rate_limiter = rate_limiter
+      end
+
+      # Create or update a document (upsert by URI).
+      #
+      # Documents are unique by `uri` across the organization. If a document
+      # with the given URI exists, it is updated; otherwise it is created.
+      # Documents become available for queries within ~1 minute.
+      #
+      # @param collection_id [String] Target collection UUID
+      # @param title [String] Document title (plain text)
+      # @param body [String] Document body (Markdown preferred)
+      # @param uri [String] Source URL (used as unique identifier and citation link)
+      # @return [Hash] { "id" => "document-uuid" }
+      def put_document(collection_id:, title:, body:, uri:)
+        request(:put, 'documents', {
+                  collectionId: collection_id,
+                  title: title,
+                  body: body,
+                  uri: uri
+                })
+      end
+
+      # Create a new collection.
+      #
+      # @param name [String] Collection name (1-32 chars)
+      # @param description [String] Collection description (1-4096 chars)
+      # @param icon_url [String, nil] Optional icon URL
+      # @return [Hash] { "id" => "collection-uuid", "name" => "...", ... }
+      def create_collection(name:, description:, icon_url: nil)
+        body = { name: name, description: description }
+        body[:iconUrl] = icon_url if icon_url
+        request(:post, 'collections', body)
+      end
+
+      # List all collections.
+      #
+      # @return [Array<Hash>] Collection objects
+      def list_collections
+        result = request(:get, 'collections')
+        result['items'] || result['data'] || [result].flatten.compact
+      end
+
+      # Delete a document by ID.
+      #
+      # @param document_id [String] Document UUID
+      # @return [Hash] API response
+      def delete_document(document_id:)
+        request(:delete, "documents/#{document_id}")
+      end
+
+      private
+
+      def request(method, path, body = nil)
+        retries = 0
+
+        loop do
+          response = @rate_limiter.track { execute_http(method, path, body) }
+
+          return parse_response(response) if response.is_a?(Net::HTTPSuccess)
+
+          if response.code == '429' && retries < MAX_RETRIES
+            retries += 1
+            wait_time = (response['Retry-After'] || (retries * 2)).to_f
+            sleep(wait_time)
+            next
+          end
+
+          raise_api_error(response)
+        end
+      end
+
+      def execute_http(method, path, body)
+        attempts = 0
+        begin
+          uri = URI("#{BASE_URL}/#{path}")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.open_timeout = DEFAULT_TIMEOUT
+          http.read_timeout = DEFAULT_TIMEOUT
+
+          req = build_request(method, uri, body)
+          http.request(req)
+        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET, Errno::ECONNREFUSED => e
+          attempts += 1
+          raise Woods::Error, "Network error after #{attempts} retries: #{e.message}" if attempts > MAX_RETRIES
+
+          sleep(2**attempts)
+          retry
+        end
+      end
+
+      def build_request(method, uri, body)
+        req = case method
+              when :put then Net::HTTP::Put.new(uri)
+              when :post then Net::HTTP::Post.new(uri)
+              when :get then Net::HTTP::Get.new(uri)
+              when :delete then Net::HTTP::Delete.new(uri)
+              else raise ArgumentError, "Unsupported HTTP method: #{method}"
+              end
+
+        req['Authorization'] = "Bearer #{@api_token}"
+        req['Content-Type'] = 'application/json'
+        req.body = JSON.generate(body) if body
+
+        req
+      end
+
+      def parse_response(response)
+        return {} if response.body.nil? || response.body.strip.empty?
+
+        JSON.parse(response.body)
+      rescue JSON::ParserError
+        {}
+      end
+
+      def raise_api_error(response)
+        parsed = begin
+          JSON.parse(response.body)
+        rescue JSON::ParserError, TypeError
+          { 'message' => response.body&.slice(0, 200) || 'Unknown error' }
+        end
+        message = parsed['message'] || parsed['error'] || 'Unknown error'
+        raise Woods::Error, "Unblocked API error #{response.code}: #{message}"
+      end
+    end
+  end
+end