woods 2.0.0 → 2.1.0

data/lib/tasks/woods.rake

@@ -58,6 +58,7 @@ namespace :woods do
   desc 'Incremental extraction based on git changes'
   task incremental: :environment do
     require 'woods/extractor'
+    require 'woods/input_rules'
 
     output_dir = ENV.fetch('WOODS_OUTPUT', Woods.configuration.output_dir)
 
@@ -71,8 +72,9 @@ namespace :woods do
     # PathDispatcher alongside the dispatch itself — a second hand-maintained
     # pattern list here would drift, and a path this filter drops never
     # reaches the index however good the dispatch behind it is (#164).
-    dispatcher = Woods::PathDispatcher.new
-    changed_files = changed_files.reject(&:empty?).select { |f| dispatcher.relevant?(f) }
+    input_rules = Woods::InputRules.new
+    changed_files = changed_files.reject(&:empty?).reject { |path| input_rules.action(path) == :ignore }
+    full = changed_files.any? { |path| input_rules.action(path) == :full }
 
     if changed_files.empty?
       puts 'No relevant files changed. Skipping extraction.'
@@ -91,13 +93,17 @@ namespace :woods do
       puts 'Warning: a watch daemon is alive but degraded — extracting anyway rather than assuming coverage.'
     end
 
-    puts "Incremental extraction for #{changed_files.size} changed files..."
+    puts "#{full ? 'Full' : 'Incremental'} extraction for #{changed_files.size} changed files..."
     changed_files.each { |f| puts "  - #{f}" }
     puts
 
     extractor = Woods::Extractor.new(output_dir: output_dir)
     affected = Woods::RakeHelpers.woods_with_extraction_lock(output_dir) do
-      extracted = extractor.extract_changed(changed_files)
+      extracted = if full
+                    extractor.extract_all.values.flatten(1).map(&:identifier).uniq
+                  else
+                    extractor.extract_changed(changed_files)
+                  end
       extractor.raise_on_publication_failure!
       extracted
     end
@@ -168,7 +174,7 @@ namespace :woods do
       # FS events do not propagate and the daemon would sit silent. Nothing
       # exposed it before, which made the advice unfollowable.
       force_polling: ENV['WOODS_WATCH_POLL'] == '1', # container autodetect also applies; see Watcher.containerized?
-      idle_timeout: ENV.fetch('WOODS_WATCH_IDLE_TIMEOUT', nil) && Float(ENV.fetch('WOODS_WATCH_IDLE_TIMEOUT')),
+      idle_timeout: Woods::RakeHelpers.woods_watch_idle_timeout,
       catch_up: ENV['WOODS_WATCH_CATCH_UP'] != '0',
       boot_snapshot: boot_snapshot,
       lifecycle: managed_child,
@@ -724,6 +730,8 @@ namespace :woods do
     env_flag = ->(name) { %w[1 true yes].include?(ENV.fetch(name, '').strip.downcase) }
     force_full = env_flag.call('UNBLOCKED_FORCE_FULL_SYNC')
     force_purge = env_flag.call('UNBLOCKED_FORCE_PURGE')
+    dry_run = env_flag.call('UNBLOCKED_DRY_RUN')
+    migrate_from_ref = ENV.fetch('UNBLOCKED_MIGRATE_FROM_REF', nil)
 
     puts 'Syncing extraction data to Unblocked...'
     puts "  Output dir:     #{output_dir}"
@@ -735,12 +743,19 @@ namespace :woods do
     exporter = Woods::Unblocked::Exporter.new(
       index_dir: output_dir,
       force_full: force_full,
-      force_purge: force_purge
+      force_purge: force_purge,
+      dry_run: dry_run,
+      migrate_from_ref: migrate_from_ref
     )
     stats = exporter.sync_all
 
     puts
-    puts 'Sync complete!'
+    if stats[:dry_run]
+      puts 'Sync preview (no remote requests or local writes):'
+      puts JSON.pretty_generate(stats)
+      next
+    end
+    puts(stats[:complete] == false ? 'Sync incomplete; re-run after resolving the reported condition.' : 'Sync complete!')
     puts "  Documents synced:   #{stats[:synced]}"
     puts "  Documents skipped:  #{stats[:skipped]}"
     puts "  Documents deleted:  #{stats[:deleted]}"
@@ -765,6 +780,7 @@ namespace :woods do
         exit 1
       end
     end
+    exit 1 if stats[:complete] == false && stats[:errors].empty?
   end
 
   desc 'Relay findings to Unblocked (alias for unblocked_sync)'

data/lib/tasks/woods_checks.rake

@@ -11,6 +11,7 @@
 #   WOODS_CHECK_STRICT=1 ...                             # exit 1 on findings (still heuristic; see below)
 
 require 'json'
+require 'woods/rake_helpers'
 
 namespace :woods do
   namespace :check do
@@ -62,8 +63,7 @@ module Woods
     # @return [String]
     def check_index_dir
       ENV.fetch('WOODS_OUTPUT') do
-        root = respond_to?(:woods_task_root, true) ? woods_task_root : Rake.application.original_dir
-        File.join(root, 'tmp/woods')
+        File.join(Woods::RakeHelpers.woods_task_root, 'tmp/woods')
       end
     end
 

data/lib/woods/agent_configuration/cli.rb

@@ -86,7 +86,7 @@ module Woods
         protected_paths = layout.allowed_paths + layout.runtime_paths
         raise Conflict, 'Plan output must differ from every managed/runtime target' if protected_paths.include?(target)
 
-        Document.validate_path!(target)
+        Plan.validate_path!(target)
         content = "#{JSON.pretty_generate(plan.data)}\n"
         raise Conflict, 'Plan exceeds the supported size' if content.bytesize > Document::MAX_BYTES
 

data/lib/woods/agent_configuration/layout.rb

@@ -64,8 +64,22 @@ module Woods
       end
 
       def identity
-        { 'client' => 'claude', 'scope' => scope, 'root' => root, 'config_dir' => config_dir,
-          'config_path' => config_path, 'receipt_path' => receipt_path }
+        { 'client' => 'claude', 'scope' => scope, 'root' => root,
+          'config_path' => config_path, 'receipt_path' => receipt_path }.tap do |value|
+          value['config_dir'] = config_dir if scope == 'user'
+        end
+      end
+
+      # Older project receipts/plans included an unused user configuration path.
+      # Only that field is irrelevant; every actual target and scope stays exact.
+      def self.validate_identity!(record, expected, message:)
+        raise Conflict, "#{message} (identity field: layout)" unless record.is_a?(Hash)
+
+        record = record.except('config_dir') if expected['client'] == 'claude' && expected['scope'] == 'project'
+        return if record == expected
+
+        fields = (record.keys | expected.keys).reject { |key| record.slice(key) == expected.slice(key) }
+        raise Conflict, "#{message} (identity fields: #{fields.join(', ')})"
       end
     end
   end

data/lib/woods/agent_configuration/plan.rb

@@ -3,6 +3,7 @@
 require 'base64'
 require 'json'
 require_relative 'document'
+require_relative 'layout'
 
 module Woods
   module AgentConfiguration
@@ -19,11 +20,19 @@ module Woods
       end
 
       def self.load(path)
+        validate_path!(path)
         object = allocate
         object.instance_variable_set(:@data, Document.new(path).json)
         object
       end
 
+      def self.validate_path!(path)
+        Document.validate_path!(path)
+      rescue Conflict => e
+        raise Conflict, "#{e.message}. Use a regular plan file under a real parent directory; " \
+                        'for temporary plans, create a private directory and resolve it with File.realpath.'
+      end
+
       def add(document, content, description:)
         return if content == document.content
 
@@ -67,12 +76,13 @@ module Woods
       private
 
       def validate_format!(layout)
-        valid = data.is_a?(Hash) && data['schema_version'] == SCHEMA_VERSION && data['layout'] == layout.identity &&
+        valid = data.is_a?(Hash) && data['schema_version'] == SCHEMA_VERSION &&
                 %w[setup update remove].include?(data['operation']) && data['changes'].is_a?(Array) &&
                 data['changes'].all?(Hash)
-        return if valid
+        message = 'Plan format or selected application/client/scope differs; create a fresh preview'
+        raise Conflict, message unless valid
 
-        raise Conflict, 'Plan format or selected application/client/scope differs; create a fresh preview'
+        Layout.validate_identity!(data['layout'], layout.identity, message: message)
       end
 
       def validate_paths!(layout)

data/lib/woods/agent_configuration/planner_validation.rb

@@ -19,11 +19,13 @@ module Woods
       end
 
       def validate_receipt_identity!
-        valid = @previous['schema_version'] == 1 && @previous['layout'] == @layout.identity &&
+        valid = @previous['schema_version'] == 1 &&
                 @previous['server_name'] == @name && @previous['sections'].is_a?(Hash) &&
                 @previous['created_files'].is_a?(Array)
-        raise Conflict, 'Installation receipt does not match this application/client/scope/server' unless valid
+        message = 'Installation receipt does not match this application/client/scope/server'
+        raise Conflict, message unless valid
 
+        Layout.validate_identity!(@previous['layout'], @layout.identity, message: message)
         validate_receipt_content!
       end
 

data/lib/woods/agent_configuration/preflight.rb

@@ -3,8 +3,8 @@
 require 'json'
 require 'open3'
 require 'timeout'
-require 'bundler'
 require_relative 'error'
+require_relative '../watch/child_environment'
 
 module Woods
   module AgentConfiguration
@@ -32,8 +32,9 @@ module Woods
         end
       RUBY
 
-      def initialize(timeout: 30)
+      def initialize(timeout: 30, environment: ENV)
         @timeout = timeout
+        @environment = environment
       end
 
       def call(launcher)
@@ -54,7 +55,8 @@ module Woods
 
       def run(launcher)
         command = launcher.probe_command(SCRIPT)
-        environment = Bundler.unbundled_env.merge(launcher.probe_environment)
+        environment = Watch::ChildEnvironment.build(@environment.to_h, root: launcher.root)
+                                             .merge(launcher.probe_environment)
         Open3.popen3(environment, *command, chdir: launcher.root, unsetenv_others: true,
                                             pgroup: true) do |stdin, out, err, wait|
           stdin.close

data/lib/woods/builder.rb

@@ -191,10 +191,9 @@ module Woods
     #   their own implementation without patching the Builder. It flows
     #   through {#build_resilient_embedding_provider} like the built-ins.
     #
-    # Strips `embedding_options` keys that belong to the ResolvedConfig layer
-    # (like `:dimension`) before splatting into the provider's constructor —
-    # those keys are useful for the Snapshotter's schema header but
-    # aren't part of the provider's API.
+    # The legacy :dimension option aliases the explicit :dimensions request.
+    # Stored observed widths use :expected_dimensions so restoring an index
+    # does not accidentally change the provider request.
     #
     # @return [Embedding::Provider::Interface] Embedding provider instance
     # @raise [ArgumentError] if the configured type is not recognized
@@ -244,8 +243,8 @@ module Woods
     end
 
     PROVIDER_OPTION_KEYS = {
-      openai: %i[api_key model dimension dimensions],
-      ollama: %i[model host num_ctx read_timeout dimension dimensions],
+      openai: %i[api_key model dimension dimensions expected_dimensions],
+      ollama: %i[model host num_ctx read_timeout dimension dimensions expected_dimensions],
       fake: %i[model dims dimension dimensions]
     }.freeze
     private_constant :PROVIDER_OPTION_KEYS
@@ -328,10 +327,9 @@ module Woods
     # Build the deterministic fake provider (#178).
     #
     # Dimension resolution: `embedding_options[:dims]` maps directly onto
-    # the {Embedding::Provider::Fake} constructor; failing that, the
-    # ResolvedConfig-level `embedding_options[:dimension]` key — normally
-    # snapshot-only bookkeeping stripped by {#provider_kwargs} — is
-    # honoured, so hosts that declare their dimension there (and the MCP
+    # the {Embedding::Provider::Fake} constructor; failing that, the legacy
+    # `embedding_options[:dimension]` alias is honoured, so hosts that
+    # declare their dimension there (and the MCP
     # boot path, which restores exactly that key from woods.json) get
     # vectors of the recorded dimension.
     #
@@ -341,54 +339,20 @@ module Woods
     end
     private :build_fake_provider
 
-    # Build a {Embedding::TextPreparer} calibrated to a given provider.
-    #
-    # OpenAI embedders use tiktoken (cl100k_base) — 4.0 chars/token is a
-    # good conservative average. Ollama BERT/WordPiece tokenizers
-    # (nomic-embed-text, bge-*) run much hotter on dense Ruby/Rails
-    # source — long CamelCase constants, docstrings, callback DSLs, and
-    # heavy symbol use all sit below 2.0 chars/token in practice.
-    # Empirically, a 16 KB chunk of `ActionMailer::Base` still blows the
-    # 8192-token budget at 2.0 chars/token, so we budget at 1.5 to stay
-    # clear of tokenizer surprises even on the densest Rails internals.
-    #
-    # `max_tokens` tracks the provider's actual input budget when it
-    # reports one, falling back to the TextPreparer default otherwise.
-    #
-    # @param provider [Embedding::Provider::Interface]
-    # @return [Embedding::TextPreparer]
+    # Build a preparer with the provider's explicit input-counting policy.
+    # Ratios remain estimates; known OpenAI models use a byte-BPE upper bound.
     def build_text_preparer(provider)
       chars_per_token = chars_per_token_for(provider)
       budget = safe_max_input_tokens(provider)
       max_tokens = budget || Embedding::TextPreparer::DEFAULT_MAX_TOKENS
 
-      Embedding::TextPreparer.new(max_tokens: max_tokens, chars_per_token: chars_per_token)
+      input_budget = Embedding::InputBudget.for(provider, limit: max_tokens, chars_per_token: chars_per_token)
+      Embedding::TextPreparer.new(max_tokens: max_tokens, chars_per_token: chars_per_token, input_budget: input_budget)
     end
 
-    # Build a {Chunking::SemanticChunker} sized to a given provider.
-    #
-    # `max_chars` is derived from the provider's input budget and the
-    # matching chars-per-token ratio, minus the context-prefix
-    # allowance the Indexer accounts for separately. Units that exceed
-    # this ceiling get sliced so no single chunk can blow the provider's
-    # input cap.
-    #
-    # For Ollama (and other BERT/WordPiece-backed models), char-based
-    # estimation is unreliable — CamelCase, `::` separators, and symbol
-    # literals tokenize much denser than chars/token averages suggest.
-    # When the optional `tokenizers` gem is installed, pass a
-    # {Embedding::TokenCounter} and `max_tokens` so the chunker can
-    # verify every slice with the real tokenizer and re-split any piece
-    # that still exceeds `num_ctx`. See docs/EMBEDDING_MODELS.md.
-    #
-    # Ollama v0.13.5+ stopped honouring `truncate: true` on `/api/embed`
-    # (ollama/ollama#14186), so any chunk that exceeds `num_ctx` returns
-    # a 400 rather than being silently truncated. Exact client-side
-    # sizing is the only reliable path until the regression is fixed
-    # upstream.
-    #
-    # @param provider [Embedding::Provider::Interface]
-    # @return [Chunking::SemanticChunker]
+    # Initial semantic sizing is approximate. TextPreparer subsequently checks
+    # every complete prefix plus source against the provider policy and splits
+    # losslessly; no fixed prefix allowance establishes final admission.
     def build_chunker(provider)
       budget = safe_max_input_tokens(provider)
       max_chars = ((budget * chars_per_token_for(provider)).floor - CHUNKER_PREFIX_ALLOWANCE if budget)
@@ -424,12 +388,8 @@ module Woods
 
     private
 
-    # Return a TokenCounter for providers that benefit from exact token
-    # counting. OpenAI's tiktoken ratios are already stable at 4.0
-    # chars/token on code, so it doesn't need this.
-    #
-    # @param provider [Embedding::Provider::Interface]
-    # @return [Embedding::TokenCounter, nil]
+    # Ollama has no universal local tokenizer. This counter is an explicitly
+    # approximate sizing hint; truncate:false enforces the server's actual cap.
     def token_counter_for(provider)
       return unless unwrap_provider(provider).is_a?(Embedding::Provider::Ollama)
 

data/lib/woods/cache/cache_middleware.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'digest'
+require 'securerandom'
 require_relative 'cache_store'
 # CachedEmbeddingProvider includes Embedding::Provider::Interface at load
 # time, so the interface must be defined before this file's class bodies run
@@ -108,6 +109,9 @@ module Woods
     class CachedEmbeddingProvider
       include Embedding::Provider::Interface
 
+      # @return [Object] underlying provider, including any resilience wrapper
+      attr_reader :provider
+
       # @param provider [Embedding::Provider::Interface] The real embedding provider
       # @param cache_store [CacheStore] Cache backend instance
       # @param ttl [Integer] TTL for cached embeddings in seconds
@@ -117,6 +121,10 @@ module Woods
         @ttl = ttl
         @inflight = {}
         @inflight_mutex = Mutex.new
+        identity = @provider.cache_identity if @provider.respond_to?(:cache_identity)
+        # Unknown custom providers cannot prove cross-instance compatibility.
+        # Hash every identity so endpoint credentials never enter backend keys.
+        @embedding_identity = Digest::SHA256.hexdigest(JSON.generate(identity || SecureRandom.hex(16)))
       end
 
       # Embed a single text, returning a cached vector when available.
@@ -128,7 +136,7 @@ module Woods
       # @param text [String] Text to embed
       # @return [Array<Float>] Embedding vector
       def embed(text)
-        cached = @cache_store.read(embedding_key(text))
+        cached = read_cached(text)
         return cached unless cached.nil?
 
         with_single_flight(text) { @provider.embed(text) }
@@ -147,13 +155,17 @@ module Woods
       # @return [Array<Array<Float>>] Embedding vectors (same order as input)
       def embed_batch(texts)
         results, misses, miss_indices = partition_cached(texts)
-        return results if misses.empty?
+        if misses.empty?
+          validate_vectors!(results, texts.size)
+          return results
+        end
 
         to_fetch, to_fetch_positions, our_entries, awaiting = claim_inflight(misses)
 
         fetch_and_fulfill(to_fetch, to_fetch_positions, our_entries, results, miss_indices)
         await_others(awaiting, results, miss_indices)
 
+        validate_vectors!(results, texts.size)
         results
       end
 
@@ -171,6 +183,23 @@ module Woods
         @provider.model_name
       end
 
+      # Preserve pure configuration declarations through this wrapper.
+      def cache_identity
+        @provider.cache_identity if @provider.respond_to?(:cache_identity)
+      end
+
+      def input_budget
+        @provider.input_budget if @provider.respond_to?(:input_budget)
+      end
+
+      def configured_dimensions
+        @provider.configured_dimensions if @provider.respond_to?(:configured_dimensions)
+      end
+
+      def requested_dimensions
+        @provider.requested_dimensions if @provider.respond_to?(:requested_dimensions)
+      end
+
       # Delegate the per-provider input cap so Builder's chunker / text
       # preparer wiring keeps working when the cache wrapper is in front
       # of the provider. Without this, `respond_to?(:max_input_tokens)`
@@ -200,6 +229,7 @@ module Woods
 
         begin
           vector = yield
+          validate_vectors!([vector], 1)
           write_cache(text, vector)
           entry.fulfill(vector)
           vector
@@ -283,13 +313,7 @@ module Woods
 
         begin
           fresh_vectors = @provider.embed_batch(to_fetch)
-          # Reject a malformed provider response up-front rather than silently
-          # fulfilling waiters with `nil` (or masking a missing tail vector by
-          # under-writing the cache).
-          if fresh_vectors.size != to_fetch.size
-            raise ArgumentError,
-                  "provider returned #{fresh_vectors.size} vectors for #{to_fetch.size} texts"
-          end
+          validate_fresh_vectors!(fresh_vectors, to_fetch.size)
         rescue StandardError => e
           our_entries.each { |entry| entry.reject(e) }
           raise
@@ -350,7 +374,7 @@ module Woods
         miss_indices = []
 
         texts.each_with_index do |text, idx|
-          cached = @cache_store.read(embedding_key(text))
+          cached = read_cached(text)
           if cached
             results[idx] = cached
           else
@@ -362,29 +386,32 @@ module Woods
         [results, misses, miss_indices]
       end
 
-      # Build a cache key for an embedding text.
-      #
-      # The provider's model_name is folded into the key: a cached embedding
-      # is only valid for the exact model that produced it. Without this, a
-      # persistent shared backend (Redis/SolidCache) returns the previous
-      # model's vector after a model switch or upgrade — different dimensions
-      # error mid-batch, same dimensions silently corrupt similarity scores
-      # (which the provider-vs-store dimension check can't detect, since it
-      # compares declared widths, not cache contents).
-      #
-      # model_name (a plain attribute) is used rather than dimensions on
-      # purpose: for every supported provider the model uniquely determines
-      # the vector dimensionality, and `Provider#dimensions` can force a live
-      # network probe (Ollama memoizes `embed('test').length`; OpenAI probes
-      # for unknown models). Keying on dimensions made every cache lookup —
-      # including hits — depend on the provider being reachable, defeating the
-      # cache exactly when the backend is down. model_name distinguishes
-      # models without any I/O.
-      #
+      def read_cached(text)
+        vector = @cache_store.read(embedding_key(text))
+        validate_vectors!([vector], 1) unless vector.nil?
+        vector
+      end
+
+      # Validate the complete batch before writing or fulfilling any entry.
+      def validate_fresh_vectors!(vectors, count)
+        raise ArgumentError, "provider returned #{vectors.size} vectors for #{count} texts" if vectors.size != count
+
+        validate_vectors!(vectors, count)
+      end
+
+      def validate_vectors!(vectors, count)
+        width = @provider.configured_dimensions if @provider.respond_to?(:configured_dimensions)
+        Embedding::Provider::VectorValidation.validate!(
+          vectors, expected_count: count, provider: 'CachedEmbeddingProvider', expected_dimensions: width
+        )
+      end
+
+      # Configuration identity is captured without probing dimensions. Version
+      # the domain to prevent reuse of old model-only keys after an upgrade.
       # @param text [String]
       # @return [String]
       def embedding_key(text)
-        Cache.cache_key(:embeddings, @provider.model_name.to_s, Digest::SHA256.hexdigest(text))
+        Cache.cache_key(:embeddings, 'configuration-v1', @embedding_identity, Digest::SHA256.hexdigest(text))
       end
     end
 
@@ -408,6 +435,8 @@ module Woods
         @retriever = retriever
         @cache_store = cache_store
         @context_ttl = context_ttl
+        @context_namespace = SecureRandom.hex(16)
+        @context_mutex = Mutex.new
       end
 
       # Expose the wrapped stores so the MCP +reload+ tool and
@@ -427,20 +456,17 @@ module Woods
         @retriever.corpus_status(include_types: include_types) if @retriever.respond_to?(:corpus_status)
       end
 
-      # Invalidate every cached context result. Called from the MCP +reload+
-      # tool after the retriever's stores have been re-hydrated from a fresh
-      # embed — otherwise cached results from the old embedding run would
-      # linger until their TTL expires and contradict the new stores.
+      # Retire this retriever's cached contexts after its corpus reloads.
       #
-      # Embedding caches (query → vector) are NOT cleared: the query-vector
-      # mapping is deterministic for a given provider+model and survives any
-      # index reload. Only context results (query → ranked units) go stale.
+      # Each retriever has its own unpredictable namespace, even when several
+      # applications share a backend. Rotating it also prevents an in-flight
+      # request from repopulating the active cache with a pre-reload result.
+      # Old entries expire by their configured TTL; no global backend deletion
+      # is needed. Embedding caches are independent and remain available.
       #
       # @return [void]
       def invalidate_context_cache!
-        @cache_store.clear(namespace: :context)
-      rescue StandardError => e
-        warn("[Woods] CachedRetriever context-cache invalidation failed: #{e.message}")
+        @context_mutex.synchronize { @context_namespace = SecureRandom.hex(16) }
       end
 
       # Execute the retrieval pipeline with context-level caching.
@@ -492,7 +518,8 @@ module Woods
       # @param exclude_types [Array<String, Symbol>, nil]
       # @return [String]
       def context_key(query, budget, types: nil, exclude_types: nil, packages: nil, source_paths: nil, evidence: 'full') # rubocop:disable Metrics/ParameterLists
-        parts = [query, budget.to_s, fingerprint(types), fingerprint(exclude_types)]
+        namespace = @context_mutex.synchronize { @context_namespace }
+        parts = [namespace, query, budget.to_s, fingerprint(types), fingerprint(exclude_types)]
         parts << 'lexical' if mode == :lexical
         parts << JSON.generate(evidence: evidence) unless evidence == 'full'
         if Retrieval::Scope.requested?(packages: packages, source_paths: source_paths)