woods 2.0.0.beta2 → 2.0.0.beta3

data/lib/woods/embedding/indexer.rb

@@ -119,40 +119,14 @@ module Woods
 
       private
 
-      # Where this run's units are read from — the published generation's
-      # payload, or the output root for a flat index.
-      #
-      # Globbing the output root would walk `payloads/` and ingest every
-      # retained generation, so each unit would be embedded once per retained
-      # payload. It also picks up `dumps/`, which holds no unit JSON but is
-      # needlessly large to walk.
-      #
-      # @return [String]
-      def units_dir
-        Woods::Generation.new(output_dir: @output_dir).payload_dir.to_s
-      end
-
       def load_units
-        Dir.glob(File.join(units_dir, '**', '*.json')).filter_map do |path|
-          next if File.basename(path) == 'checkpoint.json'
-
-          # AtomicFile.read, not File.read: a bare read tags the bytes with the
-          # process's default external encoding (US-ASCII under LANG=C), so a
-          # single multibyte character in one unit raised EncodingError out of
-          # JSON.parse and aborted the whole embed run.
-          data = JSON.parse(AtomicFile.read(path))
-          # Extraction output also contains index listings (_index.json arrays) and
-          # summary files (manifest.json, dependency_graph.json, graph_analysis.json)
-          # that live alongside per-unit JSON. Filter to the unit shape.
-          data if data.is_a?(Hash) && data.key?('type') && data.key?('identifier')
-        rescue JSON::ParserError, EncodingError => e
-          warn "[woods] skipping unreadable unit file #{path} (#{e.class}: #{e.message})"
-          nil
-        end
+        require_relative 'corpus'
+
+        Corpus.new(@output_dir).load
       end
 
-      # The invariant: **checkpoint.json never advances over a unit whose
-      # vector was not durably stored.** Two things uphold it here.
+      # The invariant: **checkpoint.json advances only after the intended
+      # vector state (including an empty set) is durable.** Two things uphold it here.
       #
       # 1. Ordering. For a store whose only durable copy is the dump
       #    (+persistable?+), the checkpoint is written *after*
@@ -170,17 +144,19 @@ module Woods
       #
       # 2. Trust, verified. {#checkpoint_satisfied?} honours a checkpoint hit
       #    only when the durable artifact actually holds a vector for that
-      #    unit. A checkpoint that ran ahead of its dump — an older gem with
+      #    unit, or preparation verifies that it intentionally has no text.
+      #    A checkpoint that ran ahead of its dump — an older gem with
       #    this bug, an interrupted promote, a store swap — self-heals into a
       #    re-embed instead of stranding the unit forever.
       def process_units(units, incremental:)
         prepare_run(incremental: incremental)
-        units = assign_storage_identities(units)
         checkpoint = incremental ? load_checkpoint : {}
+        units = assign_storage_identities(units, checkpoint: checkpoint)
         stats = { processed: 0, skipped: 0, errors: 0 }
 
         embed_batches(units, checkpoint, stats, incremental: incremental)
 
+        reconcile_empty_units(checkpoint)
         retire_legacy_identities
         report_checkpoint_misses
         vanished = incremental && persistable? ? drop_vanished_units : 0
@@ -196,12 +172,12 @@ module Woods
       end
 
       # Unambiguous existing keys stay stable. A collision uses reversible typed keys.
-      def assign_storage_identities(units)
+      def assign_storage_identities(units, checkpoint:)
         counts = units.group_by { |unit| unit['identifier'] }.transform_values(&:size)
         units.map do |unit|
           id = unit['identifier']
           typed = StorageIdentity.key(id, unit['type'])
-          existing = known_storage_key?(typed)
+          existing = known_storage_key?(typed, checkpoint: checkpoint)
           next unit unless counts[id] > 1 || existing || id.start_with?(StorageIdentity::PREFIX)
 
           unit.merge('storage_id' => typed)
@@ -224,8 +200,8 @@ module Woods
         end
       end
 
-      def known_storage_key?(key)
-        (@persisted_ids || {}).key?(key) || (@durable_ids || {}).key?(key)
+      def known_storage_key?(key, checkpoint:)
+        (@persisted_ids || {}).key?(key) || (@durable_ids || {}).key?(key) || checkpoint.key?(key)
       end
 
       def retire_legacy_key(legacy)
@@ -243,11 +219,9 @@ module Woods
       # `woods:embed_incremental` runs evict every genuinely older dump in
       # favour of copies of the same state.
       #
-      # Safe because nothing else mutates the *vector* store on a zero-processed
-      # run: `prune_superseded_vectors` is reached only from `store_vectors`,
-      # which runs only for items that were actually embedded. A checkpoint
-      # self-heal counts as processed, so a run that re-embeds a stranded unit
-      # still dumps.
+      # A zero-text transition can retire vectors without a provider call;
+      # @vectors_changed captures that case. Checkpoint self-heals still count
+      # as processed, so a run re-embedding a stranded unit also dumps.
       #
       # But "nothing embedded" is not "nothing changed" (B-069). `persist_snapshot`
       # writes the vector dump, the *metadata* dump and the config into one
@@ -266,6 +240,11 @@ module Woods
       # `@current_identifiers` is what this run saw. Anything in the first and
       # not the second is stale, and the dump has to be rewritten to drop it.
       #
+      # Existing units can also change metadata without changing source (B-119).
+      # Compare their complete records with the promoted metadata snapshot;
+      # publishing those changes must not require a provider call or invalidate
+      # the source-hash checkpoint contract.
+      #
       # Full runs always dump: they rebuild the store from scratch, so "nothing
       # processed" there means the store is genuinely empty and the dump must
       # say so rather than leave a stale one promoted.
@@ -274,7 +253,7 @@ module Woods
       def snapshot_worth_writing?(stats, vanished, incremental:)
         return true unless incremental
 
-        stats[:processed].positive? || vanished.positive?
+        stats[:processed].positive? || vanished.positive? || @metadata_changed || @vectors_changed
       end
 
       # Fraction of the persisted units the vanished-unit sweep may remove
@@ -489,10 +468,36 @@ module Woods
         @current_identifiers = Set.new
         @durable_ids = nil
         @checkpoint_misses = 0
-        hydrate_persisted_vectors if incremental && persistable?
+        @metadata_changed = false
+        @vectors_changed = false
+        @empty_units = {}
+        @persisted_metadata = nil
+        prepare_snapshot_stores(incremental: incremental)
+        retain_metadata_identities
         load_durable_store_ids if reconcilable?
       end
 
+      def prepare_snapshot_stores(incremental:)
+        return unless persistable?
+
+        if incremental
+          hydrate_persisted_metadata
+          hydrate_persisted_vectors
+        else
+          # A direct caller may reuse an in-memory adapter for a full run.
+          # Its old chunks still need replacement, including by an empty set.
+          entries = []
+          @vector_store.each_entry { |id, _vector, _metadata| entries << { id: id } }
+          @persisted_ids = index_ids_by_identifier(entries)
+        end
+      end
+
+      # Source-empty units retain metadata but intentionally have no vectors.
+      # Keep those identities in the same deletion and typed-key accounting.
+      def retain_metadata_identities
+        @persisted_metadata&.each_entry { |identifier, _unit| @persisted_ids[identifier] ||= [] }
+      end
+
       # Read back what the durable store currently holds, as base identifiers.
       #
       # One enumeration serves both halves of #211:
@@ -618,7 +623,10 @@ module Woods
         # enumeration that failed) — fall back to trusting the checkpoint.
         return true if known_ids.nil?
 
-        return true if known_ids.key?(storage_id(unit_data))
+        return true if known_ids[storage_id(unit_data)]&.any?
+        # A source-empty unit intentionally has no vector. Verify that state
+        # again rather than treating a missing nonempty vector as a cache hit.
+        return true if prepare_texts(unit_data).empty?
 
         @checkpoint_misses += 1
         false
@@ -635,7 +643,22 @@ module Woods
       def persist_unit_metadata(unit_data)
         return unless @metadata_store
 
-        @metadata_store.store(storage_id(unit_data), unit_data)
+        id = storage_id(unit_data)
+        @metadata_changed ||= @persisted_metadata && @persisted_metadata.find(id) != unit_data
+        @metadata_store.store(id, unit_data)
+      end
+
+      # Compare with the promoted artifact, not a fresh metadata store or the
+      # source-only checkpoint. Retain old records until the guarded vanished
+      # sweep permits their deletion, just as vector hydration does.
+      def hydrate_persisted_metadata
+        return unless @metadata_store.respond_to?(:each_entry) && @metadata_store.respond_to?(:bulk_load)
+
+        require_relative '../index_artifact'
+        require_relative '../storage/snapshotter'
+
+        @persisted_metadata = Storage::Snapshotter::Metadata.load_or_empty(IndexArtifact.new(@output_dir))
+        @metadata_store.bulk_load(@persisted_metadata.each_entry)
       end
 
       # Refuse to index a unit whose real identifier already matches the
@@ -653,6 +676,7 @@ module Woods
       def collect_embed_items(unit_data, items)
         texts = prepare_texts(unit_data)
         identifier = storage_id(unit_data)
+        @empty_units[identifier] = unit_data['source_hash'] if texts.empty?
 
         texts.each_with_index do |text, idx|
           embed_id = texts.length > 1 ? "#{identifier}#chunk_#{idx}" : identifier
@@ -661,6 +685,26 @@ module Woods
         end
       end
 
+      # Defer zero-text deletion until every batch has prepared/embedded
+      # successfully. A later provider failure must not retire earlier vectors.
+      # Checkpoints retain their source-hash format; a matching no-vector hash
+      # is trusted only after verifying the current input still prepares empty.
+      def reconcile_empty_units(checkpoint)
+        verify_empty_reconciliation!
+        @empty_units.each do |identifier, source_hash|
+          @vectors_changed ||= @persisted_ids[identifier]&.any?
+          prune_identifier(identifier, []) if implements_own?(@vector_store, :delete)
+          prune_durable_identifier(identifier, []) if @durable_ids && implements_own?(@vector_store, :delete)
+          checkpoint[identifier] = source_hash
+        end
+      end
+
+      def verify_empty_reconciliation!
+        return unless @empty_units.any? && reconcilable? && @durable_ids.nil?
+
+        raise Woods::Error, 'Cannot reconcile source-empty units: existing vector IDs could not be read'
+      end
+
       def prepare_texts(unit_data) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         unit = build_unit(unit_data)
         apply_chunking(unit) if @chunker && unit.chunks.empty? && needs_chunking?(unit)

data/lib/woods/embedding/openai.rb

@@ -16,7 +16,7 @@ module Woods
       #   provider = Woods::Embedding::Provider::OpenAI.new(api_key: ENV['OPENAI_API_KEY'])
       #   vector = provider.embed("class User < ApplicationRecord; end")
       #   vectors = provider.embed_batch(["text1", "text2"])
-      class OpenAI
+      class OpenAI # rubocop:disable Metrics/ClassLength
         include Interface
         include DiscardableClient
 
@@ -26,13 +26,20 @@ module Woods
           'text-embedding-3-small' => 1536,
           'text-embedding-3-large' => 3072
         }.freeze
-        # OpenAI embedding models share an 8191-token input cap across
+        # Conservatively chunk below OpenAI's 8192-token input limit across
         # text-embedding-3-small / -3-large / ada-002. The chunker uses
-        # this as a hard ceiling — the actual chunk size lands well
+        # 8191 as its ceiling — the actual chunk size lands well
         # below it once chars-per-token estimation and the prefix
         # allowance are factored in (see Builder#build_chunker).
         MAX_INPUT_TOKENS = 8191
 
+        # The API accepts at most 2048 inputs and 300,000 total tokens per
+        # request, with 8192 tokens per valid input. At most 36 inputs keeps
+        # even maximum-size texts below both limits without token estimates.
+        # https://developers.openai.com/api/reference/resources/embeddings/methods/create
+        MAX_REQUEST_INPUTS = 36
+        private_constant :MAX_REQUEST_INPUTS
+
         # @param api_key [String] OpenAI API key
         # @param model [String] OpenAI embedding model name (default: text-embedding-3-small)
         # @param dimensions [Integer, nil] Requested output size for text-embedding-3 models
@@ -57,7 +64,7 @@ module Woods
           vectors.first
         end
 
-        # Embed multiple texts in a single request.
+        # Embed multiple texts in bounded requests, preserving input order.
         #
         # Sorts results by the index field to guarantee ordering matches input.
         #
@@ -71,8 +78,12 @@ module Woods
             raise ArgumentError, 'embed_batch(texts) rejects nil/empty entries (OpenAI returns 400)'
           end
 
-          response = post_request(request_body(texts))
-          extract_validated_batch(response, texts.size)
+          vectors = texts.each_slice(MAX_REQUEST_INPUTS).flat_map do |slice|
+            response = post_request(request_body(slice))
+            extract_validated_batch(response, slice.size)
+          end
+          VectorValidation.validate!(vectors, expected_count: texts.size, provider: 'OpenAI')
+          vectors
         end
 
         # Return the dimensionality of vectors produced by this model.

data/lib/woods/evaluation/ablation_executor.rb

@@ -14,10 +14,14 @@ module Woods
       # @return [Integer, nil] the pid of the most recently spawned process
       attr_reader :pid
 
+      # @return [Integer, nil] process group owned by the current command
+      attr_reader :process_group_id
+
       # @param command [String] a full shell command line
       # @param chdir [String]
       # @return [Array(String, String, Boolean)] stdout, stderr, success
       def call(command, chdir:)
+        @pid = @process_group_id = nil
         stdout_r, stdout_w = IO.pipe
         stderr_r, stderr_w = IO.pipe
         spawn_child(command, chdir, stdout_w, stderr_w)
@@ -59,7 +63,8 @@ module Woods
       # terminated on timeout.
       def spawn_child(command, chdir, stdout_w, stderr_w)
         Thread.handle_interrupt(Timeout::Error => :never) do
-          @pid = Process.spawn(command, chdir: chdir, in: File::NULL, out: stdout_w, err: stderr_w)
+          @pid = Process.spawn(command, chdir: chdir, in: File::NULL, out: stdout_w, err: stderr_w, pgroup: true)
+          @process_group_id = @pid
         end
       end
     end

data/lib/woods/evaluation/ablation_timed_executor.rb

@@ -16,7 +16,10 @@ module Woods
     # subprocess started by the wrapped executor keeps running unless
     # something kills it. When the wrapped executor exposes its most recent
     # pid (see {AblationExecutor}), this sends TERM, waits briefly, then KILL
-    # if the process is still alive, before reporting the call as a timed-out
+    # if still alive. The default executor owns a process group, so cleanup
+    # includes ordinary descendants even if their parent already exited.
+    # Custom executors exposing only a pid retain single-process cleanup.
+    # Cleanup finishes before reporting the call as a timed-out
     # failure. An executor that does not expose a pid (for example a fake
     # executor in a spec) still gets the timeout, just without a process to
     # terminate.
@@ -47,15 +50,30 @@ module Woods
         pid = executor_pid
         return '' unless pid
 
-        Process.kill('TERM', pid)
-        Process.kill('KILL', pid) unless process_exited?(pid, within: TERM_GRACE_SECONDS)
+        target = signal_target(pid)
+        signal('TERM', target)
+        signal('KILL', target) unless process_exited?(target, within: TERM_GRACE_SECONDS)
         reap(pid, within: REAP_GRACE_SECONDS) ? '' : " (pid #{pid} did not reap within #{REAP_GRACE_SECONDS}s)"
       rescue Errno::ESRCH, Errno::ECHILD
         ''
       end
 
       def executor_pid
-        @executor.pid if @executor.respond_to?(:pid)
+        pid = @executor.pid if @executor.respond_to?(:pid)
+        pid if pid.is_a?(Integer) && pid.positive?
+      end
+
+      # A group created by spawn(pgroup: true) is named after that child.
+      # Never infer a group from a custom pid or signal our own group.
+      def signal_target(pid)
+        group = @executor.process_group_id if @executor.respond_to?(:process_group_id)
+        group == pid && group != Process.getpgrp ? -group : pid
+      end
+
+      def signal(name, target)
+        Process.kill(name, target)
+      rescue Errno::ESRCH
+        nil
       end
 
       def process_exited?(pid, within:)

data/lib/woods/export/typed_reader.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'woods/mcp/index_reader'
+
+module Woods
+  module Export
+    # Typed reads for export consumers. Published identifiers remain unchanged.
+    class TypedReader
+      def initialize(reader)
+        @reader = reader
+      end
+
+      # Refuse missing or mismatched identities instead of exporting a sibling.
+      def find(identifier, type)
+        unit = @reader.find_unit(identifier, type: type)
+        unless unit.is_a?(Hash) && unit['identifier'] == identifier && unit['type'] == type
+          raise ExtractionError, "export unit missing or mismatched: #{type}:#{identifier}"
+        end
+
+        unit
+      rescue ArgumentError => e
+        raise ExtractionError, "export reader requires typed lookup: #{e.message}"
+      rescue IOError, SystemCallError, JSON::ParserError => e
+        raise ExtractionError, "export unit unreadable: #{type}:#{identifier}: #{e.message}"
+      end
+
+      # IndexReader validates every published artifact, including mixed buckets.
+      # The fallback supports injected readers with the documented list/find API.
+      def all(only: nil)
+        units = @reader.respond_to?(:each_unit) ? @reader.each_unit : injected_units(only)
+        units.select { |unit| !only || only.include?(unit['type']) }
+      rescue IOError, SystemCallError, JSON::ParserError => e
+        raise ExtractionError, "export index incomplete: #{e.message}"
+      end
+
+      private
+
+      def injected_units(only)
+        MCP::IndexReader::UNIT_TYPES_BY_DIR.flat_map do |dir, types|
+          next [] if only && (types & only).empty?
+
+          @reader.list_units(type: MCP::IndexReader::DIR_TO_TYPE.fetch(dir)).map do |entry|
+            find(entry['identifier'], entry_type(entry, types))
+          end
+        end
+      end
+
+      def entry_type(entry, types)
+        type = entry['type'] || (types.first if types.one?)
+        return type if types.include?(type)
+
+        raise ExtractionError, "export entry requires actual type: #{entry['identifier']}"
+      end
+    end
+  end
+end