woods 2.0.0.beta1 → 2.0.0.beta3

data/lib/woods/dependency_graph.rb

@@ -32,9 +32,14 @@ module Woods
     # enforce_dependencies: package units (Task 7)
     # commit_count, change_frequency: git enrichment (Task 5)
     NODE_ATTRIBUTE_KEYS = %i[
-      database table foreign_key_tables package enforce_dependencies commit_count change_frequency
+      database table foreign_key_tables package enforce_dependencies commit_count change_frequency kind
     ].freeze
 
+    # These extractors describe a whole source file rather than one constant.
+    # Other types remain unclassified: a path-like identifier is not evidence
+    # that a unit is a profile, and an unmarked unit need not name a constant.
+    FILE_PROFILE_TYPES = %i[caching configuration test_mapping rails_source gem_source].freeze
+
     # Optional edge keys beyond target and via. `through` is the through
     # association name; `through_db` is the through model's resolved
     # database, emitted only when present; `disable_joins` is emitted only
@@ -80,14 +85,14 @@ module Woods
       }.merge(node_attributes_from(unit))
 
       (@edges[unit.identifier] ||= {})[unit.type] =
-        unit.dependencies.map { |d| { target: d[:target], via: d[:via] }.merge(self.class.edge_attributes(d)) }
+        self.class.normalize_edges(unit.dependencies, strict: true)
       (@file_map[unit.file_path] ||= Set.new).add(unit.identifier) if unit.file_path
 
       # Type index for filtering (Set-based for O(1) insert)
       (@type_index[unit.type] ||= Set.new).add(unit.identifier)
 
       # Build reverse edges (Set-based for O(1) insert)
-      unit.dependencies.each do |dep|
+      @edges[unit.identifier][unit.type].each do |dep|
         (@reverse[dep[:target]] ||= Set.new).add(unit.identifier)
         (@reverse_via[[dep[:target], dep[:via]]] ||= Set.new).add(unit.identifier)
       end
@@ -645,7 +650,7 @@ module Woods
     # @return [Hash{Symbol => Object}] normalized, nil-free
     def node_attributes_from(unit)
       metadata = unit.respond_to?(:metadata) && unit.metadata.is_a?(Hash) ? unit.metadata : {}
-      attrs = {}
+      attrs = self.class.file_profile_attributes(unit.type)
       attrs[:database] = metadata[:database] unless metadata[:database].nil?
       attrs[:table] = metadata[:table_name] if unit.type == :model && !metadata[:table_name].nil?
       tables = Array(metadata[:foreign_keys]).filter_map { |fk| fk[:to_table] || fk['to_table'] if fk.is_a?(Hash) }
@@ -729,14 +734,15 @@ module Woods
     # `nodes` and `edges` stay keyed on the bare identifier, holding the
     # **primary** node — the type that sorts first. Identifiers registered
     # under more than one type put their remaining nodes in `variants`, a flat
-    # array that is **omitted entirely when empty**. So the serialized form of
-    # a graph with no collisions is byte-for-byte what it has always been, and
-    # a `dependency_graph.json` written before this change loads through
+    # array that is **omitted entirely when empty**. A
+    # `dependency_graph.json` written before this change loads through
     # {.from_h} unmodified: it simply carries no `variants`.
     #
     # `reverse`, `file_map` and `type_index` need no new shape. They are
     # already identifier-valued sets, so a Scenic view `reports` and a factory
     # `reports` each contribute to their own type bucket and their own path.
+    # The additive `reverse_via` index preserves typed source identities and
+    # complete relationship records without changing the legacy `reverse` map.
     #
     # @return [Hash] Complete graph data
     def to_h
@@ -747,6 +753,7 @@ module Woods
           nodes: key_sorted(self.class.relativize_nodes(primary_nodes, root)),
           edges: key_sorted(primary_edges),
           reverse: key_sorted(@reverse.transform_values { |ids| ids.to_a.sort }),
+          reverse_via: published_reverse_via,
           file_map: key_sorted(self.class.relativize_file_map(@file_map, root)),
           type_index: key_sorted(@type_index.transform_values { |ids| ids.to_a.sort }),
           stats: {
@@ -770,6 +777,23 @@ module Woods
     end
     private :key_sorted
 
+    # Reverse buckets are derived from every typed forward registration, rather
+    # than the bare-name in-memory filter index, which cannot preserve variants
+    # or distinguish association attributes. Legacy nil relationships stay nil.
+    def published_reverse_via
+      buckets = {}
+      @edges.each do |source, by_type|
+        by_type.each do |type, edges|
+          edges.each do |edge|
+            record = { source: source, source_type: type }.merge(edge.except(:target))
+            (buckets[edge[:target]] ||= []) << record
+          end
+        end
+      end
+      key_sorted(buckets.transform_values { |records| records.sort_by { |record| JSON.generate(record) } })
+    end
+    private :published_reverse_via
+
     # A copy whose edge containers are the caller's alone.
     #
     # `dup` copies only the top level, so `to_h[:edges][id]` used to be the
@@ -783,6 +807,9 @@ module Woods
     def detached_snapshot(memo)
       snapshot = memo.dup
       snapshot[:edges] = memo[:edges].transform_values { |list| list.map(&:dup) }
+      snapshot[:reverse_via] = memo[:reverse_via].transform_values do |records|
+        records.map { |record| record.transform_values { |value| value.is_a?(String) ? value.dup : value } }
+      end
       if memo.key?(:variants)
         snapshot[:variants] = memo[:variants].map do |record|
           record.merge(edges: record[:edges].map(&:dup))
@@ -794,7 +821,12 @@ module Woods
 
     # @return [Hash{String => Hash}] identifier => primary node
     def primary_nodes
-      @nodes.transform_values { |nodes| primary_of(nodes) }
+      # Late enrichment and JSON loading can insert optional fields in a
+      # different order. Canonicalize like variant_records so a graph's JSON
+      # fingerprint is stable across full and incremental publication.
+      @nodes.transform_values do |nodes|
+        primary_of(nodes).slice(:type, :file_path, :namespace, *NODE_ATTRIBUTE_KEYS)
+      end
     end
     private :primary_nodes
 
@@ -969,10 +1001,17 @@ module Woods
     # @param node [Hash]
     # @return [Hash{Symbol => Object}]
     def self.persisted_node_attributes(node)
-      NODE_ATTRIBUTE_KEYS.each_with_object({}) do |key, attrs|
+      attributes = NODE_ATTRIBUTE_KEYS.each_with_object({}) do |key, attrs|
         value = node.key?(key) ? node[key] : node[key.to_s]
         attrs[key] = normalize_node_attribute(key, value) unless value.nil?
       end
+      attributes.merge(file_profile_attributes(node[:type] || node['type']))
+    end
+
+    # Backfill the additive marker when an older graph is republished, so
+    # unchanged nodes in an incremental run agree with a fresh full graph.
+    def self.file_profile_attributes(type)
+      FILE_PROFILE_TYPES.include?(type&.to_sym) ? { kind: 'file_profile' } : {}
     end
 
     # Edge attributes present in a dependency or persisted edge hash.
@@ -1032,7 +1071,10 @@ module Woods
       graph.instance_variable_set(:@edges, edges)
 
       raw_reverse = data[:reverse] || data['reverse'] || {}
-      graph.instance_variable_set(:@reverse, raw_reverse.transform_values { |v| v.is_a?(Set) ? v : Set.new(v) })
+      reverse = raw_reverse.each_with_object({}) do |(target, sources), index|
+        (index[target.to_s] ||= Set.new).merge(sources)
+      end
+      graph.instance_variable_set(:@reverse, reverse)
 
       raw_file_map = data[:file_map] || data['file_map'] || {}
       graph.instance_variable_set(:@file_map, absolutize_file_map(normalize_file_map(raw_file_map), root))
@@ -1126,7 +1168,12 @@ module Woods
       }.merge(persisted_node_attributes(node))
     end
 
-    # Normalize edge data from either old format (bare strings) or new format (hashes).
+    # Normalize fresh dependencies and persisted edges to the same identity:
+    # string targets and symbol relationship labels. Extractors may emit
+    # symbolic external targets (such as :http_api); retaining those symbols
+    # after a JSON-loaded baseline splits reverse indexes on re-registration.
+    # Unit dependency metadata stays unchanged; only graph records normalize.
+    # Also accepts the old bare-string edge format.
     #
     # ROUND-TRIP INVARIANT (do not break when refactoring):
     #   DependencyGraph#to_h -> JSON.generate -> JSON.parse -> DependencyGraph.from_h
@@ -1147,15 +1194,20 @@ module Woods
     # explicit data conversion.
     #
     # @param edges [Array] Edge entries — either strings or hashes
+    # @param strict [Boolean] require hashes for fresh units; legacy loads stay tolerant
     # @return [Array<Hash>] Normalized edges with :target and :via keys
-    def self.normalize_edges(edges)
+    def self.normalize_edges(edges, strict: false)
+      raise ArgumentError, 'Fresh graph dependencies must be an array' if strict && !edges.is_a?(Array)
+
       return [] unless edges.is_a?(Array)
 
       edges.map do |edge|
+        raise ArgumentError, 'Fresh graph dependencies must be hashes' if strict && !edge.is_a?(Hash)
+
         if edge.is_a?(String)
           { target: edge, via: nil }
         elsif edge.is_a?(Hash)
-          { target: edge[:target] || edge['target'], via: (edge[:via] || edge['via'])&.to_sym }
+          { target: (edge[:target] || edge['target'])&.to_s, via: (edge[:via] || edge['via'])&.to_sym }
             .merge(edge_attributes(edge))
         else
           { target: edge.to_s, via: nil }

data/lib/woods/embedding/corpus.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../atomic_file'
+require_relative '../generation'
+
+module Woods
+  class Error < StandardError; end unless defined?(Woods::Error)
+
+  module Embedding
+    # Collect the complete extraction input before an embedding run can mutate
+    # stores. Native publications use their authoritative listings, never a glob
+    # that could mistake a missing/corrupt payload for intentional deletion.
+    class Corpus
+      class Incomplete < Woods::Error; end
+
+      def initialize(output_dir)
+        @output_dir = output_dir.to_s
+      end
+
+      def load
+        native? ? published_units : legacy_units
+      rescue IOError, SystemCallError, JSON::ParserError, EncodingError, ArgumentError => e
+        raise Incomplete, "Embedding input incomplete: #{e.message}. Repair or rebuild extraction before embedding."
+      end
+
+      private
+
+      def native?
+        path = File.join(@output_dir, Generation::FILENAME)
+        if File.exist?(path)
+          marker = JSON.parse(AtomicFile.read(path))
+          raise IOError, 'invalid generation marker' unless marker.is_a?(Hash)
+        end
+
+        if !marker || marker['payload'].nil?
+          if File.directory?(File.join(@output_dir, 'payloads'))
+            raise IOError, 'missing publication pointer beside payloads'
+          end
+
+          return false
+        end
+
+        validate_marker(marker)
+        true
+      end
+
+      def validate_marker(marker)
+        strings_valid = %w[payload token].all? { |key| marker[key].is_a?(String) && !marker[key].empty? }
+        return if strings_valid && marker['number'].is_a?(Integer) && marker['number'].positive?
+
+        raise IOError, 'invalid published generation marker'
+      end
+
+      def published_units
+        require_relative '../mcp/index_reader'
+
+        reader = MCP::IndexReader.new(@output_dir)
+        reader.with_pinned_generation do
+          # Generation's compatibility fallback to the root is useful for
+          # readers, but cannot prove completeness for destructive reconciliation.
+          if reader.payload_dir.expand_path == Pathname.new(@output_dir).expand_path
+            raise IOError,
+                  'published payload missing or invalid'
+          end
+
+          validate_counts(reader.manifest)
+          reader.each_unit.to_a
+        end
+      end
+
+      def validate_counts(manifest)
+        counts = manifest.is_a?(Hash) && manifest['counts']
+        valid = counts.is_a?(Hash) && counts.all? do |dir, count|
+          MCP::IndexReader::TYPE_DIRS.include?(dir) && count.is_a?(Integer) && count >= 0
+        end
+        raise IOError, 'invalid published manifest counts' unless valid
+      end
+
+      # Pre-pointer indexes may use arbitrary filenames and have no manifest
+      # or authoritative listing. Keep that input contract, but never silently
+      # omit unreadable JSON before reconciling persisted identities.
+      def legacy_units
+        Dir.glob(File.join(@output_dir, '**', '*.json')).filter_map do |path|
+          relative = path.delete_prefix("#{@output_dir}/")
+          next if relative.start_with?('dumps/', 'payloads/') || File.basename(path) == 'checkpoint.json'
+
+          data = JSON.parse(AtomicFile.read(path))
+          data if data.is_a?(Hash) && data.key?('type') && data.key?('identifier')
+        end
+      end
+    end
+  end
+end

data/lib/woods/embedding/indexer.rb

@@ -119,40 +119,14 @@ module Woods
 
       private
 
-      # Where this run's units are read from — the published generation's
-      # payload, or the output root for a flat index.
-      #
-      # Globbing the output root would walk `payloads/` and ingest every
-      # retained generation, so each unit would be embedded once per retained
-      # payload. It also picks up `dumps/`, which holds no unit JSON but is
-      # needlessly large to walk.
-      #
-      # @return [String]
-      def units_dir
-        Woods::Generation.new(output_dir: @output_dir).payload_dir.to_s
-      end
-
       def load_units
-        Dir.glob(File.join(units_dir, '**', '*.json')).filter_map do |path|
-          next if File.basename(path) == 'checkpoint.json'
-
-          # AtomicFile.read, not File.read: a bare read tags the bytes with the
-          # process's default external encoding (US-ASCII under LANG=C), so a
-          # single multibyte character in one unit raised EncodingError out of
-          # JSON.parse and aborted the whole embed run.
-          data = JSON.parse(AtomicFile.read(path))
-          # Extraction output also contains index listings (_index.json arrays) and
-          # summary files (manifest.json, dependency_graph.json, graph_analysis.json)
-          # that live alongside per-unit JSON. Filter to the unit shape.
-          data if data.is_a?(Hash) && data.key?('type') && data.key?('identifier')
-        rescue JSON::ParserError, EncodingError => e
-          warn "[woods] skipping unreadable unit file #{path} (#{e.class}: #{e.message})"
-          nil
-        end
+        require_relative 'corpus'
+
+        Corpus.new(@output_dir).load
       end
 
-      # The invariant: **checkpoint.json never advances over a unit whose
-      # vector was not durably stored.** Two things uphold it here.
+      # The invariant: **checkpoint.json advances only after the intended
+      # vector state (including an empty set) is durable.** Two things uphold it here.
       #
       # 1. Ordering. For a store whose only durable copy is the dump
       #    (+persistable?+), the checkpoint is written *after*
@@ -170,17 +144,19 @@ module Woods
       #
       # 2. Trust, verified. {#checkpoint_satisfied?} honours a checkpoint hit
       #    only when the durable artifact actually holds a vector for that
-      #    unit. A checkpoint that ran ahead of its dump — an older gem with
+      #    unit, or preparation verifies that it intentionally has no text.
+      #    A checkpoint that ran ahead of its dump — an older gem with
       #    this bug, an interrupted promote, a store swap — self-heals into a
       #    re-embed instead of stranding the unit forever.
       def process_units(units, incremental:)
         prepare_run(incremental: incremental)
-        units = assign_storage_identities(units)
         checkpoint = incremental ? load_checkpoint : {}
+        units = assign_storage_identities(units, checkpoint: checkpoint)
         stats = { processed: 0, skipped: 0, errors: 0 }
 
         embed_batches(units, checkpoint, stats, incremental: incremental)
 
+        reconcile_empty_units(checkpoint)
         retire_legacy_identities
         report_checkpoint_misses
         vanished = incremental && persistable? ? drop_vanished_units : 0
@@ -196,12 +172,12 @@ module Woods
       end
 
       # Unambiguous existing keys stay stable. A collision uses reversible typed keys.
-      def assign_storage_identities(units)
+      def assign_storage_identities(units, checkpoint:)
         counts = units.group_by { |unit| unit['identifier'] }.transform_values(&:size)
         units.map do |unit|
           id = unit['identifier']
           typed = StorageIdentity.key(id, unit['type'])
-          existing = known_storage_key?(typed)
+          existing = known_storage_key?(typed, checkpoint: checkpoint)
           next unit unless counts[id] > 1 || existing || id.start_with?(StorageIdentity::PREFIX)
 
           unit.merge('storage_id' => typed)
@@ -224,8 +200,8 @@ module Woods
         end
       end
 
-      def known_storage_key?(key)
-        (@persisted_ids || {}).key?(key) || (@durable_ids || {}).key?(key)
+      def known_storage_key?(key, checkpoint:)
+        (@persisted_ids || {}).key?(key) || (@durable_ids || {}).key?(key) || checkpoint.key?(key)
       end
 
       def retire_legacy_key(legacy)
@@ -243,11 +219,9 @@ module Woods
       # `woods:embed_incremental` runs evict every genuinely older dump in
       # favour of copies of the same state.
       #
-      # Safe because nothing else mutates the *vector* store on a zero-processed
-      # run: `prune_superseded_vectors` is reached only from `store_vectors`,
-      # which runs only for items that were actually embedded. A checkpoint
-      # self-heal counts as processed, so a run that re-embeds a stranded unit
-      # still dumps.
+      # A zero-text transition can retire vectors without a provider call;
+      # @vectors_changed captures that case. Checkpoint self-heals still count
+      # as processed, so a run re-embedding a stranded unit also dumps.
       #
       # But "nothing embedded" is not "nothing changed" (B-069). `persist_snapshot`
       # writes the vector dump, the *metadata* dump and the config into one
@@ -266,6 +240,11 @@ module Woods
       # `@current_identifiers` is what this run saw. Anything in the first and
       # not the second is stale, and the dump has to be rewritten to drop it.
       #
+      # Existing units can also change metadata without changing source (B-119).
+      # Compare their complete records with the promoted metadata snapshot;
+      # publishing those changes must not require a provider call or invalidate
+      # the source-hash checkpoint contract.
+      #
       # Full runs always dump: they rebuild the store from scratch, so "nothing
       # processed" there means the store is genuinely empty and the dump must
       # say so rather than leave a stale one promoted.
@@ -274,7 +253,7 @@ module Woods
       def snapshot_worth_writing?(stats, vanished, incremental:)
         return true unless incremental
 
-        stats[:processed].positive? || vanished.positive?
+        stats[:processed].positive? || vanished.positive? || @metadata_changed || @vectors_changed
       end
 
       # Fraction of the persisted units the vanished-unit sweep may remove
@@ -489,10 +468,36 @@ module Woods
         @current_identifiers = Set.new
         @durable_ids = nil
         @checkpoint_misses = 0
-        hydrate_persisted_vectors if incremental && persistable?
+        @metadata_changed = false
+        @vectors_changed = false
+        @empty_units = {}
+        @persisted_metadata = nil
+        prepare_snapshot_stores(incremental: incremental)
+        retain_metadata_identities
         load_durable_store_ids if reconcilable?
       end
 
+      def prepare_snapshot_stores(incremental:)
+        return unless persistable?
+
+        if incremental
+          hydrate_persisted_metadata
+          hydrate_persisted_vectors
+        else
+          # A direct caller may reuse an in-memory adapter for a full run.
+          # Its old chunks still need replacement, including by an empty set.
+          entries = []
+          @vector_store.each_entry { |id, _vector, _metadata| entries << { id: id } }
+          @persisted_ids = index_ids_by_identifier(entries)
+        end
+      end
+
+      # Source-empty units retain metadata but intentionally have no vectors.
+      # Keep those identities in the same deletion and typed-key accounting.
+      def retain_metadata_identities
+        @persisted_metadata&.each_entry { |identifier, _unit| @persisted_ids[identifier] ||= [] }
+      end
+
       # Read back what the durable store currently holds, as base identifiers.
       #
       # One enumeration serves both halves of #211:
@@ -618,7 +623,10 @@ module Woods
         # enumeration that failed) — fall back to trusting the checkpoint.
         return true if known_ids.nil?
 
-        return true if known_ids.key?(storage_id(unit_data))
+        return true if known_ids[storage_id(unit_data)]&.any?
+        # A source-empty unit intentionally has no vector. Verify that state
+        # again rather than treating a missing nonempty vector as a cache hit.
+        return true if prepare_texts(unit_data).empty?
 
         @checkpoint_misses += 1
         false
@@ -635,7 +643,22 @@ module Woods
       def persist_unit_metadata(unit_data)
         return unless @metadata_store
 
-        @metadata_store.store(storage_id(unit_data), unit_data)
+        id = storage_id(unit_data)
+        @metadata_changed ||= @persisted_metadata && @persisted_metadata.find(id) != unit_data
+        @metadata_store.store(id, unit_data)
+      end
+
+      # Compare with the promoted artifact, not a fresh metadata store or the
+      # source-only checkpoint. Retain old records until the guarded vanished
+      # sweep permits their deletion, just as vector hydration does.
+      def hydrate_persisted_metadata
+        return unless @metadata_store.respond_to?(:each_entry) && @metadata_store.respond_to?(:bulk_load)
+
+        require_relative '../index_artifact'
+        require_relative '../storage/snapshotter'
+
+        @persisted_metadata = Storage::Snapshotter::Metadata.load_or_empty(IndexArtifact.new(@output_dir))
+        @metadata_store.bulk_load(@persisted_metadata.each_entry)
       end
 
       # Refuse to index a unit whose real identifier already matches the
@@ -653,6 +676,7 @@ module Woods
       def collect_embed_items(unit_data, items)
         texts = prepare_texts(unit_data)
         identifier = storage_id(unit_data)
+        @empty_units[identifier] = unit_data['source_hash'] if texts.empty?
 
         texts.each_with_index do |text, idx|
           embed_id = texts.length > 1 ? "#{identifier}#chunk_#{idx}" : identifier
@@ -661,6 +685,26 @@ module Woods
         end
       end
 
+      # Defer zero-text deletion until every batch has prepared/embedded
+      # successfully. A later provider failure must not retire earlier vectors.
+      # Checkpoints retain their source-hash format; a matching no-vector hash
+      # is trusted only after verifying the current input still prepares empty.
+      def reconcile_empty_units(checkpoint)
+        verify_empty_reconciliation!
+        @empty_units.each do |identifier, source_hash|
+          @vectors_changed ||= @persisted_ids[identifier]&.any?
+          prune_identifier(identifier, []) if implements_own?(@vector_store, :delete)
+          prune_durable_identifier(identifier, []) if @durable_ids && implements_own?(@vector_store, :delete)
+          checkpoint[identifier] = source_hash
+        end
+      end
+
+      def verify_empty_reconciliation!
+        return unless @empty_units.any? && reconcilable? && @durable_ids.nil?
+
+        raise Woods::Error, 'Cannot reconcile source-empty units: existing vector IDs could not be read'
+      end
+
       def prepare_texts(unit_data) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         unit = build_unit(unit_data)
         apply_chunking(unit) if @chunker && unit.chunks.empty? && needs_chunking?(unit)

data/lib/woods/embedding/openai.rb

@@ -16,7 +16,7 @@ module Woods
       #   provider = Woods::Embedding::Provider::OpenAI.new(api_key: ENV['OPENAI_API_KEY'])
       #   vector = provider.embed("class User < ApplicationRecord; end")
       #   vectors = provider.embed_batch(["text1", "text2"])
-      class OpenAI
+      class OpenAI # rubocop:disable Metrics/ClassLength
         include Interface
         include DiscardableClient
 
@@ -26,13 +26,20 @@ module Woods
           'text-embedding-3-small' => 1536,
           'text-embedding-3-large' => 3072
         }.freeze
-        # OpenAI embedding models share an 8191-token input cap across
+        # Conservatively chunk below OpenAI's 8192-token input limit across
         # text-embedding-3-small / -3-large / ada-002. The chunker uses
-        # this as a hard ceiling — the actual chunk size lands well
+        # 8191 as its ceiling — the actual chunk size lands well
         # below it once chars-per-token estimation and the prefix
         # allowance are factored in (see Builder#build_chunker).
         MAX_INPUT_TOKENS = 8191
 
+        # The API accepts at most 2048 inputs and 300,000 total tokens per
+        # request, with 8192 tokens per valid input. At most 36 inputs keeps
+        # even maximum-size texts below both limits without token estimates.
+        # https://developers.openai.com/api/reference/resources/embeddings/methods/create
+        MAX_REQUEST_INPUTS = 36
+        private_constant :MAX_REQUEST_INPUTS
+
         # @param api_key [String] OpenAI API key
         # @param model [String] OpenAI embedding model name (default: text-embedding-3-small)
         # @param dimensions [Integer, nil] Requested output size for text-embedding-3 models
@@ -57,7 +64,7 @@ module Woods
           vectors.first
         end
 
-        # Embed multiple texts in a single request.
+        # Embed multiple texts in bounded requests, preserving input order.
         #
         # Sorts results by the index field to guarantee ordering matches input.
         #
@@ -71,8 +78,12 @@ module Woods
             raise ArgumentError, 'embed_batch(texts) rejects nil/empty entries (OpenAI returns 400)'
           end
 
-          response = post_request(request_body(texts))
-          extract_validated_batch(response, texts.size)
+          vectors = texts.each_slice(MAX_REQUEST_INPUTS).flat_map do |slice|
+            response = post_request(request_body(slice))
+            extract_validated_batch(response, slice.size)
+          end
+          VectorValidation.validate!(vectors, expected_count: texts.size, provider: 'OpenAI')
+          vectors
         end
 
         # Return the dimensionality of vectors produced by this model.

data/lib/woods/evaluation/ablation_executor.rb

@@ -14,10 +14,14 @@ module Woods
       # @return [Integer, nil] the pid of the most recently spawned process
       attr_reader :pid
 
+      # @return [Integer, nil] process group owned by the current command
+      attr_reader :process_group_id
+
       # @param command [String] a full shell command line
       # @param chdir [String]
       # @return [Array(String, String, Boolean)] stdout, stderr, success
       def call(command, chdir:)
+        @pid = @process_group_id = nil
         stdout_r, stdout_w = IO.pipe
         stderr_r, stderr_w = IO.pipe
         spawn_child(command, chdir, stdout_w, stderr_w)
@@ -59,7 +63,8 @@ module Woods
       # terminated on timeout.
       def spawn_child(command, chdir, stdout_w, stderr_w)
         Thread.handle_interrupt(Timeout::Error => :never) do
-          @pid = Process.spawn(command, chdir: chdir, in: File::NULL, out: stdout_w, err: stderr_w)
+          @pid = Process.spawn(command, chdir: chdir, in: File::NULL, out: stdout_w, err: stderr_w, pgroup: true)
+          @process_group_id = @pid
         end
       end
     end

data/lib/woods/evaluation/ablation_timed_executor.rb

@@ -16,7 +16,10 @@ module Woods
     # subprocess started by the wrapped executor keeps running unless
     # something kills it. When the wrapped executor exposes its most recent
     # pid (see {AblationExecutor}), this sends TERM, waits briefly, then KILL
-    # if the process is still alive, before reporting the call as a timed-out
+    # if still alive. The default executor owns a process group, so cleanup
+    # includes ordinary descendants even if their parent already exited.
+    # Custom executors exposing only a pid retain single-process cleanup.
+    # Cleanup finishes before reporting the call as a timed-out
     # failure. An executor that does not expose a pid (for example a fake
     # executor in a spec) still gets the timeout, just without a process to
     # terminate.
@@ -47,15 +50,30 @@ module Woods
         pid = executor_pid
         return '' unless pid
 
-        Process.kill('TERM', pid)
-        Process.kill('KILL', pid) unless process_exited?(pid, within: TERM_GRACE_SECONDS)
+        target = signal_target(pid)
+        signal('TERM', target)
+        signal('KILL', target) unless process_exited?(target, within: TERM_GRACE_SECONDS)
         reap(pid, within: REAP_GRACE_SECONDS) ? '' : " (pid #{pid} did not reap within #{REAP_GRACE_SECONDS}s)"
       rescue Errno::ESRCH, Errno::ECHILD
         ''
       end
 
       def executor_pid
-        @executor.pid if @executor.respond_to?(:pid)
+        pid = @executor.pid if @executor.respond_to?(:pid)
+        pid if pid.is_a?(Integer) && pid.positive?
+      end
+
+      # A group created by spawn(pgroup: true) is named after that child.
+      # Never infer a group from a custom pid or signal our own group.
+      def signal_target(pid)
+        group = @executor.process_group_id if @executor.respond_to?(:process_group_id)
+        group == pid && group != Process.getpgrp ? -group : pid
+      end
+
+      def signal(name, target)
+        Process.kill(name, target)
+      rescue Errno::ESRCH
+        nil
       end
 
       def process_exited?(pid, within:)