woods 2.0.0 → 2.1.0

data/lib/woods/chunking/contributor_chunks.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require_relative 'chunk'
+require_relative '../source_contributors'
+
+module Woods
+  module Chunking
+    # Exact physical attribution for contiguous slices of validated contributors.
+    # Generated composite headers and cross-file content never get a physical span.
+    module ContributorChunks
+      module_function
+
+      def chunks(unit)
+        SourceContributors.records(unit).each_with_index.map do |record, index|
+          first = record.fetch('published_start_byte')
+          last = record.fetch('published_end_byte')
+          Chunk.new(content: unit.source_code.byteslice(first...last), chunk_type: :"contributor_#{index}",
+                    parent_identifier: unit.identifier, parent_type: unit.type,
+                    metadata: location(unit, first, last))
+        end
+      end
+
+      # Preserve existing bounded chunks only if they cover every raw byte once,
+      # in contributor order. Caller-supplied ranges never authorize citations.
+      def ensure!(unit)
+        records = SourceContributors.records(unit)
+        return if records.empty?
+
+        unit.chunks = chunks(unit).map(&:to_h) unless complete_coverage?(unit, records)
+        unit.chunks.each do |chunk|
+          range = published_range(chunk)
+          chunk[:metadata] = (chunk[:metadata] || {}).merge(location(unit, range[:start_byte], range[:end_byte]))
+        end
+      end
+
+      def complete_coverage?(unit, records)
+        remaining = unit.chunks.dup
+        records.all? do |record|
+          cursor = record['published_start_byte']
+          while cursor < record['published_end_byte']
+            chunk = remaining.shift
+            return false unless chunk
+
+            span = published_range(chunk)
+            return false unless contiguous?(unit, chunk, span, cursor, record['published_end_byte'])
+
+            cursor = span[:end_byte]
+          end
+          true
+        end && remaining.empty?
+      end
+
+      def contiguous?(unit, chunk, span, cursor, last)
+        span[:start_byte] == cursor && span[:end_byte].is_a?(Integer) &&
+          span[:end_byte] > cursor && span[:end_byte] <= last &&
+          unit.source_code.byteslice(cursor...span[:end_byte]) == chunk[:content]
+      end
+
+      def published_range(chunk)
+        metadata = SourceContributors.field(chunk, :metadata) || {}
+        span = SourceContributors.field(metadata, :published_location)
+        span.is_a?(Hash) ? span.transform_keys(&:to_sym) : {}
+      end
+
+      def location(unit, first, last)
+        physical = SourceContributors.physical_span(unit, start_byte: first, end_byte: last)
+        return {} unless physical
+
+        record = SourceContributors.records(unit).find { |entry| entry['file_path'] == physical[:file_path] }
+        physical = physical.merge(start_byte: first - record['published_start_byte'],
+                                  end_byte: last - record['published_start_byte'])
+        { physical_location: physical, published_location: { start_byte: first, end_byte: last } }
+      end
+
+      # Offsets are relative to this chunk, including for a second splitting pass.
+      def slice_metadata(metadata, source, first, last)
+        metadata = symbol_keys(metadata)
+        physical = symbol_keys(metadata[:physical_location])
+        published = symbol_keys(metadata[:published_location])
+        clean = metadata.except(:physical_location, :published_location)
+        return clean unless mapped_slice?(physical, published, source, first, last)
+
+        clean.merge(physical_location: physical_slice(physical, source, first, last),
+                    published_location: { start_byte: published[:start_byte] + first,
+                                          end_byte: published[:start_byte] + last })
+      end
+
+      def physical_slice(physical, source, first, last)
+        start_line = physical[:start_line] + source.byteslice(0...first).count("\n")
+        end_line = start_line + source.byteslice(first...last).delete_suffix("\n").count("\n")
+        physical.merge(start_byte: physical[:start_byte] + first, end_byte: physical[:start_byte] + last,
+                       start_line: start_line, end_line: end_line)
+      end
+
+      def symbol_keys(value)
+        value.is_a?(Hash) ? value.transform_keys(&:to_sym) : {}
+      end
+
+      def mapped_slice?(physical, published, source, first, last)
+        integers = [physical[:start_byte], physical[:end_byte], physical[:start_line],
+                    published[:start_byte], published[:end_byte]]
+        integers.all?(Integer) &&
+          [physical, published].all? { |span| span[:end_byte] - span[:start_byte] == source.bytesize } &&
+          first >= 0 && last > first && last <= source.bytesize
+      end
+
+      # Re-derive spans from the published source when hydrating a metadata dump.
+      def vector_metadata(unit, chunk_metadata)
+        return {} unless SourceContributors.multiple?(unit)
+
+        span = published_range(metadata: chunk_metadata)
+        facts = location(unit, span[:start_byte], span[:end_byte])
+        { source_paths: SourceContributors.paths(unit),
+          file_path: facts.dig(:physical_location, :file_path) }.merge(facts)
+      end
+      private_class_method :complete_coverage?, :contiguous?, :mapped_slice?, :physical_slice, :symbol_keys
+    end
+  end
+end

data/lib/woods/chunking/semantic_chunker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'chunk'
+require_relative 'contributor_chunks'
 
 module Woods
   module Chunking
@@ -181,11 +182,8 @@ module Woods
       # Default token threshold below which units stay whole.
       DEFAULT_THRESHOLD = 200
 
-      # Minimum chars-per-slice budget during tokenizer-driven recursive
-      # splitting. Prevents unbounded halving on pathological content
-      # (e.g., a single 2000-char regex line that tokenizes into 6000
-      # tokens because BERT WordPiece fragments every `\w+` boundary).
-      MIN_SLICE_CHARS = 256
+      # Stop at one Unicode character and refuse if even that cannot fit.
+      MIN_SLICE_CHARS = 1
       private_constant :MIN_SLICE_CHARS
 
       # @param threshold [Integer] Token count threshold for chunking
@@ -222,7 +220,8 @@ module Woods
       # @return [Array<Chunk>] Ordered list of chunks
       def chunk(unit)
         return [] if unit.source_code.nil? || unit.source_code.strip.empty?
-        return [build_whole_chunk(unit)] if unit.estimated_tokens <= @threshold
+        return enforce_char_limit(ContributorChunks.chunks(unit), unit) if SourceContributors.multiple?(unit)
+        return enforce_char_limit([build_whole_chunk(unit)], unit) if unit.estimated_tokens <= @threshold
 
         enforce_char_limit(chunks_for(unit), unit)
       end
@@ -247,6 +246,11 @@ module Woods
         unit.chunks = unit.chunks.flat_map { |chunk| split_oversize_hash_chunk(chunk) }
       end
 
+      def preparation_identity
+        { 'class' => self.class.name, 'contributors_version' => 1, 'threshold' => @threshold, 'max_chars' => @max_chars,
+          'max_tokens' => @max_tokens, 'counter' => @token_counter&.class&.name }
+      end
+
       private
 
       # True when either the char ceiling or the token-based verifier is
@@ -270,12 +274,20 @@ module Woods
       # @param chunk [Hash] a unit-chunk hash (symbol or string keys)
       # @return [Array<Hash>]
       def split_oversize_hash_chunk(chunk)
-        content = chunk[:content] || chunk['content']
+        chunk = chunk.transform_keys(&:to_sym)
+        content = chunk[:content]
         return [chunk] if content.nil? || !oversize?(content)
 
-        chunk_type = chunk[:chunk_type] || chunk['chunk_type'] || :whole
+        chunk_type = chunk[:chunk_type] || :whole
+        offset = SourceContributors.field(chunk[:embedding_slice], :start_byte) || 0
+        initial_offset = offset
         verified_slices(content).each_with_index.map do |slice, idx|
-          { content: slice, chunk_type: :"#{chunk_type}_part_#{idx}" }
+          first = offset
+          offset += slice.bytesize
+          metadata = ContributorChunks.slice_metadata(chunk[:metadata], content, first - initial_offset,
+                                                      offset - initial_offset)
+          chunk.merge(content: slice, chunk_type: :"#{chunk_type}_part_#{idx}", metadata: metadata,
+                      embedding_slice: { start_byte: first, end_byte: offset })
         end
       end
 
@@ -332,12 +344,16 @@ module Woods
       def split_oversize_chunk(chunk, unit)
         return [chunk] unless oversize?(chunk.content)
 
+        offset = 0
         verified_slices(chunk.content).each_with_index.map do |slice, idx|
+          first = offset
+          offset += slice.bytesize
           Chunk.new(
             content: slice,
             chunk_type: :"#{chunk.chunk_type}_part_#{idx}",
             parent_identifier: unit.identifier,
-            parent_type: unit.type
+            parent_type: unit.type,
+            metadata: ContributorChunks.slice_metadata(chunk.metadata, chunk.content, first, offset)
           )
         end
       end
@@ -359,9 +375,8 @@ module Woods
       end
 
       # Ensure a single post-line-split slice fits the token budget.
-      # Halves the char limit and reslices if it doesn't. Stops at
-      # {MIN_SLICE_CHARS} to avoid unbounded recursion on content that
-      # cannot be split line-wise (minified output, huge regex literals).
+      # Halves the character limit and reslices if necessary. A minimum-sized
+      # slice that still exceeds the counter's limit is refused, never emitted.
       #
       # @param slice [String]
       # @param char_limit [Integer]
@@ -370,7 +385,9 @@ module Woods
         return [slice] unless @token_counter.count(slice) > @max_tokens
 
         smaller = char_limit / 2
-        return [slice] if smaller < MIN_SLICE_CHARS
+        if smaller < MIN_SLICE_CHARS || slice.length <= 1
+          raise ArgumentError, 'Chunk input limit cannot fit one source character'
+        end
 
         slice_by_lines(slice, smaller).flat_map { |sub| verify_slice(sub, smaller) }
       end
@@ -385,13 +402,15 @@ module Woods
       end
 
       # Greedy line-based slicing that respects a supplied `limit`.
-      # Lines longer than `limit` are hard-cut (lossy — but such lines
-      # are already pathological: minified JSON dumps, long regexes).
+      # Lines longer than `limit` are cut at Unicode character boundaries;
+      # concatenating the slices preserves every source character.
       #
       # @param content [String]
       # @param limit [Integer]
       # @return [Array<String>]
       def slice_by_lines(content, limit = @max_chars)
+        raise ArgumentError, 'Chunk size must be positive' unless limit.positive?
+
         slices = []
         current = String.new
         content.each_line do |line|
@@ -658,6 +677,8 @@ module Woods
       include ChunkBuilder
       include LineDepthTracking
 
+      METHOD_NAME = /\A\s*def\s+((?:self\s*\.\s*)?(?:#{OPERATOR_METHOD_NAMES}|[[:word:]]+[?!=]?))/
+
       # @param unit [ExtractedUnit]
       def initialize(unit)
         @unit = unit
@@ -677,7 +698,7 @@ module Woods
       # @return [Hash]
       def parse_lines(lines)
         state = {
-          summary: [], methods: {}, private_methods: [],
+          summary: [], methods: {}, method_occurrences: Hash.new(0), private_methods: [],
           current_method: nil, depth: 0, in_private: false
         }
         lines.each do |line|
@@ -725,14 +746,16 @@ module Woods
       end
 
       def start_method(state, line)
-        method_name = line[/def\s+(?:self\.)?(\w+)/, 1] || operator_def_name(line)
+        method_name = line[METHOD_NAME, 1]&.delete(" \t")
 
         if state[:in_private]
           state[:private_methods] << line
         else
-          # Preserve insertion order — Hash does this by default, but we
-          # initialize the entry here so `build_chunks` below walks
-          # methods in source order.
+          # Repeated definitions (nested owners, singleton bodies, or inlined
+          # overrides) must never overwrite earlier source. Preserve existing
+          # keys for unique methods and number later occurrences in source order.
+          occurrence = state[:method_occurrences][method_name] += 1
+          method_name = "#{method_name}@#{occurrence}" if occurrence > 1
           state[:methods][method_name] = [line]
         end
 

data/lib/woods/console/adapter_family.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Woods
+  module Console
+    # Adapter names differ from the SQL grammar and session settings they use.
+    module AdapterFamily
+      # Classify a live connection by adapter ancestry, database configuration,
+      # then its display name. Unknown families remain nil: SQL callers must
+      # refuse rather than silently omit dialect-specific safeguards.
+      # @param connection [Object] Active Record connection or compatible adapter
+      # @return [Symbol, nil] :postgres, :mysql, :sqlite, or unknown
+      def self.for(connection)
+        ancestry = connection.class.ancestors.filter_map(&:name)
+        return :postgres if ancestry.include?('ActiveRecord::ConnectionAdapters::PostgreSQLAdapter')
+        return :mysql if ancestry.any? { |name| name.match?(/::(?:AbstractMysql|Mysql2|Trilogy)Adapter\z/) }
+        return :sqlite if ancestry.include?('ActiveRecord::ConnectionAdapters::SQLite3Adapter')
+
+        from_name(configured_adapter(connection)) || from_name(connection.adapter_name)
+      end
+
+      def self.configured_adapter(connection)
+        return unless connection.respond_to?(:pool) && connection.pool.respond_to?(:db_config)
+
+        connection.pool.db_config.adapter
+      end
+      private_class_method :configured_adapter
+
+      def self.from_name(value)
+        name = value.to_s.downcase
+        return :mysql if name.include?('mysql') || %w[trilogy mariadb].include?(name)
+        return :postgres if name.include?('postgre') || %w[postgis cockroachdb redshift].include?(name)
+        return :sqlite if name.include?('sqlite')
+
+        nil
+      end
+      private_class_method :from_name
+    end
+  end
+end

data/lib/woods/console/connection_manager.rb

@@ -17,6 +17,12 @@ module Woods
     # process with a direct, Docker, or SSH command.
     class ConnectionManager
       DEFAULT_COMMAND = 'bundle exec rake woods:console'
+      MODE_OPTIONS = {
+        'direct' => %w[mode command directory],
+        'docker' => %w[mode command container],
+        'ssh' => %w[mode command host user]
+      }.freeze
+      CONFIG_KEYS = MODE_OPTIONS.values.flatten.uniq.freeze
 
       # @param config [Hash] Process-launch configuration
       # @option config [String] 'mode' direct, docker, or ssh (default: direct)
@@ -27,8 +33,9 @@ module Woods
       # @option config [String] 'user' Optional SSH user
       def initialize(config:)
         @config = config
+        validate_config!
         @mode = config.fetch('mode', 'direct')
-        @embedded_command = config.fetch('command', DEFAULT_COMMAND)
+        @embedded_command = config['command']
       end
 
       # Return the argv used to launch the embedded MCP server.
@@ -36,6 +43,7 @@ module Woods
       # @return [Array<String>]
       # @raise [ConnectionError] when the mode is invalid or incomplete
       def command
+        validate_mode_options!
         case @mode
         when 'direct' then embedded_argv
         when 'docker' then docker_command
@@ -53,10 +61,11 @@ module Woods
       # @return [void]
       # @raise [ConnectionError] when the process cannot be launched
       def replace_process!
+        argv = command
         if @mode == 'direct' && @config['directory']
-          Dir.chdir(@config['directory']) { exec(*command) }
+          Dir.chdir(@config['directory']) { exec(*argv) }
         else
-          exec(*command)
+          exec(*argv)
         end
       rescue SystemCallError, ArgumentError => e
         raise ConnectionError, "Failed to launch embedded Console MCP (#{@mode}): #{e.message}"
@@ -64,7 +73,43 @@ module Woods
 
       private
 
+      def validate_config!
+        raise ConnectionError, 'Console configuration must be a YAML mapping' unless @config.is_a?(Hash)
+        raise ConnectionError, 'Console configuration keys must be strings' unless @config.keys.all?(String)
+
+        unknown = @config.keys - CONFIG_KEYS
+        unless unknown.empty?
+          raise ConnectionError,
+                "Unsupported console.yml keys: #{unknown.map(&:inspect).join(', ')}. " \
+                "Use top-level #{CONFIG_KEYS.join(', ')} launch options; " \
+                'configure access controls and redaction in the Rails initializer.'
+        end
+
+        @config.each do |key, value|
+          next if valid_launch_string?(value)
+
+          raise ConnectionError, "Console #{key} must be a non-empty string without NUL bytes"
+        end
+      end
+
+      def valid_launch_string?(value)
+        value.is_a?(String) && !value.strip.empty? && !value.include?("\0")
+      end
+
+      def validate_mode_options!
+        accepted = MODE_OPTIONS[@mode]
+        return unless accepted
+
+        unused = @config.keys - accepted
+        return if unused.empty?
+
+        raise ConnectionError,
+              "Console options #{unused.join(', ')} are not used in #{@mode} mode; select the intended mode"
+      end
+
       def embedded_argv
+        return default_argv unless @embedded_command
+
         argv = @embedded_command.to_s.shellsplit
         raise ConnectionError, 'Console command must not be empty' if argv.empty?
 
@@ -73,6 +118,14 @@ module Woods
         raise ConnectionError, "Invalid console command: #{e.message}"
       end
 
+      def default_argv
+        if @mode == 'direct'
+          binstub = File.expand_path('bin/rake', @config.fetch('directory', Dir.pwd))
+          return [binstub, 'woods:console'] if File.file?(binstub) && File.executable?(binstub)
+        end
+        DEFAULT_COMMAND.shellsplit
+      end
+
       def docker_command
         container = @config['container'] || raise(ConnectionError, 'Docker mode requires container name')
         ['docker', 'exec', '-i', container] + embedded_argv

data/lib/woods/console/credential_index.rb

@@ -46,7 +46,7 @@ module Woods
     #   index.redact("token: sk_live_actual_secret_value")
     #     # => "token: [REDACTED:credential]"
     #
-    class CredentialIndex
+    class CredentialIndex # rubocop:disable Metrics/ClassLength
       # Captured at require time so the mtime-check warning has a stable
       # reference point even if the clock skews later. Frozen immediately
       # to prevent accidental mutation.
@@ -187,7 +187,10 @@ module Woods
       def initialize(secrets:)
         filtered = Array(secrets).select { |s| s.is_a?(String) && s.length >= MIN_LENGTH }
         @secrets = filtered.to_set.freeze
-        @pattern = @secrets.empty? ? nil : Regexp.union(@secrets.to_a)
+        # Regexp alternatives match in order: a shorter prefix must not consume
+        # only the start of a longer indexed credential and expose its suffix.
+        longest_first = @secrets.each_with_index.sort_by { |secret, idx| [-secret.length, idx] }.map(&:first)
+        @pattern = @secrets.empty? ? nil : Regexp.union(longest_first)
       end
 
       # @return [Boolean] true when no secrets were collected (missing key,
@@ -212,7 +215,34 @@ module Woods
       def redact(str)
         return str if empty? || !str.is_a?(String) || !@pattern.match?(str)
 
-        str.gsub(@pattern, REDACTED)
+        offset = 0
+        parts = []
+        redaction_spans(str).each do |start, finish|
+          parts << str[offset...start] << REDACTED
+          offset = finish
+        end
+        parts << str[offset..]
+        parts.join
+      end
+
+      private
+
+      # Search from each match's start, rather than its end, so an overlapping
+      # credential cannot expose its suffix after an earlier replacement.
+      # Union the covered spans before changing the original string.
+      def redaction_spans(str)
+        spans = []
+        offset = 0
+        while (match = @pattern.match(str, offset))
+          start, finish = match.offset(0)
+          if spans.last && start < spans.last[1]
+            spans.last[1] = [spans.last[1], finish].max
+          else
+            spans << [start, finish]
+          end
+          offset = start + 1
+        end
+        spans
       end
     end
   end