woods 2.0.1 → 2.1.0

data/lib/woods/extractors/standalone_module_discovery.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'set'
+require_relative '../source_references/runtime_lookup'
+require_relative 'concern_extractor'
+
+module Woods
+  module Extractors
+    # Discovers callable modules whose canonical declaration and own methods live
+    # in app/models. It does not autoload constants or invoke application methods.
+    # Cross-file reopenings deliberately do not become additional source owners.
+    class StandaloneModuleDiscovery
+      CORE_SINGLETON = Object.instance_method(:singleton_class)
+      CORE_SOURCE = Module.instance_method(:const_source_location)
+      CORE_METHOD = Module.instance_method(:instance_method)
+      CORE_OWNER = UnboundMethod.instance_method(:owner)
+      CORE_LOCATION = UnboundMethod.instance_method(:source_location)
+      VISIBILITIES = %i[public protected private].freeze
+      CORE_LISTS = VISIBILITIES.to_h { |visibility| [visibility, Module.instance_method("#{visibility}_instance_methods")] }
+                               .freeze
+
+      def initialize(root: Rails.root, concerns: ConcernExtractor.new)
+        @root = File.expand_path(root)
+        @lookup = SourceReferences::RuntimeLookup.new
+        @concerns = concerns
+      end
+
+      # @param path [String] original application source path
+      # @param analysis [Hash] SourceReferences::Collector result for this file
+      # @return [Array<Hash>] canonical identifiers and source-verified method metadata
+      def call(path, analysis:)
+        return [] unless eligible_path?(path)
+
+        declarations = analysis.fetch('declarations').select do |declaration|
+          declaration['kind'] == 'module' && declaration.fetch('singleton_depth', 0).zero?
+        end
+        declarations.group_by { |declaration| declaration['owner'] }.filter_map do |identifier, candidates|
+          next if claimed_identity?(identifier)
+
+          value = verified_module(identifier, candidates, path)
+          next unless value
+
+          methods = own_methods(value, path, candidates)
+          next if methods[:all].empty?
+
+          { identifier: identifier, public_methods: methods[:public], class_methods: methods[:singleton],
+            method_count: methods[:all].size }
+        end
+      end
+
+      private
+
+      def eligible_path?(path)
+        absolute = File.expand_path(path, @root)
+        return false unless absolute.start_with?("#{@root}/app/models/") && absolute.end_with?('.rb')
+        return false if absolute.include?('/concerns/')
+
+        File.realpath(absolute).start_with?("#{File.realpath(@root)}/app/models/")
+      end
+
+      def claimed_identity?(identifier)
+        @claimed ||= @concerns.runtime_model_mixins.values.flatten.to_set do |mod|
+          @lookup.reflect(mod, :name)
+        end
+        @claimed.include?(identifier)
+      end
+
+      def verified_module(identifier, declarations, path)
+        declarations.each do |declaration|
+          result = @lookup.call(declaration['name'], nesting: declaration.fetch('enclosing_nesting', []),
+                                                     allow_private: true)
+          value = result[:value]
+          next unless result[:status] == :resolved && result[:target] == identifier
+          next if @lookup.class_object?(value)
+          next unless canonical_path(identifier) == File.realpath(path)
+
+          return value
+        end
+        nil
+      end
+
+      def canonical_path(identifier)
+        parts = identifier.split('::')
+        name = parts.pop
+        scope = parts.empty? ? Object : @lookup.call("::#{parts.join('::')}", allow_private: true)[:value]
+        return unless @lookup.module_object?(scope)
+
+        location = CORE_SOURCE.bind(scope).call(name, false)
+        File.realpath(location.first) if location && File.file?(location.first)
+      end
+
+      def own_methods(value, path, declarations)
+        instance = methods_at(value, path, declarations)
+        singleton = methods_at(CORE_SINGLETON.bind(value).call, path, declarations)
+        all = (instance.values.flat_map(&:to_a) + singleton.values.flat_map(&:to_a)).uniq
+        { public: instance.fetch(:public).keys.sort, singleton: singleton.fetch(:public).keys.sort, all: all }
+      end
+
+      def methods_at(scope, path, declarations)
+        VISIBILITIES.to_h do |visibility|
+          methods = CORE_LISTS.fetch(visibility).bind(scope).call(false).filter_map do |name|
+            method = CORE_METHOD.bind(scope).call(name)
+            next unless SourceReferences::RuntimeLookup::CORE_EQUAL.bind(CORE_OWNER.bind(method).call).call(scope)
+
+            location = CORE_LOCATION.bind(method).call
+            next unless local_method?(location, path, declarations)
+
+            [name.to_s, location]
+          end
+          [visibility, methods.to_h]
+        end
+      end
+
+      def local_method?(location, path, declarations)
+        unless location && File.file?(location.first) && File.realpath(location.first) == File.realpath(path)
+          return false
+        end
+
+        declarations.any? { |declaration| (declaration['line']..declaration['end_line']).cover?(location.last) }
+      end
+    end
+  end
+end

data/lib/woods/extractors/state_machine_extractor.rb

@@ -5,6 +5,7 @@ require_relative '../source_inputs/consumer_errors'
 require_relative 'shared_utility_methods'
 require_relative 'shared_dependency_scanner'
 require_relative 'source_nesting'
+require_relative 'class_declarations'
 
 module Woods
   module Extractors
@@ -13,7 +14,7 @@ module Woods
     # Supports three state machine gems:
     # - AASM: files that include AASM with +aasm do...end+ blocks
     # - Statesman: files that include Statesman::Machine with state/transition calls
-    # - state_machines: files using the +state_machine :attr do...end+ DSL
+    # - state_machines: literal +state_machine+ declarations, with default or named attributes
     #
     # Produces one ExtractedUnit per state machine definition found.
     # A single model file can produce multiple units (e.g., two state_machine blocks).
@@ -184,7 +185,8 @@ module Woods
 
       # Extract state_machines gem state machine units from source.
       #
-      # Handles multiple state_machine blocks for different attributes.
+      # Handles default/named attributes and parenthesized declarations, with
+      # each machine's metadata restricted to its own block.
       #
       # @param source [String] Ruby source code
       # @param class_name [String] Model class name
@@ -193,16 +195,19 @@ module Woods
       def extract_state_machines_units(source, class_name, file_path)
         return [] unless source.match?(/\bstate_machine\b/)
 
-        units = []
-        source.scan(/state_machine\s+:(\w+)/) do |match|
-          attr_name = match[0]
-          block = extract_block_for_state_machine(source, attr_name)
+        owners = ClassDeclarations.read(source).select { |record| record[:identifier] == class_name }
+        declarations = owners.flat_map { |record| state_machine_declarations(record[:node].body) }
+        declarations.filter_map do |declaration|
+          attr_name = state_machine_attribute(declaration)
+          next unless attr_name
+
+          block = declaration.block&.location&.slice.to_s
           states = block.scan(/^\s*state\s+:(\w+)/).flatten
           events = parse_events_from_source(block, /\Aevent\s+:(\w+)/)
-          initial_state = source.match(/state_machine\s+:#{Regexp.escape(attr_name)}[^#\n]*initial:\s*:(\w+)/)&.[](1)
+          initial_state = state_machine_initial_state(declaration)
           callbacks = parse_state_machine_callbacks(block)
 
-          units << build_unit(
+          build_unit(
             identifier: "#{class_name}::state_machine_#{attr_name}",
             class_name: class_name,
             file_path: file_path,
@@ -215,42 +220,47 @@ module Woods
             callbacks: callbacks
           )
         end
-
-        units
       end
 
-      # Extract the block body for a specific state_machine attribute.
-      #
-      # Uses depth tracking (do/end balance) to find the block boundaries.
+      # Read only direct calls in the selected class body. Nested classes,
+      # singleton classes and deferred/receiver blocks do not belong to it.
       #
-      # @param source [String] Ruby source code
-      # @param attr_name [String] Attribute name (e.g., "status")
-      # @return [String] Block body source
-      def extract_block_for_state_machine(source, attr_name)
-        lines = source.lines
-        result = []
-        depth = 0
-        capturing = false
+      # @param node [Prism::Node, nil] selected class body or direct statement
+      # @return [Array<Prism::CallNode>] state machine declarations in source order
+      def state_machine_declarations(node)
+        if node.is_a?(Prism::StatementsNode)
+          return node.body.flat_map { |statement| state_machine_declarations(statement) }
+        end
 
-        lines.each do |line|
-          stripped = line.strip
+        if node.is_a?(Prism::CallNode) && node.name == :state_machine &&
+           (node.receiver.nil? || node.receiver.is_a?(Prism::SelfNode)) &&
+           (node.block.nil? || node.block.is_a?(Prism::BlockNode))
+          return [node]
+        end
 
-          unless capturing
-            if stripped.match?(/\Astate_machine\s+:#{Regexp.escape(attr_name)}.*\bdo\b/)
-              capturing = true
-              depth = 1
-            end
-            next
-          end
+        []
+      end
 
-          depth += 1 if block_opener?(stripped)
-          depth -= 1 if stripped == 'end'
-          break if depth <= 0
+      # @param declaration [Prism::CallNode] state_machine DSL call
+      # @return [String, nil] literal attribute, defaulting to state for options-only calls
+      def state_machine_attribute(declaration)
+        argument = declaration.arguments&.arguments&.first
+        return 'state' if argument.nil? || argument.is_a?(Prism::KeywordHashNode) || argument.is_a?(Prism::HashNode)
+        return argument.unescaped if argument.is_a?(Prism::SymbolNode) && argument.unescaped.match?(/\A\w+\z/)
 
-          result << line
-        end
+        nil
+      end
+
+      # @param declaration [Prism::CallNode] state_machine DSL call
+      # @return [String, nil] literal initial state; dynamic initializers are never called
+      def state_machine_initial_state(declaration)
+        options = declaration.arguments&.arguments&.last
+        return unless options.is_a?(Prism::KeywordHashNode) || options.is_a?(Prism::HashNode)
 
-        result.join
+        initial = options.elements.find do |entry|
+          entry.is_a?(Prism::AssocNode) && entry.key.is_a?(Prism::SymbolNode) && entry.key.unescaped == 'initial'
+        end
+        initial.value.unescaped if initial&.value.is_a?(Prism::SymbolNode)
       end
 
       # ──────────────────────────────────────────────────────────────────────
@@ -347,10 +357,6 @@ module Woods
         nil
       end
 
-      # NOTE: block_opener? (the RakeTaskExtractor-style depth-tracking
-      # discipline used by extract_block_for_state_machine above) is provided
-      # by the included SourceNesting module.
-
       # ──────────────────────────────────────────────────────────────────────
       # Unit Construction
       # ──────────────────────────────────────────────────────────────────────

data/lib/woods/extractors/view_component_extractor.rb

@@ -67,9 +67,7 @@ module Woods
         # "addition", extract it, get nil, and dirty the dependents pass for
         # nothing. Anonymous classes go too: `extract_component` rejects a nil
         # name for the same reason.
-        @component_base.descendants.reject do |component|
-          component.name.nil? || preview_class?(component)
-        end
+        @component_base.descendants.select { |component| app_component?(component) }
       end
 
       # Extract a single ViewComponent component
@@ -77,8 +75,7 @@ module Woods
       # @param component [Class] The component class
       # @return [ExtractedUnit, nil] The extracted unit, or nil on failure
       def extract_component(component)
-        return nil if component.name.nil?
-        return nil if preview_class?(component)
+        return nil unless app_component?(component)
 
         unit = ExtractedUnit.new(
           type: :view_component,
@@ -103,6 +100,11 @@ module Woods
 
       private
 
+      def app_component?(component)
+        @component_base && component.name && component < @component_base &&
+          !preview_class?(component) && app_source_file?(source_file_for(component))
+      end
+
       # Find the ViewComponent::Base class if the gem is loaded
       #
       # @return [Class, nil]
@@ -222,7 +224,7 @@ module Woods
       # Detect sidecar template file (.html.erb next to the .rb file)
       #
       # @param component [Class]
-      # @return [String, nil] Path to sidecar template if found
+      # @return [String, nil] App-relative path to sidecar template if found
       def detect_sidecar_template(component)
         base_path = Rails.root.join("app/components/#{component.name.underscore}")
 
@@ -234,7 +236,7 @@ module Woods
           "#{base_path}/#{component.name.demodulize.underscore}.html.erb"
         ]
 
-        candidates.find { |path| File.exist?(path) }
+        candidates.find { |path| File.exist?(path) }&.delete_prefix(File.join(Rails.root.to_s, ''))
       rescue StandardError
         nil
       end

data/lib/woods/flow_assembler.rb

@@ -390,7 +390,10 @@ module Woods
       ]
 
       filenames.each do |filename|
-        Dir[File.join(@extracted_dir, '*', filename)].each do |path|
+        Dir.children(@extracted_dir).sort.each do |directory|
+          path = File.join(@extracted_dir, directory, filename)
+          next unless File.file?(path)
+
           # Force UTF-8: the extractor writes the routes-comment header in
           # source_code using Unicode box-drawing characters; reading under
           # the platform default (US-ASCII on some CIs) raises

data/lib/woods/generation.rb

@@ -42,6 +42,9 @@ module Woods
   class Generation
     FILENAME = 'generation.json'
 
+    # Raised by strict readers instead of treating corruption as a legacy index.
+    class InvalidMarker < IOError; end
+
     # An immutable snapshot of the generation file.
     #
     # @!attribute number
@@ -104,6 +107,23 @@ module Woods
       UNPUBLISHED
     end
 
+    # Read and validate an existing marker for a long-lived reader. Missing
+    # markers still describe legacy flat indexes; malformed existing files do not.
+    # @return [Marker]
+    # @raise [InvalidMarker] if the existing marker cannot identify a generation
+    def current!
+      data = JSON.parse(AtomicFile.read(@path))
+      raise InvalidMarker, 'Published generation marker has an invalid shape' unless valid_marker_data?(data)
+      raise InvalidMarker, 'Published generation payload path is malformed' if data['payload']&.include?("\0")
+
+      Marker.new(number: data['number'], token: data['token'], updated_at: data['updated_at'],
+                 reason: data['reason'], payload: data['payload'])
+    rescue Errno::ENOENT
+      UNPUBLISHED
+    rescue JSON::ParserError, EncodingError, SystemCallError
+      raise InvalidMarker, 'Published generation marker is unreadable or malformed'
+    end
+
     # Advance to the next generation.
     #
     # Call this *after* the index files are written, never before, and never
@@ -162,6 +182,11 @@ module Woods
 
     private
 
+    def valid_marker_data?(data)
+      data.is_a?(Hash) && data['number'].is_a?(Integer) && data['number'] >= 0 &&
+        %w[token updated_at reason payload].all? { |key| data[key].nil? || data[key].is_a?(String) }
+    end
+
     # Does +candidate+ resolve inside the index root?
     #
     # Compared through +realpath+, mirroring

data/lib/woods/hooks/context_hint.rb

@@ -52,8 +52,13 @@ module Woods
       end
 
       def source_freshness
-        SourceInputs::Status.new(output_dir: @output, root: @event.root, payload_dir: @reader.payload_dir,
-                                 generation: @reader.loaded_generation).call.fetch('state')
+        status = SourceInputs::Status.new(output_dir: @output, root: @event.root, payload_dir: @reader.payload_dir,
+                                          generation: @reader.loaded_generation).call
+        return status.fetch('state') unless status['state'] == 'unavailable'
+
+        evidence = status.fetch('unavailable')
+        "unavailable (#{evidence.fetch('reason')}; #{evidence.fetch('size_bytes')} bytes; " \
+          "limit #{evidence.fetch('limit_bytes')})"
       end
 
       def orientation(freshness)

data/lib/woods/mcp/bootstrapper.rb

@@ -11,6 +11,7 @@ require_relative '../index_artifact'
 require_relative '../builder'
 require_relative '../resolved_config'
 require_relative '../generation'
+require_relative '../chunking/contributor_chunks'
 require_relative '../storage/snapshotter'
 require_relative '../storage/inapplicable_backend'
 
@@ -327,7 +328,10 @@ module Woods
         end
         resolved = build_resolved_config(config)
 
-        return refresh_reader_only(reader, zero_counts) unless refreshable_stores?(target)
+        unless refreshable_stores?(target)
+          assert_no_snapshot_vector_reload!(target, served)
+          return refresh_reader_only(reader, zero_counts)
+        end
 
         # A captured dump whose embedded config names store types the
         # refreshable live target cannot refresh cannot be turned into
@@ -449,6 +453,20 @@ module Woods
       end
       private_class_method :refreshable_stores?
 
+      # The local preset combines snapshot vectors with live SQLite metadata.
+      # A reader-only reload leaves those vectors stale; report that limitation
+      # explicitly until both stores can participate in an atomic replacement.
+      def self.assert_no_snapshot_vector_reload!(target, served)
+        return unless target.vector_store.respond_to?(:clear!) && target.vector_store.respond_to?(:bulk_load)
+
+        raise ReloadDegraded.new(
+          'reload cannot atomically refresh snapshot vectors with this metadata store; restart woods-mcp ' \
+          'to load the latest vectors after woods:embed',
+          generation: served.number, stores: %w[vector metadata]
+        )
+      end
+      private_class_method :assert_no_snapshot_vector_reload!
+
       # The captured dump's embedded config and the refreshable live target
       # must agree on store types (M2). Each is valid alone: the live target
       # is refreshable (in-memory), and the dump is complete and valid. But
@@ -736,20 +754,24 @@ module Woods
       private_class_method :build_artifact
 
       def self.build_retriever_from_config(config, resolved, artifact, state = nil)
+        builder = Woods::Builder.new(config)
         vector_store = hydrated_vector_store(config, resolved, artifact, state)
         metadata_store = hydrated_metadata_store(config, resolved, artifact, state)
+        metadata_store ||= builder.build_metadata_store if config.metadata_store == :sqlite
         graph_store = hydrated_graph_store(config, artifact, state)
 
         # Cross-populate the vector store's per-entry metadata cache from
         # the metadata store. The WVF1 binary format stores only id + float
         # blob (no per-vector hash) — it's numeric-only to keep dumps
-        # mmap-friendly. The metadata lives in metadata.msgpack, and
+        # mmap-friendly. The metadata lives in metadata.msgpack or the configured
+        # SQLite store, which must be opened before this back-fill. The same
+        # metadata instance is then handed to the retriever below, and
         # InMemory::VectorStore#search uses its per-entry metadata for
         # filter predicates. Without this back-fill, a type-filtered
         # search returns zero results after a dump/reload.
         populate_vector_metadata(vector_store, metadata_store) if vector_store && metadata_store
 
-        Woods::Builder.new(config).build_retriever(
+        builder.build_retriever(
           vector_store: vector_store, metadata_store: metadata_store,
           graph_store: graph_store
         )
@@ -848,9 +870,10 @@ module Woods
       private_constant :CHUNK_SUFFIX_PATTERN
 
       # Back-fill the vector store's per-entry metadata hashes from the
-      # metadata store. Only makes sense when both are in-memory — durable
-      # backends return nil from the hydration helpers and never reach
-      # this path. Both callers hand over stores they built themselves —
+      # metadata store. Only applies to in-memory vectors; metadata may be an
+      # in-memory snapshot or the configured SQLite store. Durable vector
+      # backends return nil from hydration and never reach this path.
+      # Both callers hand over stores they built themselves —
       # boot's freshly hydrated pair and {.build_reload_candidates}'
       # off-side candidates (the M7 transaction retired the old in-place
       # variant that reached a LIVE store) — but the guard below still has
@@ -868,7 +891,10 @@ module Woods
           meta = metadata_store.find(id.to_s.sub(CHUNK_SUFFIX_PATTERN, ''))
           next if meta.nil? || (meta.respond_to?(:empty?) && meta.empty?)
 
-          vector_store.store(id, vec, vector_filter_metadata(meta))
+          chunk_index = id.to_s[/#chunk_(\d+)\z/, 1].to_i
+          chunk = Array(meta['embedding_chunks'])[chunk_index]
+          facts = vector_filter_metadata(meta).merge(Chunking::ContributorChunks.vector_metadata(meta, chunk))
+          vector_store.store(id, vec, facts)
         end
       end
       private_class_method :populate_vector_metadata

data/lib/woods/mcp/config_resolver.rb

@@ -164,6 +164,13 @@ module Woods
       # @param artifact [Woods::IndexArtifact]
       # @return [Woods::Configuration]
       def self.populate_from_stored(config, stored, artifact:, env: ENV)
+        if stored.requires_host_provider?
+          raise ConfigMismatch,
+                'This embedding snapshot requires an explicit host provider configuration. ' \
+                'Configure the original provider in the host initializer, or use lexical retrieval; ' \
+                'Woods cannot safely reconstruct these settings from woods.json.'
+        end
+
         config.output_dir = artifact.output_dir.to_s
         config.vector_store = stored.stores[:vector_store] || :in_memory
         config.metadata_store = stored.stores[:metadata_store] || :in_memory
@@ -177,7 +184,11 @@ module Woods
         opts[:host] = stored.embedding_provider[:host] if stored.embedding_provider[:host]
         opts[:num_ctx] = stored.embedding_provider[:num_ctx] if stored.embedding_provider[:num_ctx]
         opts[:read_timeout] = stored.embedding_provider[:read_timeout] if stored.embedding_provider[:read_timeout]
-        opts[:dimension] = stored.embedding_provider[:dimension] if stored.embedding_provider[:dimension]&.positive?
+        if stored.dimension.positive?
+          opts[provider_sym == :fake ? :dimension : :expected_dimensions] = stored.dimension
+        end
+        requested = stored.embedding_provider[:requested_dimensions] || legacy_requested_dimensions(stored)
+        opts[:dimensions] = requested if requested
 
         # OpenAI needs an api_key to construct. `woods.json` deliberately
         # never stores credentials; pull from env here so the standalone
@@ -207,6 +218,19 @@ module Woods
       end
       private_class_method :populate_from_stored
 
+      # Old snapshots stored only the resulting width. A reduced width for
+      # either known OpenAI v3 model can only be reproduced by requesting it.
+      # Do not infer this capability for ada, Ollama, or arbitrary model names.
+      def self.legacy_requested_dimensions(stored)
+        return if stored.embedding_provider.key?(:requested_dimensions)
+        return unless provider_symbol(stored.embedding_provider[:class]) == :openai
+
+        native = { 'text-embedding-3-small' => 1536, 'text-embedding-3-large' => 3072 }
+        width = native[stored.embedding_provider[:model]]
+        stored.dimension if width && stored.dimension.positive? && stored.dimension < width
+      end
+      private_class_method :legacy_requested_dimensions
+
       def self.restore_store_options(config, stored, artifact:, env:)
         if config.metadata_store == :sqlite
           config.metadata_store_options = { database: artifact.output_dir.join('metadata.sqlite3').to_s }
@@ -250,12 +274,7 @@ module Woods
       # @param class_name [String]
       # @return [Symbol, String] symbol for known providers, raw string otherwise
       def self.provider_symbol(class_name)
-        case class_name.to_s
-        when /Ollama/ then :ollama
-        when /OpenAI/ then :openai
-        when /Fake/ then :fake
-        else class_name.to_s
-        end
+        ResolvedConfig::BUILTIN_PROVIDERS.fetch(class_name.to_s, class_name.to_s)
       end
       private_class_method :provider_symbol