woods 1.0.0

data/lib/woods/temporal/snapshot_store.rb

@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+
+module Woods
+  module Temporal
+    # SnapshotStore captures and queries temporal snapshots of extraction runs.
+    #
+    # Each snapshot is anchored to a git commit SHA and stores per-unit content
+    # hashes for efficient diff computation. Full source is not duplicated —
+    # only hashes of source, metadata, and dependencies are stored per snapshot.
+    #
+    # @example Capturing a snapshot
+    #   store = SnapshotStore.new(connection: db)
+    #   store.capture(manifest, unit_hashes)
+    #
+    # @example Comparing snapshots
+    #   diff = store.diff("abc123", "def456")
+    #   diff[:added]    # => [{ identifier: "NewModel", ... }]
+    #   diff[:modified] # => [{ identifier: "User", ... }]
+    #   diff[:deleted]  # => [{ identifier: "OldService", ... }]
+    #
+    class SnapshotStore # rubocop:disable Metrics/ClassLength
+      # @param connection [Object] Database connection supporting #execute and #get_first_row
+      def initialize(connection:)
+        @db = connection
+      end
+
+      # Capture a snapshot after extraction completes.
+      #
+      # Stores the manifest metadata and per-unit content hashes.
+      # Computes diff stats vs. the most recent previous snapshot.
+      #
+      # @param manifest [Hash] The manifest data (string or symbol keys)
+      # @param unit_hashes [Array<Hash>] Per-unit content hashes
+      # @return [Hash] Snapshot record with diff stats
+      def capture(manifest, unit_hashes)
+        git_sha = mget(manifest, 'git_sha')
+        return nil unless git_sha
+
+        previous = find_latest
+        upsert_snapshot(manifest, git_sha, unit_hashes.size)
+
+        snapshot_id = fetch_snapshot_id(git_sha)
+        @db.execute('DELETE FROM woods_snapshot_units WHERE snapshot_id = ?', [snapshot_id])
+        insert_unit_hashes(snapshot_id, unit_hashes)
+
+        update_diff_stats(snapshot_id, previous)
+        find(git_sha)
+      end
+
+      # List snapshots, optionally filtered by branch.
+      #
+      # @param limit [Integer] Max results (default 20)
+      # @param branch [String, nil] Filter by branch name
+      # @return [Array<Hash>] Snapshot summaries sorted by extracted_at descending
+      def list(limit: 20, branch: nil)
+        rows = if branch
+                 @db.execute(
+                   'SELECT * FROM woods_snapshots WHERE git_branch = ? ORDER BY extracted_at DESC LIMIT ?',
+                   [branch, limit]
+                 )
+               else
+                 @db.execute(
+                   'SELECT * FROM woods_snapshots ORDER BY extracted_at DESC LIMIT ?',
+                   [limit]
+                 )
+               end
+
+        rows.map { |row| row_to_hash(row) }
+      end
+
+      # Find a specific snapshot by git SHA.
+      #
+      # @param git_sha [String]
+      # @return [Hash, nil] Snapshot metadata or nil if not found
+      def find(git_sha)
+        row = @db.get_first_row('SELECT * FROM woods_snapshots WHERE git_sha = ?', [git_sha])
+        return nil unless row
+
+        row_to_hash(row)
+      end
+
+      # Compute diff between two snapshots.
+      #
+      # @param sha_a [String] Before snapshot git SHA
+      # @param sha_b [String] After snapshot git SHA
+      # @return [Hash] {added: [...], modified: [...], deleted: [...]}
+      def diff(sha_a, sha_b)
+        id_a = fetch_snapshot_id(sha_a)
+        id_b = fetch_snapshot_id(sha_b)
+
+        return { added: [], modified: [], deleted: [] } unless id_a && id_b
+
+        units_a = load_snapshot_units(id_a)
+        units_b = load_snapshot_units(id_b)
+
+        compute_diff(units_a, units_b)
+      end
+
+      # History of a single unit across snapshots.
+      #
+      # @param identifier [String] Unit identifier
+      # @param limit [Integer] Max snapshots to return (default 20)
+      # @return [Array<Hash>] Entries with git_sha, extracted_at, source_hash, changed flag
+      def unit_history(identifier, limit: 20)
+        rows = @db.execute(<<~SQL, [identifier, limit])
+          SELECT su.source_hash, su.metadata_hash, su.dependencies_hash, su.unit_type,
+                 s.git_sha, s.extracted_at, s.git_branch
+          FROM woods_snapshot_units su
+          JOIN woods_snapshots s ON s.id = su.snapshot_id
+          WHERE su.identifier = ?
+          ORDER BY s.extracted_at DESC
+          LIMIT ?
+        SQL
+
+        entries = rows.map { |row| history_entry_from_row(row) }
+        mark_changed_entries(entries)
+      end
+
+      private
+
+      # Build a history entry hash from a database row.
+      #
+      # @param row [Hash]
+      # @return [Hash]
+      def history_entry_from_row(row)
+        {
+          git_sha: row['git_sha'],
+          extracted_at: row['extracted_at'],
+          git_branch: row['git_branch'],
+          unit_type: row['unit_type'],
+          source_hash: row['source_hash'],
+          metadata_hash: row['metadata_hash'],
+          dependencies_hash: row['dependencies_hash']
+        }
+      end
+
+      # Mark changed flag on history entries by comparing source hashes.
+      #
+      # @param entries [Array<Hash>]
+      # @return [Array<Hash>]
+      def mark_changed_entries(entries)
+        entries.each_with_index do |entry, i|
+          entry[:changed] = if i == entries.size - 1
+                              true # Oldest version is always "changed" (first appearance)
+                            else
+                              entry[:source_hash] != entries[i + 1][:source_hash]
+                            end
+        end
+        entries
+      end
+
+      # Get a value from a hash that may have string or symbol keys.
+      #
+      # @param hash [Hash]
+      # @param key [String]
+      # @return [Object, nil]
+      def mget(hash, key)
+        hash[key] || hash[key.to_sym]
+      end
+
+      # Insert or replace the snapshot row from manifest data.
+      #
+      # @param manifest [Hash]
+      # @param git_sha [String]
+      # @param default_total [Integer]
+      # @return [void]
+      def upsert_snapshot(manifest, git_sha, default_total)
+        params = [
+          git_sha,
+          mget(manifest, 'git_branch'),
+          mget(manifest, 'extracted_at') || Time.now.iso8601,
+          mget(manifest, 'rails_version'),
+          mget(manifest, 'ruby_version'),
+          mget(manifest, 'total_units') || default_total,
+          JSON.generate(mget(manifest, 'counts') || {}),
+          mget(manifest, 'gemfile_lock_sha'),
+          mget(manifest, 'schema_sha')
+        ]
+        @db.execute(<<~SQL, params)
+          INSERT OR REPLACE INTO woods_snapshots
+            (git_sha, git_branch, extracted_at, rails_version, ruby_version,
+             total_units, unit_counts, gemfile_lock_sha, schema_sha)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        SQL
+      end
+
+      # Update a snapshot's diff stats vs. a previous snapshot.
+      #
+      # @param snapshot_id [Integer]
+      # @param previous [Hash, nil]
+      # @return [void]
+      def update_diff_stats(snapshot_id, previous)
+        diff_stats = compute_diff_stats(snapshot_id, previous)
+        @db.execute(
+          'UPDATE woods_snapshots SET units_added = ?, units_modified = ?, units_deleted = ? WHERE id = ?',
+          [diff_stats[:added], diff_stats[:modified], diff_stats[:deleted], snapshot_id]
+        )
+      end
+
+      # Find the most recent snapshot.
+      #
+      # @return [Hash, nil]
+      def find_latest
+        row = @db.get_first_row('SELECT * FROM woods_snapshots ORDER BY extracted_at DESC LIMIT 1')
+        return nil unless row
+
+        row_to_hash(row)
+      end
+
+      # Fetch a snapshot's ID by git SHA.
+      #
+      # @param git_sha [String]
+      # @return [Integer, nil]
+      def fetch_snapshot_id(git_sha)
+        @db.get_first_value('SELECT id FROM woods_snapshots WHERE git_sha = ?', [git_sha])
+      end
+
+      # Insert per-unit hash records for a snapshot.
+      #
+      # @param snapshot_id [Integer]
+      # @param unit_hashes [Array<Hash>]
+      # @return [void]
+      def insert_unit_hashes(snapshot_id, unit_hashes)
+        sql = <<~SQL
+          INSERT INTO woods_snapshot_units
+            (snapshot_id, identifier, unit_type, source_hash, metadata_hash, dependencies_hash)
+          VALUES (?, ?, ?, ?, ?, ?)
+        SQL
+
+        # Wrap in a transaction to batch all inserts into a single commit,
+        # reducing per-row fsync overhead from O(n) to O(1).
+        @db.transaction do
+          unit_hashes.each do |uh|
+            params = [
+              snapshot_id,
+              mget(uh, 'identifier'),
+              mget(uh, 'type').to_s,
+              mget(uh, 'source_hash'),
+              mget(uh, 'metadata_hash'),
+              mget(uh, 'dependencies_hash')
+            ]
+            @db.execute(sql, params)
+          end
+        end
+      end
+
+      # Load all unit records for a snapshot as a hash keyed by identifier.
+      #
+      # @param snapshot_id [Integer]
+      # @return [Hash{String => Hash}]
+      def load_snapshot_units(snapshot_id)
+        sql = <<~SQL
+          SELECT identifier, unit_type, source_hash, metadata_hash, dependencies_hash
+          FROM woods_snapshot_units WHERE snapshot_id = ?
+        SQL
+        rows = @db.execute(sql, [snapshot_id])
+
+        rows.to_h do |row|
+          [row['identifier'], {
+            unit_type: row['unit_type'],
+            source_hash: row['source_hash'],
+            metadata_hash: row['metadata_hash'],
+            dependencies_hash: row['dependencies_hash']
+          }]
+        end
+      end
+
+      # Compute diff between two sets of unit hashes.
+      #
+      # @param units_a [Hash{String => Hash}] Before
+      # @param units_b [Hash{String => Hash}] After
+      # @return [Hash] {added: [...], modified: [...], deleted: [...]}
+      def compute_diff(units_a, units_b) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        added = []
+        modified = []
+        deleted = []
+
+        # Units in B but not A → added
+        # Units in both → check for modifications
+        units_b.each do |identifier, data_b|
+          if units_a.key?(identifier)
+            data_a = units_a[identifier]
+            if data_a[:source_hash] != data_b[:source_hash] ||
+               data_a[:metadata_hash] != data_b[:metadata_hash] ||
+               data_a[:dependencies_hash] != data_b[:dependencies_hash]
+              modified << { identifier: identifier, unit_type: data_b[:unit_type] }
+            end
+          else
+            added << { identifier: identifier, unit_type: data_b[:unit_type] }
+          end
+        end
+
+        # Units in A but not B → deleted
+        units_a.each do |identifier, data_a|
+          deleted << { identifier: identifier, unit_type: data_a[:unit_type] } unless units_b.key?(identifier)
+        end
+
+        { added: added, modified: modified, deleted: deleted }
+      end
+
+      # Compute aggregate diff stats.
+      #
+      # @param current_snapshot_id [Integer]
+      # @param previous_snapshot [Hash, nil]
+      # @return [Hash] {added:, modified:, deleted:}
+      def compute_diff_stats(current_snapshot_id, previous_snapshot)
+        return { added: 0, modified: 0, deleted: 0 } unless previous_snapshot
+
+        prev_id = fetch_snapshot_id(previous_snapshot[:git_sha])
+        return { added: 0, modified: 0, deleted: 0 } unless prev_id
+
+        units_prev = load_snapshot_units(prev_id)
+        units_curr = load_snapshot_units(current_snapshot_id)
+
+        result = compute_diff(units_prev, units_curr)
+        { added: result[:added].size, modified: result[:modified].size, deleted: result[:deleted].size }
+      end
+
+      # Convert a database row to a normalized hash.
+      #
+      # @param row [Hash] SQLite3 result row
+      # @return [Hash]
+      def row_to_hash(row)
+        {
+          id: row['id'],
+          git_sha: row['git_sha'],
+          git_branch: row['git_branch'],
+          extracted_at: row['extracted_at'],
+          rails_version: row['rails_version'],
+          ruby_version: row['ruby_version'],
+          total_units: row['total_units'],
+          unit_counts: row['unit_counts'] ? JSON.parse(row['unit_counts']) : {},
+          gemfile_lock_sha: row['gemfile_lock_sha'],
+          schema_sha: row['schema_sha'],
+          units_added: row['units_added'],
+          units_modified: row['units_modified'],
+          units_deleted: row['units_deleted']
+        }
+      end
+    end
+  end
+end

data/lib/woods/token_utils.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Woods
+  # Shared token estimation utility.
+  #
+  # Uses project convention: (string.length / 4.0).ceil
+  # See docs/TOKEN_BENCHMARK.md — conservative floor (~10.6% overestimate).
+  module TokenUtils
+    module_function
+
+    # Estimate token count for a string.
+    #
+    # @param text [String] Text to estimate
+    # @return [Integer] Estimated token count
+    def estimate_tokens(text)
+      (text.length / 4.0).ceil
+    end
+  end
+end

data/lib/woods/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Woods
+  VERSION = '1.0.0'
+end

data/lib/woods.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+# Woods - Rails Codebase Indexing and Retrieval
+#
+# A system for extracting, indexing, and retrieving context from Rails codebases
+# to enable AI-assisted development, debugging, and analytics.
+#
+# ## Quick Start
+#
+#   # Extract codebase
+#   Woods.extract!
+#
+#   # Or via rake
+#   bundle exec rake woods:extract
+#
+# ## Configuration
+#
+#   Woods.configure do |config|
+#     config.output_dir = Rails.root.join("tmp/woods")
+#     config.max_context_tokens = 8000
+#     config.include_framework_sources = true
+#   end
+#
+require_relative 'woods/version'
+
+module Woods
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class ExtractionError < Error; end
+  class SessionTracerError < Error; end
+
+  CONFIG_MUTEX = Mutex.new
+
+  # ════════════════════════════════════════════════════════════════════════
+  # Configuration
+  # ════════════════════════════════════════════════════════════════════════
+
+  class Configuration
+    attr_accessor :embedding_model, :include_framework_sources, :gem_configs,
+                  :vector_store, :metadata_store, :graph_store, :embedding_provider, :log_level,
+                  :vector_store_options, :metadata_store_options, :embedding_options,
+                  :concurrent_extraction, :precompute_flows, :enable_snapshots,
+                  :session_tracer_enabled, :session_store, :session_id_proc, :session_exclude_paths,
+                  :console_mcp_enabled, :console_mcp_path, :console_redacted_columns,
+                  :notion_api_token, :notion_database_ids,
+                  :cache_store, :cache_options
+    attr_reader :max_context_tokens, :similarity_threshold, :extractors, :pretty_json, :context_format,
+                :cache_enabled
+
+    def initialize # rubocop:disable Metrics/MethodLength
+      @output_dir = nil # Resolved lazily; Rails.root is nil at require time
+      @embedding_model = 'text-embedding-3-small'
+      @max_context_tokens = 8000
+      @similarity_threshold = 0.7
+      @include_framework_sources = true
+      @gem_configs = {}
+      @extractors = %i[models controllers services components view_components jobs mailers graphql serializers
+                       managers policies validators rails_source]
+      @pretty_json = true
+      @concurrent_extraction = false
+      @precompute_flows = false
+      @enable_snapshots = false
+      @context_format = :markdown
+      @session_tracer_enabled = false
+      @session_store = nil
+      @session_id_proc = nil
+      @session_exclude_paths = []
+      @console_mcp_enabled = false
+      @console_mcp_path = '/mcp/console'
+      @console_redacted_columns = []
+      @notion_api_token = nil
+      @notion_database_ids = {}
+      @cache_enabled = false
+      @cache_store = nil      # :redis, :solid_cache, :memory, or a CacheStore instance
+      @cache_options = {}     # { redis: client, cache: store, ttl: { embeddings: 86400, ... } }
+    end
+
+    # @return [Pathname, String] Output directory, defaulting to Rails.root/tmp/woods
+    def output_dir
+      @output_dir ||= defined?(Rails) && Rails.root ? Rails.root.join('tmp/woods') : 'tmp/woods'
+    end
+
+    # @param value [Object] Must respond to #to_s
+    # @raise [ConfigurationError] if value is nil
+    def output_dir=(value)
+      raise ConfigurationError, 'output_dir cannot be nil' if value.nil?
+
+      @output_dir = value
+    end
+
+    # @param value [Integer] Must be a positive Integer
+    # @raise [ConfigurationError] if value is not a positive Integer
+    def max_context_tokens=(value)
+      unless value.is_a?(Integer) && value.positive?
+        raise ConfigurationError, "max_context_tokens must be a positive Integer, got #{value.inspect}"
+      end
+
+      @max_context_tokens = value
+    end
+
+    # @param value [Numeric] Must be between 0.0 and 1.0 inclusive
+    # @raise [ConfigurationError] if value is out of range or not numeric
+    def similarity_threshold=(value)
+      raise ConfigurationError, "similarity_threshold must be Numeric, got #{value.inspect}" unless value.is_a?(Numeric)
+
+      float_val = value.to_f
+      unless float_val.between?(0.0, 1.0)
+        raise ConfigurationError, "similarity_threshold must be between 0.0 and 1.0, got #{value.inspect}"
+      end
+
+      @similarity_threshold = float_val
+    end
+
+    # @param value [Array<Symbol>] List of extractor names
+    # @raise [ConfigurationError] if value is not an Array of Symbols
+    def extractors=(value)
+      unless value.is_a?(Array) && value.all?(Symbol)
+        raise ConfigurationError, "extractors must be an Array of Symbols, got #{value.inspect}"
+      end
+
+      @extractors = value
+    end
+
+    # @param value [Boolean] Must be true or false
+    # @raise [ConfigurationError] if value is not a boolean
+    def pretty_json=(value)
+      validate_boolean!(:pretty_json, value)
+      @pretty_json = value
+    end
+
+    # @param value [Symbol] Must be one of :claude, :markdown, :plain, :json
+    # @raise [ConfigurationError] if value is not a valid format
+    def context_format=(value)
+      valid = %i[claude markdown plain json]
+      unless valid.include?(value)
+        raise ConfigurationError, "context_format must be one of #{valid.inspect}, got #{value.inspect}"
+      end
+
+      @context_format = value
+    end
+
+    # @param value [Boolean] Enable or disable the cache layer
+    # @raise [ConfigurationError] if value is not a boolean
+    def cache_enabled=(value)
+      validate_boolean!(:cache_enabled, value)
+      @cache_enabled = value
+    end
+
+    # Add a gem to be indexed
+    #
+    # @param gem_name [String] Name of the gem
+    # @param paths [Array<String>] Relative paths within the gem to index
+    # @param priority [Symbol] :high, :medium, or :low
+    def add_gem(gem_name, paths:, priority: :medium)
+      @gem_configs[gem_name] = { paths: paths, priority: priority }
+    end
+
+    private
+
+    def validate_boolean!(name, value)
+      return if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+
+      raise ConfigurationError, "#{name} must be true or false, got #{value.inspect}"
+    end
+  end
+
+  # ════════════════════════════════════════════════════════════════════════
+  # Module Interface
+  # ════════════════════════════════════════════════════════════════════════
+
+  class << self
+    attr_accessor :configuration
+
+    def configure
+      CONFIG_MUTEX.synchronize do
+        self.configuration ||= Configuration.new
+        yield(configuration) if block_given?
+        configuration
+      end
+    end
+
+    # Configure the module using a named preset and optional block customization.
+    #
+    # Valid preset names: :local, :postgresql, :production
+    #
+    # @param name [Symbol] Preset name
+    # @yield [config] Optional block for further customization after preset is applied
+    # @yieldparam config [Configuration] The configuration object
+    # @return [Configuration] The applied configuration
+    def configure_with_preset(name)
+      CONFIG_MUTEX.synchronize do
+        self.configuration = Builder.preset_config(name)
+        yield configuration if block_given?
+        configuration
+      end
+    end
+
+    # Build a Retriever wired with adapters from the current configuration.
+    #
+    # @return [Retriever] A fully wired retriever instance
+    def build_retriever
+      Builder.new(configuration).build_retriever
+    end
+
+    # Retrieve context for a natural language query using the current configuration.
+    #
+    # @param query [String] Natural language query
+    # @param opts [Hash] Options passed through to the retriever (e.g., budget:)
+    # @return [Retriever::RetrievalResult] Retrieval result
+    def retrieve(query, **opts)
+      build_retriever.retrieve(query, **opts)
+    end
+
+    # Perform full extraction
+    #
+    # @param output_dir [String] Override output directory
+    # @return [Hash] Extraction results
+    def extract!(output_dir: nil)
+      require_relative 'woods/extractor'
+
+      dir = output_dir || configuration.output_dir
+      extractor = Extractor.new(output_dir: dir)
+      extractor.extract_all
+    end
+
+    # Perform incremental extraction
+    #
+    # @param changed_files [Array<String>] List of changed files
+    # @return [Array<String>] Re-extracted unit identifiers
+    def extract_changed!(changed_files)
+      require_relative 'woods/extractor'
+
+      extractor = Extractor.new(output_dir: configuration.output_dir)
+      extractor.extract_changed(changed_files)
+    end
+  end
+
+  # Initialize with defaults
+  configure
+end
+
+require_relative 'woods/builder'
+require_relative 'woods/cost_model'
+require_relative 'woods/cache/cache_store'
+require_relative 'woods/cache/cache_middleware'
+require_relative 'woods/railtie' if defined?(Rails::Railtie)