woods 1.2.0 → 1.3.0

data/lib/generators/woods/install_generator.rb

@@ -5,20 +5,29 @@ require 'rails/generators/active_record'
 
 module Woods
   module Generators
-    # Rails generator that creates a migration for Woods tables.
+    # Rails generator that installs Woods into a Rails application.
     #
     # Usage:
     #   rails generate woods:install
     #
-    # Creates a migration with woods_units, woods_edges, and
-    # woods_embeddings tables. Works with PostgreSQL, MySQL, and SQLite.
+    # Creates:
+    #   config/initializers/woods.rb        — annotated configuration file
+    #   db/migrate/<ts>_create_woods_tables.rb — migration for Woods tables
+    #
+    # The migration creates woods_units, woods_edges, and woods_embeddings
+    # tables. Works with PostgreSQL, MySQL, and SQLite.
     #
     class InstallGenerator < Rails::Generators::Base
       include ActiveRecord::Generators::Migration
 
       source_root File.expand_path('templates', __dir__)
 
-      desc 'Creates a migration for Woods tables (units, edges, embeddings)'
+      desc 'Creates a Woods initializer and migration for Woods tables'
+
+      # @return [void]
+      def create_initializer_file
+        template 'woods.rb.tt', 'config/initializers/woods.rb'
+      end
 
       # @return [void]
       def create_migration_file

data/lib/generators/woods/templates/woods.rb.tt

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+# Woods configuration
+# Full reference: https://github.com/bigcartel/woods/blob/main/docs/CONFIGURATION_REFERENCE.md
+#
+# Quick-start presets (uncomment one instead of the full block below):
+#   Woods.configure_with_preset(:local)       # in-memory + Ollama, no external services
+#   Woods.configure_with_preset(:postgresql)  # pgvector + OpenAI (PostgreSQL required)
+#   Woods.configure_with_preset(:production)  # Qdrant + OpenAI (production-scale)
+#
+# Presets accept a block for overrides:
+#   Woods.configure_with_preset(:local) { |c| c.max_context_tokens = 16_000 }
+
+Woods.configure do |config|
+  # ── Core ────────────────────────────────────────────────────────────────
+
+  # Directory where extracted JSON is written.
+  # Default: Rails.root.join('tmp/woods')
+  config.output_dir = Rails.root.join('tmp/woods')
+
+  # Maximum tokens returned in a retrieval context window.
+  # config.max_context_tokens = 8_000
+
+  # Minimum vector similarity score (0.0–1.0) for retrieval results.
+  # config.similarity_threshold = 0.7
+
+  # Output format for retrieval: :claude, :markdown, :plain, :json
+  # config.context_format = :markdown
+
+  # Pretty-print extracted JSON (disable in CI to save disk space).
+  # config.pretty_json = true
+
+  # ── Extractors ──────────────────────────────────────────────────────────
+
+  # Enabled extractors. Default set covers the most common Rails layers.
+  # See CONFIGURATION_REFERENCE.md for the full list of available symbols.
+  # config.extractors = %i[
+  #   models controllers services components view_components
+  #   jobs mailers graphql serializers managers policies validators
+  #   rails_source
+  # ]
+
+  # Include Rails / gem source in the index (increases extraction time).
+  # config.include_framework_sources = true
+
+  # Enable parallel extraction (experimental — may conflict with some apps).
+  # config.concurrent_extraction = false
+
+  # ── Embedding ───────────────────────────────────────────────────────────
+
+  # Embedding provider: :openai or :ollama
+  # config.embedding_provider = :openai
+  # config.embedding_model    = 'text-embedding-3-small'
+  # config.embedding_options  = { api_key: ENV['OPENAI_API_KEY'] }
+
+  # Ollama (local, no API key needed). `num_ctx` is auto-selected per model
+  # (nomic-embed-text → 2048, bge-m3 → 8192). Install `gem "tokenizers"` for
+  # exact BERT WordPiece token counting on dense Ruby source. See
+  # docs/EMBEDDING_MODELS.md for the full model comparison.
+  # config.embedding_provider = :ollama
+  # config.embedding_options  = {
+  #   model: 'nomic-embed-text',
+  #   host: ENV.fetch('OLLAMA_URL', 'http://localhost:11434')
+  # }
+
+  # ── Storage ─────────────────────────────────────────────────────────────
+
+  # Vector store: :in_memory, :pgvector (PostgreSQL), :qdrant
+  # config.vector_store = :in_memory
+
+  # pgvector — run `rails generate woods:pgvector && rails db:migrate` first.
+  # config.vector_store         = :pgvector
+  # config.vector_store_options = {
+  #   connection: ActiveRecord::Base.connection,
+  #   dimensions: 1_536
+  # }
+
+  # Qdrant:
+  # config.vector_store         = :qdrant
+  # config.vector_store_options = {
+  #   url:        ENV.fetch('QDRANT_URL', 'http://localhost:6333'),
+  #   collection: 'woods',
+  #   dimensions: 1_536
+  # }
+
+  # Metadata store: :in_memory, :sqlite
+  # config.metadata_store = :in_memory
+  # config.metadata_store_options = {
+  #   database: Rails.root.join('tmp/woods/metadata.sqlite3').to_s
+  # }
+
+  # ── Pipeline ────────────────────────────────────────────────────────────
+
+  # Pre-compute per-action request flow maps during extraction (slow).
+  # config.precompute_flows = false
+
+  # Extract link_to / redirect_to / form_action navigation edges.
+  # config.extract_navigation_edges = true
+
+  # Temporal snapshots — requires migrations 004+005.
+  # config.enable_snapshots = false
+
+  # ── Console MCP ─────────────────────────────────────────────────────────
+  #
+  # The Console MCP server lets AI tools query your live Rails app.
+  # It is DISABLED by default. Enable only after reviewing the security
+  # documentation in docs/CONSOLE_MCP_SETUP.md.
+  #
+  # Defense layers (all active by default when the server is on):
+  #   Layer 1 — SqlValidator: rejects DML/DDL before any DB interaction.
+  #   Layer 2 — SafeContext: wraps every request in a rolled-back transaction;
+  #              writes are silently discarded even if Layer 1 is bypassed.
+  #   Layer 3 — Column redaction: credential columns are replaced with
+  #              [REDACTED] in every tool response.
+
+  # config.console_mcp_enabled = false
+  # config.console_mcp_path    = '/mcp/console'
+
+  # Credential-column redaction (Layer 3).
+  # Starts from a safe default list (passwords, tokens, secrets).
+  # Extend: Woods::DEFAULT_CONSOLE_REDACTED_COLUMNS + %w[my_secret_col]
+  # Override entirely to remove a default:
+  # config.console_redacted_columns = %w[password token api_key]
+
+  # Key-value pairs where the value should be redacted (e.g., env var names).
+  # config.console_redacted_key_values = []
+
+  # Tables completely blocked from queries.
+  # config.console_blocked_tables = []
+
+  # Disable specific scanner patterns (rare — prefer blocked_tables).
+  # config.console_disabled_scanner_patterns = []
+
+  # Allow the AI console to execute Ruby eval (off by default; very dangerous).
+  # config.console_unsafe_eval_enabled = false
+
+  # Expose SQL/query read tools inside embedded console (adds read-only DB
+  # access via the rake task or Docker exec path; SqlValidator still applies).
+  # config.console_embedded_read_tools = false
+
+  # ── Caching ─────────────────────────────────────────────────────────────
+
+  # Cache embedding and retrieval responses to reduce API cost.
+  # config.cache_enabled = false
+  # config.cache_store   = :redis   # :redis, :solid_cache, :memory
+  # config.cache_options = { redis: Redis.new(url: ENV['REDIS_URL']) }
+
+  # ── Notion Export ───────────────────────────────────────────────────────
+
+  # config.notion_api_token   = ENV['NOTION_API_TOKEN']
+  # config.notion_database_ids = {
+  #   data_models: 'your-database-id',
+  #   columns:     'your-database-id'
+  # }
+end

data/lib/tasks/woods.rake

@@ -354,33 +354,11 @@ namespace :woods do
   desc 'Embed all extracted units'
   task embed: :environment do
     require 'woods'
-    require 'woods/embedding/indexer'
-    require 'woods/embedding/text_preparer'
-    require 'woods/embedding/provider'
-    require 'woods/storage/vector_store'
-
-    config = Woods.configuration
-    output_dir = ENV.fetch('WOODS_OUTPUT', config.output_dir)
-
-    provider = Woods::Embedding::Provider::Ollama.new
-    text_preparer = Woods::Embedding::TextPreparer.new
-    vector_store = Woods::Storage::VectorStore::InMemory.new
-
-    indexer = Woods::Embedding::Indexer.new(
-      provider: provider,
-      text_preparer: text_preparer,
-      vector_store: vector_store,
-      output_dir: output_dir
-    )
+    require 'woods/tasks'
 
+    indexer = Woods::Tasks.build_embed_indexer
     puts 'Embedding all extracted units...'
-    stats = indexer.index_all
-
-    puts
-    puts 'Embedding complete!'
-    puts "  Processed: #{stats[:processed]}"
-    puts "  Skipped:   #{stats[:skipped]}"
-    puts "  Errors:    #{stats[:errors]}"
+    Woods::Tasks.print_embed_stats(indexer.index_all, mode: :full)
   end
 
   desc 'Nest the data — embed all units (alias for embed)'
@@ -389,33 +367,11 @@ namespace :woods do
   desc 'Embed changed units only (incremental)'
   task embed_incremental: :environment do
     require 'woods'
-    require 'woods/embedding/indexer'
-    require 'woods/embedding/text_preparer'
-    require 'woods/embedding/provider'
-    require 'woods/storage/vector_store'
-
-    config = Woods.configuration
-    output_dir = ENV.fetch('WOODS_OUTPUT', config.output_dir)
-
-    provider = Woods::Embedding::Provider::Ollama.new
-    text_preparer = Woods::Embedding::TextPreparer.new
-    vector_store = Woods::Storage::VectorStore::InMemory.new
-
-    indexer = Woods::Embedding::Indexer.new(
-      provider: provider,
-      text_preparer: text_preparer,
-      vector_store: vector_store,
-      output_dir: output_dir
-    )
+    require 'woods/tasks'
 
+    indexer = Woods::Tasks.build_embed_indexer
     puts 'Embedding changed units (incremental)...'
-    stats = indexer.index_incremental
-
-    puts
-    puts 'Incremental embedding complete!'
-    puts "  Processed: #{stats[:processed]}"
-    puts "  Skipped:   #{stats[:skipped]}"
-    puts "  Errors:    #{stats[:errors]}"
+    Woods::Tasks.print_embed_stats(indexer.index_incremental, mode: :incremental)
   end
 
   desc 'Hone the blade — incremental embedding (alias for embed_incremental)'
@@ -672,4 +628,13 @@ namespace :woods do
 
   desc 'Relay findings to Unblocked (alias for unblocked_sync)'
   task relay: :unblocked_sync
+
+  desc 'Generate a random bearer token for woods-mcp-http (WOODS_MCP_HTTP_TOKEN)'
+  task :generate_token do
+    require 'securerandom'
+    token = SecureRandom.hex(32)
+    puts token
+    warn 'Set WOODS_MCP_HTTP_TOKEN to this value in the environment where woods-mcp-http runs,'
+    warn 'and send it as `Authorization: Bearer <token>` from clients.'
+  end
 end

data/lib/woods/builder.rb

@@ -8,6 +8,10 @@ require_relative 'storage/metadata_store'
 require_relative 'storage/graph_store'
 require_relative 'embedding/provider'
 require_relative 'embedding/openai'
+require_relative 'embedding/text_preparer'
+require_relative 'embedding/token_counter'
+require_relative 'token_utils'
+require_relative 'chunking/semantic_chunker'
 
 module Woods
   # Builder reads a {Configuration} and instantiates the appropriate adapters,
@@ -29,9 +33,13 @@ module Woods
   class Builder # rubocop:disable Metrics/ClassLength
     # Named presets mapping to default adapter types.
     #
-    # :local      — fully local, no external services required
-    # :postgresql — pgvector for vectors, OpenAI for embeddings
-    # :production — Qdrant for vectors, OpenAI for embeddings
+    # :local              — fully local, no external services required (requires sqlite3 gem)
+    # :shared_filesystem  — Shape 2: rake embed → separate MCP server reads from disk.
+    #                       All stores in-memory + persisted to output_dir via the
+    #                       Snapshotter. No sqlite3 gem needed. Requires output_dir set
+    #                       AND readable by both the embed process and the MCP server.
+    # :postgresql         — pgvector for vectors, OpenAI for embeddings
+    # :production         — Qdrant for vectors, OpenAI for embeddings
     PRESETS = {
       local: {
         vector_store: :in_memory,
@@ -39,6 +47,12 @@ module Woods
         graph_store: :in_memory,
         embedding_provider: :ollama
       },
+      shared_filesystem: {
+        vector_store: :in_memory,
+        metadata_store: :in_memory,
+        graph_store: :in_memory,
+        embedding_provider: :ollama
+      },
       postgresql: {
         vector_store: :pgvector,
         metadata_store: :sqlite,
@@ -78,17 +92,30 @@ module Woods
     # {Cache::CachedEmbeddingProvider} and the retriever is wrapped with
     # {Cache::CachedRetriever} for transparent caching of expensive operations.
     #
+    # Callers that need stores pre-populated from a dump (the Shape-2
+    # MCP-serve path) can inject them via +vector_store:+ / +metadata_store:+.
+    # Without these, fresh empty stores are constructed from config. This
+    # is how the Bootstrapper hydrates from `Snapshotter.load_or_empty`
+    # without Builder needing to know the Snapshotter exists.
+    #
+    # @param vector_store [Storage::VectorStore::Interface, nil]
+    # @param metadata_store [Storage::MetadataStore::Interface, nil]
+    # @param graph_store [Storage::GraphStore::Interface, nil] Pre-populated
+    #   graph store. Without this, the retriever gets a fresh empty graph,
+    #   which silently degrades +:hybrid+ retrieval (graph expansion returns
+    #   no candidates). The Bootstrapper hydrates from +dependency_graph.json+
+    #   on disk and passes the populated store here.
     # @return [Retriever, Cache::CachedRetriever] A fully wired retriever
-    def build_retriever
+    def build_retriever(vector_store: nil, metadata_store: nil, graph_store: nil)
       provider = build_embedding_provider
       cache = build_cache_store
 
       provider = wrap_with_embedding_cache(provider, cache) if cache
 
       retriever = Retriever.new(
-        vector_store: build_vector_store,
-        metadata_store: build_metadata_store,
-        graph_store: build_graph_store,
+        vector_store: vector_store || build_vector_store,
+        metadata_store: metadata_store || build_metadata_store,
+        graph_store: graph_store || build_graph_store,
         embedding_provider: provider
       )
 
@@ -110,18 +137,154 @@ module Woods
 
     # Instantiate the embedding provider specified by the configuration.
     #
+    # Strips `embedding_options` keys that belong to the ResolvedConfig layer
+    # (like `:dimension`) before splatting into the provider's constructor —
+    # those keys are useful for the Snapshotter's schema header but
+    # aren't part of the provider's API.
+    #
     # @return [Embedding::Provider::Interface] Embedding provider instance
     # @raise [ArgumentError] if the configured type is not recognized
     def build_embedding_provider
+      opts = provider_kwargs
       case @config.embedding_provider
-      when :openai then Embedding::Provider::OpenAI.new(**(@config.embedding_options || {}))
-      when :ollama then Embedding::Provider::Ollama.new(**(@config.embedding_options || {}))
+      when :openai then Embedding::Provider::OpenAI.new(**opts)
+      when :ollama then Embedding::Provider::Ollama.new(**opts)
       else raise ArgumentError, "Unknown embedding_provider: #{@config.embedding_provider}"
       end
     end
 
+    # Kwargs accepted by embedding provider constructors — everything in
+    # `embedding_options` except metadata fields that live there for
+    # ResolvedConfig bookkeeping.
+    SNAPSHOT_ONLY_KEYS = %i[dimension].freeze
+    private_constant :SNAPSHOT_ONLY_KEYS
+
+    def provider_kwargs
+      opts = (@config.embedding_options || {}).transform_keys(&:to_sym)
+      SNAPSHOT_ONLY_KEYS.each { |k| opts.delete(k) }
+      opts
+    end
+    private :provider_kwargs
+
+    # Build a {Embedding::TextPreparer} calibrated to a given provider.
+    #
+    # OpenAI embedders use tiktoken (cl100k_base) — 4.0 chars/token is a
+    # good conservative average. Ollama BERT/WordPiece tokenizers
+    # (nomic-embed-text, bge-*) run much hotter on dense Ruby/Rails
+    # source — long CamelCase constants, docstrings, callback DSLs, and
+    # heavy symbol use all sit below 2.0 chars/token in practice.
+    # Empirically, a 16 KB chunk of `ActionMailer::Base` still blows the
+    # 8192-token budget at 2.0 chars/token, so we budget at 1.5 to stay
+    # clear of tokenizer surprises even on the densest Rails internals.
+    #
+    # `max_tokens` tracks the provider's actual input budget when it
+    # reports one, falling back to the TextPreparer default otherwise.
+    #
+    # @param provider [Embedding::Provider::Interface]
+    # @return [Embedding::TextPreparer]
+    def build_text_preparer(provider)
+      chars_per_token = chars_per_token_for(provider)
+      budget = provider.respond_to?(:max_input_tokens) ? provider.max_input_tokens : nil
+      max_tokens = budget || Embedding::TextPreparer::DEFAULT_MAX_TOKENS
+
+      Embedding::TextPreparer.new(max_tokens: max_tokens, chars_per_token: chars_per_token)
+    end
+
+    # Build a {Chunking::SemanticChunker} sized to a given provider.
+    #
+    # `max_chars` is derived from the provider's input budget and the
+    # matching chars-per-token ratio, minus the context-prefix
+    # allowance the Indexer accounts for separately. Units that exceed
+    # this ceiling get sliced so no single chunk can blow the provider's
+    # input cap.
+    #
+    # For Ollama (and other BERT/WordPiece-backed models), char-based
+    # estimation is unreliable — CamelCase, `::` separators, and symbol
+    # literals tokenize much denser than chars/token averages suggest.
+    # When the optional `tokenizers` gem is installed, pass a
+    # {Embedding::TokenCounter} and `max_tokens` so the chunker can
+    # verify every slice with the real tokenizer and re-split any piece
+    # that still exceeds `num_ctx`. See docs/EMBEDDING_MODELS.md.
+    #
+    # Ollama v0.13.5+ stopped honouring `truncate: true` on `/api/embed`
+    # (ollama/ollama#14186), so any chunk that exceeds `num_ctx` returns
+    # a 400 rather than being silently truncated. Exact client-side
+    # sizing is the only reliable path until the regression is fixed
+    # upstream.
+    #
+    # @param provider [Embedding::Provider::Interface]
+    # @return [Chunking::SemanticChunker]
+    def build_chunker(provider)
+      budget = provider.respond_to?(:max_input_tokens) ? provider.max_input_tokens : nil
+      max_chars = ((budget * chars_per_token_for(provider)).floor - CHUNKER_PREFIX_ALLOWANCE if budget)
+
+      # Guard against a budget so small that the prefix allowance leaves
+      # no room for content. Without this, SemanticChunker#slice_by_lines
+      # passes a negative repeat count to String#scan, which returns []
+      # — every chunk becomes empty and is silently dropped, producing
+      # zero embeddings with no error. Surface the misconfiguration loudly.
+      raise ArgumentError, chunker_budget_message(provider, budget) if max_chars && max_chars <= 0
+
+      token_counter = token_counter_for(provider)
+      max_tokens = token_counter && budget ? budget - PREFIX_TOKEN_ALLOWANCE : nil
+
+      Chunking::SemanticChunker.new(
+        max_chars: max_chars,
+        token_counter: token_counter,
+        max_tokens: max_tokens
+      )
+    end
+
+    # Character allowance reserved for the TextPreparer context prefix
+    # ([type] id / namespace / file / deps) — kept in sync with the
+    # Indexer's own PREFIX_CHAR_ALLOWANCE constant.
+    CHUNKER_PREFIX_ALLOWANCE = 512
+    private_constant :CHUNKER_PREFIX_ALLOWANCE
+
+    # Token-side sibling of {CHUNKER_PREFIX_ALLOWANCE}. Reserved for the
+    # TextPreparer prefix when tokenizer-driven sizing is active — a bit
+    # generous to cover long file paths and dep lists.
+    PREFIX_TOKEN_ALLOWANCE = 256
+    private_constant :PREFIX_TOKEN_ALLOWANCE
+
     private
 
+    # Return a TokenCounter for providers that benefit from exact token
+    # counting. OpenAI's tiktoken ratios are already stable at 4.0
+    # chars/token on code, so it doesn't need this.
+    #
+    # @param provider [Embedding::Provider::Interface]
+    # @return [Embedding::TokenCounter, nil]
+    def token_counter_for(provider)
+      return unless provider.is_a?(Embedding::Provider::Ollama)
+
+      Embedding::TokenCounter.new
+    end
+
+    # Tokenizer-calibrated chars/token ratio for the given provider.
+    # Delegates to {Woods::TokenUtils.chars_per_token_for} — the single
+    # source of truth — after reducing the provider instance to a symbol.
+    #
+    # @param provider [Embedding::Provider::Interface]
+    # @return [Float]
+    def chars_per_token_for(provider)
+      symbol = case provider
+               when Embedding::Provider::Ollama then :ollama
+               else :openai
+               end
+      TokenUtils.chars_per_token_for(symbol)
+    end
+
+    # Diagnostic for the build_chunker budget guard.
+    def chunker_budget_message(provider, budget)
+      "embedding model '#{provider.respond_to?(:model) ? provider.model : provider.class}' " \
+        "reports a max_input_tokens of #{budget}, which leaves no room for " \
+        "the chunk prefix (#{CHUNKER_PREFIX_ALLOWANCE} chars). Configure a " \
+        'model with a larger native context, or set num_ctx explicitly.'
+    end
+
+    public
+
     # Instantiate the metadata store adapter specified by the configuration.
     #
     # @return [Storage::MetadataStore::Interface] Metadata store adapter instance
@@ -145,6 +308,8 @@ module Woods
       end
     end
 
+    private
+
     # Build a cache store from configuration, or nil if caching is disabled.
     #
     # @return [Cache::CacheStore, nil]