woods 1.6.1 → 2.0.0.beta2

data/lib/generators/woods/install_generator.rb

@@ -11,18 +11,19 @@ module Woods
     #   rails generate woods:install
     #
     # Creates:
-    #   config/initializers/woods.rb        — annotated configuration file
-    #   db/migrate/<ts>_create_woods_tables.rb — migration for Woods tables
+    #   config/initializers/woods.rb — annotated configuration file
+    #   db/migrate/<ts>_create_woods_tables.rb — legacy compatibility migration
     #
-    # The migration creates woods_units, woods_edges, and woods_embeddings
-    # tables. Works with PostgreSQL, MySQL, and SQLite.
+    # Shipped v2 paths do not use the migration's application tables. New
+    # default installs should remove it; retain it only for an older/custom
+    # integration that deliberately uses woods_units/edges/embeddings.
     #
     class InstallGenerator < Rails::Generators::Base
       include ActiveRecord::Generators::Migration
 
       source_root File.expand_path('templates', __dir__)
 
-      desc 'Creates a Woods initializer and migration for Woods tables'
+      desc 'Creates a Woods initializer and a legacy compatibility migration'
 
       # @return [void]
       def create_initializer_file

data/lib/generators/woods/pgvector_generator.rb

@@ -7,8 +7,11 @@ module Woods
   module Generators
     # Rails generator that adds pgvector support to Woods.
     #
-    # Requires the pgvector PostgreSQL extension. Adds a native vector column
-    # and HNSW index to the woods_embeddings table.
+    # Requires the pgvector PostgreSQL extension. Creates the `woods_vectors`
+    # table (id, native vector column, JSONB metadata) plus an HNSW index —
+    # the schema read and written by Woods::Storage::VectorStore::Pgvector.
+    # The migration mirrors the adapter's idempotent ensure_schema! DDL, so
+    # it coexists with schema creation at embed time.
     #
     # Usage:
     #   rails generate woods:pgvector
@@ -19,7 +22,7 @@ module Woods
 
       source_root File.expand_path('templates', __dir__)
 
-      desc 'Adds pgvector native vector column and HNSW index to woods_embeddings'
+      desc 'Creates the woods_vectors table (pgvector column + HNSW index) used by the Woods vector store'
 
       class_option :dimensions, type: :numeric, default: 1536,
                                 desc: 'Vector dimensions (1536 for text-embedding-3-small, 3072 for large)'

data/lib/generators/woods/templates/add_pgvector_to_woods.rb.erb

@@ -1,15 +1,35 @@
-class AddPgvectorToWoods < ActiveRecord::Migration[7.0]
-  def change
+class AddPgvectorToWoods < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  # Creates the `woods_vectors` table read and written by
+  # Woods::Storage::VectorStore::Pgvector. The DDL mirrors the adapter's own
+  # idempotent ensure_schema! (CREATE ... IF NOT EXISTS), so it is safe to run
+  # whether or not Woods has already created the table at embed time.
+  #
+  # Raw SQL is used deliberately: the pgvector `vector` column type is not a
+  # native Rails column type, and this keeps the migration byte-for-byte in
+  # agreement with the adapter. Note: the `vector` type does not round-trip
+  # through schema.rb; use `config.active_record.schema_format = :sql`
+  # (structure.sql) if you need the dump to reproduce this table.
+  def up
     enable_extension 'vector' unless extension_enabled?('vector')
 
-    add_column :woods_embeddings, :embedding_vector, :vector,
-               limit: <%= @dimensions || 1536 %>, null: true
+    execute <<~SQL
+      CREATE TABLE IF NOT EXISTS woods_vectors (
+        id TEXT PRIMARY KEY,
+        embedding vector(<%= @dimensions || 1536 %>),
+        metadata JSONB DEFAULT '{}',
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+      )
+    SQL
 
     # HNSW index for fast approximate nearest neighbor search
-    # Using cosine distance operator (vector_cosine_ops)
-    add_index :woods_embeddings, :embedding_vector,
-              using: :hnsw,
-              opclass: :vector_cosine_ops,
-              name: 'idx_woods_embeddings_vector_hnsw'
+    # using the cosine distance operator (vector_cosine_ops).
+    execute <<~SQL
+      CREATE INDEX IF NOT EXISTS idx_woods_vectors_embedding_hnsw
+      ON woods_vectors USING hnsw (embedding vector_cosine_ops)
+    SQL
+  end
+
+  def down
+    drop_table :woods_vectors, if_exists: true
   end
 end

data/lib/generators/woods/templates/create_woods_tables.rb.erb

@@ -1,4 +1,8 @@
-class CreateWoodsTables < ActiveRecord::Migration[7.0]
+# Legacy compatibility migration. Woods 2's shipped extraction, storage, and
+# retrieval paths do not use these application tables. New default installs
+# should remove this file without running it; keep it only for an older/custom
+# integration that deliberately uses the tables.
+class CreateWoodsTables < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
     create_table :woods_units do |t|
       t.string :unit_type, null: false

data/lib/generators/woods/templates/woods.rb.tt

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 # Woods configuration
-# Full reference: https://github.com/lost-in-the/woods/blob/main/docs/CONFIGURATION_REFERENCE.md
+# Full reference: https://github.com/lost-in-the/woods/blob/v<%= Woods::VERSION %>/docs/CONFIGURATION_REFERENCE.md
 #
 # Quick-start presets (uncomment one instead of the full block below):
-#   Woods.configure_with_preset(:local)       # in-memory + Ollama, no external services
-#   Woods.configure_with_preset(:postgresql)  # pgvector + OpenAI (PostgreSQL required)
-#   Woods.configure_with_preset(:production)  # Qdrant + OpenAI (production-scale)
+#   Woods.configure_with_preset(:local)       # SQLite metadata + local Ollama (sqlite3 gem required)
+#   Woods.configure_with_preset(:shared_filesystem) # persisted files + local Ollama; no sqlite3
+#   :postgresql  # pgvector + OpenAI; requires sqlite3, API key, and connection options
+#   :production  # Qdrant + OpenAI; requires sqlite3, API key, URL, and collection
+# Copy the complete hosted-preset blocks from CONFIGURATION_REFERENCE.md;
+# the preset name alone cannot supply credentials or backend connection details.
 #
 # Presets accept a block for overrides:
 #   Woods.configure_with_preset(:local) { |c| c.max_context_tokens = 16_000 }
@@ -32,13 +35,8 @@ Woods.configure do |config|
 
   # ── Extractors ──────────────────────────────────────────────────────────
 
-  # Enabled extractors. Default set covers the most common Rails layers.
-  # See CONFIGURATION_REFERENCE.md for the full list of available symbols.
-  # config.extractors = %i[
-  #   models controllers services components view_components
-  #   jobs mailers graphql serializers managers policies validators
-  #   rails_source
-  # ]
+  # All built-in extractors run. `config.extractors` is accepted for forward
+  # compatibility but does not select a subset; non-default values warn.
 
   # Include Rails / gem source in the index (increases extraction time).
   # config.include_framework_sources = true
@@ -48,7 +46,8 @@ Woods.configure do |config|
 
   # ── Embedding ───────────────────────────────────────────────────────────
 
-  # Embedding provider: :openai or :ollama
+  # Embedding provider: :openai, :ollama, or :fake. `:fake` is deterministic
+  # and offline for tests/CI; it does not provide meaningful semantic ranking.
   # config.embedding_provider = :openai
   # config.embedding_model    = 'text-embedding-3-small'
   # config.embedding_options  = { api_key: ENV['OPENAI_API_KEY'] }
@@ -56,7 +55,8 @@ Woods.configure do |config|
   # Ollama (local, no API key needed). `num_ctx` is auto-selected per model
   # (nomic-embed-text → 2048, bge-m3 → 8192). Install `gem "tokenizers"` for
   # exact BERT WordPiece token counting on dense Ruby source. See
-  # docs/EMBEDDING_MODELS.md for the full model comparison.
+  # https://github.com/lost-in-the/woods/blob/v<%= Woods::VERSION %>/docs/EMBEDDING_MODELS.md
+  # has the full model comparison.
   # config.embedding_provider = :ollama
   # config.embedding_options  = {
   #   model: 'nomic-embed-text',
@@ -68,7 +68,9 @@ Woods.configure do |config|
   # Vector store: :in_memory, :pgvector (PostgreSQL), :qdrant
   # config.vector_store = :in_memory
 
-  # pgvector — run `rails generate woods:pgvector && rails db:migrate` first.
+  # pgvector — Woods creates its `woods_vectors` table automatically on first
+  # use (idempotent DDL). To manage the schema through your app's migrations
+  # instead, run `rails generate woods:pgvector && rails db:migrate`.
   # config.vector_store         = :pgvector
   # config.vector_store_options = {
   #   connection: ActiveRecord::Base.connection,
@@ -80,7 +82,8 @@ Woods.configure do |config|
   # config.vector_store_options = {
   #   url:        ENV.fetch('QDRANT_URL', 'http://localhost:6333'),
   #   collection: 'woods',
-  #   dimensions: 1_536
+  #   dimensions: 1_536,
+  #   allow_private_hosts: true # required for trusted localhost/RFC1918 URLs
   # }
 
   # Metadata store: :in_memory, :sqlite
@@ -97,41 +100,59 @@ Woods.configure do |config|
   # Extract link_to / redirect_to / form_action navigation edges.
   # config.extract_navigation_edges = true
 
-  # Temporal snapshots — requires migrations 004+005.
+  # Temporal snapshots prefer Woods' output-directory SQLite database and fall
+  # back to JSON when the sqlite3 gem is unavailable. Internal migrations run
+  # automatically; this does not require `rails db:migrate`.
   # config.enable_snapshots = false
 
   # ── Console MCP ─────────────────────────────────────────────────────────
   #
   # The Console MCP server lets AI tools query your live Rails app.
   # It is DISABLED by default. Enable only after reviewing the security
-  # documentation in docs/CONSOLE_MCP_SETUP.md.
+  # documentation:
+  # https://github.com/lost-in-the/woods/blob/v<%= Woods::VERSION %>/docs/CONSOLE_MCP_SETUP.md
   #
   # Defense layers (all active by default when the server is on):
-  #   Layer 1 — SqlValidator: rejects DML/DDL before any DB interaction.
-  #   Layer 2 — SafeContext: wraps every request in a rolled-back transaction;
-  #              writes are silently discarded even if Layer 1 is bypassed.
-  #   Layer 3 — Column redaction: credential columns are replaced with
-  #              [REDACTED] in every tool response.
+  #   Layer 1 — Table gate: rejects tool calls that touch a blocked table.
+  #   Layer 2 — Credential scanner: redacts credential-shaped content (API
+  #              keys, JWTs, etc.) from the response, by shape, not by name.
+  #   Layer 3 — Column + EAV redaction: named credential columns and
+  #              key/value pairs are replaced with [REDACTED].
+  #   Layer 4 — SqlValidator + SafeContext (always on, no config): rejects
+  #              DML/DDL before any DB interaction, and rolls back work on the
+  #              current database connection. It cannot contain jobs, mail,
+  #              HTTP calls, callbacks, threads, or other connections/shards.
 
   # config.console_mcp_enabled = false
   # config.console_mcp_path    = '/mcp/console'
 
+  # Console HTTP requires a strong bearer token. Its Origin/Host guard is
+  # loopback-only by default; list the public MCP host and any browser client
+  # origin for a deliberate non-loopback deployment.
+  # config.console_mcp_token = ENV.fetch('WOODS_CONSOLE_MCP_TOKEN')
+  # config.console_mcp_allowed_origins = ['https://rails.internal.example']
+
   # Credential-column redaction (Layer 3).
   # Starts from a safe default list (passwords, tokens, secrets).
-  # Extend: Woods::DEFAULT_CONSOLE_REDACTED_COLUMNS + %w[my_secret_col]
-  # Override entirely to remove a default:
-  # config.console_redacted_columns = %w[password token api_key]
+  # config.console_redacted_columns =
+  #   Woods::DEFAULT_CONSOLE_REDACTED_COLUMNS + %w[my_secret_col]
+  # Replace the defaults only after auditing the full list against your schema.
 
   # Key-value pairs where the value should be redacted (e.g., env var names).
   # config.console_redacted_key_values = []
 
-  # Tables completely blocked from queries.
-  # config.console_blocked_tables = []
+  # Tables completely blocked from queries (Layer 1).
+  # Starts from a safe default list (sessions, api_keys, credentials, ...).
+  # Extend: Woods::DEFAULT_CONSOLE_BLOCKED_TABLES + %w[my_secret_table]
+  # An empty array DISABLES this layer entirely — it does not mean "add
+  # nothing to the defaults". Override entirely only if you mean that:
+  # config.console_blocked_tables = %w[only_this_table]
 
   # Disable specific scanner patterns (rare — prefer blocked_tables).
   # config.console_disabled_scanner_patterns = []
 
-  # Allow the AI console to execute Ruby eval (off by default; very dangerous).
+  # Legacy unsafe-eval flag. `console_eval` is unavailable in supported modes;
+  # setting this to true fails closed at Console Server construction.
   # config.console_unsafe_eval_enabled = false
 
   # Expose SQL/query read tools inside embedded console (adds read-only DB