woods 1.6.1 → 2.0.0.beta2

data/lib/woods/session_tracer/store.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'base64'
+
 module Woods
   module SessionTracer
     # Abstract store interface for session trace data.
@@ -60,7 +62,18 @@ module Woods
       # @param session_id [String] Raw session identifier
       # @return [String] Sanitized identifier (alphanumeric, hyphens, underscores only)
       def sanitize_session_id(session_id)
-        session_id.to_s.gsub(/[^a-zA-Z0-9_-]/, '_')
+        raw = session_id.to_s
+        raise ArgumentError, 'session_id exceeds 180 bytes' if raw.bytesize > 180
+
+        "b64.#{Base64.urlsafe_encode64(raw, padding: false)}"
+      end
+
+      def restore_session_id(encoded)
+        return encoded unless encoded.start_with?('b64.')
+
+        Base64.urlsafe_decode64(encoded.delete_prefix('b64.'))
+      rescue ArgumentError
+        encoded
       end
 
       # Build a session summary hash from a session ID and its requests.

data/lib/woods/storage/metadata_store.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'fileutils'
 require 'time'
 
 module Woods
@@ -18,11 +19,20 @@ module Woods
     # its methods. The {SQLite} adapter is provided for local persistence.
     #
     # @example Using the SQLite adapter
-    #   store = Woods::Storage::MetadataStore::SQLite.new(":memory:")
+    #   store = Woods::Storage::MetadataStore::SQLite.new(database: ":memory:")
     #   store.store("User", { type: "model", file_path: "app/models/user.rb" })
     #   store.find("User")
     #
     module MetadataStore
+      # Conservative whitelist for caller-supplied search field names —
+      # mirrors the filter-key standard in {Woods::Storage::Pgvector#build_where}.
+      # The SQLite adapter interpolates field names into a
+      # `json_extract(data, '$.<field>')` JSON-path literal, so anything
+      # outside this charset must be rejected before any SQL is built: a
+      # crafted field name (quotes, parens, `--`) can otherwise break out of
+      # the string literal and alter the query shape.
+      SEARCH_FIELD_NAME = /\A[a-zA-Z_][a-zA-Z0-9_]*\z/
+
       # Interface that all metadata store adapters must implement.
       module Interface
         # Store or update metadata for a unit.
@@ -68,6 +78,30 @@ module Woods
 
         # Search metadata by text query across specified fields.
         #
+        # **Contract every adapter must honour** (pinned by the
+        # `'hardened search'` shared examples in
+        # spec/storage/metadata_store_spec.rb, which run against every
+        # adapter — they were substitutable in name only until the semantics
+        # were written down):
+        #
+        # - **Substring match**, not word or prefix match.
+        # - **Case-insensitive.** InMemory used a case-sensitive
+        #   `String#include?` while SQLite used `LIKE`, so the same query
+        #   returned different results depending on the configured backend.
+        #   Case-insensitive is both the search-like expectation and what the
+        #   durable adapter already did. (SQLite's `LIKE` folds ASCII only, so
+        #   non-ASCII case folding remains backend-specific — do not rely on
+        #   it either way.)
+        # - **`fields: nil` searches the whole record**, including keys, as
+        #   serialized JSON. A query can therefore match a field *name*.
+        # - **LIKE metacharacters are literal.** `%` and `_` in a query match
+        #   themselves rather than acting as wildcards.
+        # - Field names are validated against {SEARCH_FIELD_NAME} by every
+        #   adapter, so a hostile name raises the same ArgumentError
+        #   everywhere.
+        # - **An empty `fields` list matches nothing.** Adapters return an
+        #   empty result before querying their backing store.
+        #
         # @param query [String] Text to search for
         # @param fields [Array<String>, nil] Specific fields to search (nil = all)
         # @return [Array<Hash>] Matching metadata records
@@ -76,6 +110,21 @@ module Woods
           raise NotImplementedError
         end
 
+        # Every stored identifier.
+        #
+        # Used by the evaluation harness's {Woods::Evaluation::BaselineRunner}
+        # to build naive baselines (grep/random/file-level) to score the real
+        # retrieval pipeline against. It called this on the configured store
+        # while no adapter implemented it, so every baseline strategy raised
+        # NoMethodError — invisible because the harness had no working entry
+        # point either (#212).
+        #
+        # @return [Array<String>] All identifiers, order unspecified
+        # @raise [NotImplementedError] if not implemented by adapter
+        def all_identifiers
+          raise NotImplementedError
+        end
+
         # Delete a unit by ID.
         #
         # @param id [String] The identifier to delete
@@ -91,6 +140,30 @@ module Woods
         def count
           raise NotImplementedError
         end
+
+        private
+
+        # Validate caller-supplied search field names against
+        # {SEARCH_FIELD_NAME}, stringifying as it goes (symbol fields were
+        # always accepted). Every adapter's {#search} must run its `fields:`
+        # through this BEFORE touching its backing store, so a hostile field
+        # name raises the same {ArgumentError} everywhere — including
+        # adapters (InMemory) where the name is not an injection vector,
+        # keeping the adapters substitutable.
+        #
+        # @param fields [Array<String, Symbol>, nil] Field names, or nil for "all fields"
+        # @return [Array<String>, nil] Stringified field names, or nil if fields was nil
+        # @raise [ArgumentError] if any field name fails the whitelist
+        def validate_search_fields!(fields)
+          return nil if fields.nil?
+
+          fields.map do |field|
+            field_s = field.to_s
+            raise ArgumentError, "Invalid search field: #{field_s.inspect}" unless field_s.match?(SEARCH_FIELD_NAME)
+
+            field_s
+          end
+        end
       end
 
       # Pure-Ruby metadata store backed by a hash. No external dependencies,
@@ -113,7 +186,7 @@ module Woods
 
         # @see Interface#store
         def store(id, metadata)
-          @data[id] = stringify_keys(metadata).merge('updated_at' => Time.now.iso8601)
+          @data[id] = normalize(metadata).merge('updated_at' => Time.now.iso8601)
         end
 
         # @see Interface#find
@@ -143,16 +216,43 @@ module Woods
         end
 
         # @see Interface#search
+        #
+        # Matching is literal substring inclusion — `%` and `_` in the query
+        # have no special meaning here, and the SQLite adapter escapes them
+        # so the two adapters agree.
+        #
+        # @raise [ArgumentError] if a field name fails {SEARCH_FIELD_NAME}
         def search(query, fields: nil)
-          needle = query.to_s
+          fields = validate_search_fields!(fields)
+          return [] if fields == []
+
+          # Case-insensitive, matching the SQLite adapter's `LIKE` (see the
+          # contract note on {Interface#search}). This used to be a
+          # case-sensitive `String#include?`, so the same query returned
+          # different results depending on which backend a host had configured.
+          needle = query.to_s.downcase
           @data.each_with_object([]) do |(id, record), out|
-            haystacks = fields ? fields.map { |f| record[f.to_s] } : [JSON.generate(record)]
-            next unless haystacks.compact.any? { |h| h.to_s.include?(needle) }
+            # `updated_at` is excluded from the whole-record haystack: SQLite's
+            # `data` column never carries it (it's a store-level column there,
+            # not part of the JSON blob), so leaving it in here made a query
+            # that only matched a timestamp (e.g. "2026-08") return every
+            # record on InMemory and none on SQLite.
+            haystacks = if fields
+                          fields.map { |f| field_haystack(record[f]) }
+                        else
+                          [JSON.generate(record.except('updated_at'))]
+                        end
+            next unless haystacks.compact.any? { |h| h.downcase.include?(needle) }
 
             out << record.except('updated_at').merge('id' => id)
           end
         end
 
+        # @see Interface#all_identifiers
+        def all_identifiers
+          @data.keys
+        end
+
         # @see Interface#delete
         def delete(id)
           @data.delete(id)
@@ -196,13 +296,37 @@ module Woods
 
         private
 
-        # Match the SQLite adapter's string-key contract regardless of how
-        # the caller serialises the input hash. Without this, find/search
-        # consumers that expect string keys (the SQLite path round-trips
-        # through JSON, which always returns strings) would break under
-        # symbol-keyed test fixtures.
-        def stringify_keys(hash)
-          hash.each_with_object({}) { |(k, v), out| out[k.to_s] = v }
+        # Match the SQLite adapter's contract regardless of how the caller
+        # serialises the input hash.
+        #
+        # The SQLite path stores +JSON.generate(metadata)+ and reads it back
+        # with +JSON.parse+, so every key *and value* comes back as JSON
+        # types: symbols become strings, nested hashes are stringified all the
+        # way down. Stringifying only the top-level keys left the two adapters
+        # returning different values for the same input (STO-8), so the round
+        # trip is performed here too rather than approximated.
+        #
+        # @param metadata [Hash]
+        # @return [Hash] string-keyed, JSON-typed values
+        def normalize(metadata)
+          JSON.parse(JSON.generate(metadata))
+        end
+
+        # The text SQLite's +json_extract(data, '$.field')+ would compare
+        # against for one field value: strings come back raw, structured
+        # values as their JSON text, scalars as their decimal form. A Ruby
+        # +Hash#to_s+ haystack used to leak `=>` and `:sym` syntax that no
+        # JSON document contains (STO-8).
+        #
+        # @param value [Object] the stored field value
+        # @return [String, nil] nil for a missing field, which never matches
+        def field_haystack(value)
+          case value
+          when nil then nil
+          when String then value
+          when Hash, Array then JSON.generate(value)
+          else value.to_s
+          end
         end
       end
 
@@ -212,15 +336,19 @@ module Woods
       # for efficient filtering. Uses upsert semantics for store operations.
       #
       # @example
-      #   store = SQLite.new(":memory:")
+      #   store = SQLite.new(database: ":memory:")
       #   store.store("User", { type: "model", namespace: "Admin" })
       #   store.find("User")  # => { "type" => "model", "namespace" => "Admin" }
       #
-      class SQLite
+      class SQLite # rubocop:disable Metrics/ClassLength
         include Interface
 
-        # @param db_path [String] Path to the SQLite database file, or ":memory:" for in-memory
-        def initialize(db_path = ':memory:')
+        # Attempts allowed for a contended write, including the first.
+        # Mirrors {Woods::Temporal::SnapshotStore::LOCK_ATTEMPTS}.
+        LOCK_ATTEMPTS = 3
+
+        # @param database [String, Pathname] Database file, or ":memory:" for an explicit in-memory store
+        def initialize(database:)
           begin
             require 'sqlite3'
           rescue LoadError
@@ -229,21 +357,42 @@ module Woods
                   "Add `gem 'sqlite3'` and re-bundle, or set " \
                   "`config.metadata_store = :in_memory` if you don't need cross-process persistence."
           end
-          @db = ::SQLite3::Database.new(db_path)
+          database = database.to_s
+          FileUtils.mkdir_p(File.dirname(File.expand_path(database))) unless database == ':memory:'
+          @db = ::SQLite3::Database.new(database)
           @db.results_as_hash = true
+          # SQLite's busy handler only helps when it is set. Two processes
+          # sharing one metadata database (a long extraction while an MCP
+          # server stores a checkpoint) used to raise SQLite3::BusyException
+          # immediately on the second writer (O2). Same value as the twin
+          # {Woods::Temporal::SnapshotStore}.
+          @db.busy_timeout = 5_000
           create_table
         end
 
         # @see Interface#store
+        # @raise [ArgumentError] if +metadata+ carries no type key
+        #
+        # The type column backs {#find_by_type}, so an absent key used to
+        # fall through +type.to_s+ and be stored as +""+ — fabricating a
+        # type where none existed rather than surfacing the missing field
+        # (L22). A present-but-blank type fabricates exactly the same empty
+        # column, so it is refused on the same grounds (STO-9).
         def store(id, metadata)
           type = metadata[:type] || metadata['type']
+          if type.nil? || type.to_s.strip.empty?
+            raise ArgumentError, "metadata for #{id.inspect} has no usable type key (got #{type.inspect})"
+          end
+
           data = JSON.generate(metadata)
 
-          @db.execute(<<~SQL, [id, type.to_s, data, Time.now.iso8601])
-            INSERT INTO units (id, type, data, updated_at) VALUES (?, ?, ?, ?)
-            ON CONFLICT(id) DO UPDATE SET
-              type = excluded.type, data = excluded.data, updated_at = excluded.updated_at
-          SQL
+          with_lock_retry do
+            @db.execute(<<~SQL, [id, type.to_s, data, Time.now.iso8601])
+              INSERT INTO units (id, type, data, updated_at) VALUES (?, ?, ?, ?)
+              ON CONFLICT(id) DO UPDATE SET
+                type = excluded.type, data = excluded.data, updated_at = excluded.updated_at
+            SQL
+          end
         end
 
         # @see Interface#find
@@ -272,21 +421,42 @@ module Woods
         end
 
         # @see Interface#search
+        #
+        # Field names are interpolated into a `json_extract` JSON-path
+        # literal, so they are validated against {SEARCH_FIELD_NAME} first —
+        # a crafted name could otherwise break out of the literal and alter
+        # the SQL shape. LIKE metacharacters (`%`, `_`, `\`) in the query are
+        # escaped (with an explicit ESCAPE clause) so they match literally,
+        # aligning with the InMemory adapter's substring semantics instead of
+        # silently broadening matches.
+        #
+        # @raise [ArgumentError] if a field name fails the whitelist
         def search(query, fields: nil)
+          fields = validate_search_fields!(fields)
+          return [] if fields == []
+
+          pattern = "%#{escape_like(query)}%"
           if fields
-            conditions = fields.map { "json_extract(data, '$.#{_1}') LIKE ?" }.join(' OR ')
-            params = fields.map { "%#{query}%" }
+            conditions = fields.map { "json_extract(data, '$.#{_1}') LIKE ? ESCAPE '\\'" }.join(' OR ')
+            params = Array.new(fields.size, pattern)
             rows = @db.execute("SELECT id, data FROM units WHERE #{conditions}", params)
           else
-            rows = @db.execute('SELECT id, data FROM units WHERE data LIKE ?', ["%#{query}%"])
+            rows = @db.execute("SELECT id, data FROM units WHERE data LIKE ? ESCAPE '\\'", [pattern])
           end
 
           rows.map { |row| parse_row(row) }
         end
 
+        # @see Interface#all_identifiers
+        def all_identifiers
+          @db.execute('SELECT id FROM units').map { |row| row['id'] }
+        end
+
         # @see Interface#delete
         def delete(id)
-          @db.execute('DELETE FROM units WHERE id = ?', [id])
+          with_lock_retry do
+            @db.execute('DELETE FROM units WHERE id = ?', [id])
+          end
         end
 
         # @see Interface#count
@@ -296,6 +466,40 @@ module Woods
 
         private
 
+        # Bounded retry around a contended write, mirroring
+        # {Woods::Temporal::SnapshotStore#with_lock_retry}: SQLite skips the
+        # busy handler when it detects a lock-order deadlock, so a write can
+        # raise "database is locked" at once despite the busy timeout. A
+        # short bounded retry lets the other writer finish; anything else,
+        # or a non-SQLite connection, raises straight through.
+        def with_lock_retry
+          attempt = 0
+          begin
+            attempt += 1
+            yield
+          rescue StandardError => e
+            raise unless sqlite_busy?(e) && attempt < LOCK_ATTEMPTS
+
+            sleep(0.05 * attempt)
+            retry
+          end
+        end
+
+        def sqlite_busy?(error)
+          defined?(SQLite3::BusyException) && error.is_a?(SQLite3::BusyException)
+        end
+
+        # Escape SQL LIKE metacharacters in a user query so they match
+        # literally under the `ESCAPE '\'` clause {#search} emits. Without
+        # this, `%` and `_` in a query act as wildcards and silently broaden
+        # matches (`"user_name"` would match `"userXname"`).
+        #
+        # @param query [String] Raw search query
+        # @return [String] Query safe for embedding in a LIKE pattern
+        def escape_like(query)
+          query.to_s.gsub(/[\\%_]/) { |ch| "\\#{ch}" }
+        end
+
         # Parse a database row into a metadata hash with the id field injected.
         #
         # @param row [Hash] Database row with 'id' and 'data' keys