woods 2.0.0.beta2 → 2.0.0.beta4

data/lib/woods/console/sql_validator.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'woods/console/sql_noise_stripper'
+require 'woods/console/sqlite_read_guard'
 
 # @see Woods
 module Woods
@@ -28,11 +29,11 @@ module Woods
     # named constants), not imperative logic.
     class SqlValidator # rubocop:disable Metrics/ClassLength
       # SQL dialects whose normalization the lock-clause check can run over.
-      KNOWN_DIALECTS = %i[postgres mysql].freeze
+      KNOWN_DIALECTS = %i[postgres mysql sqlite].freeze
 
       # The dialect the statement will execute under, when the caller knows
       # it. `nil` (the default) keeps the conservative union: every check
-      # runs against both dialect normalizations, which can reject a
+      # runs against all supported dialect normalizations, which can reject a
       # statement that is valid under one dialect's quote grammar (a MySQL
       # `\'` escape hides prose that the PostgreSQL view reads as SQL). When
       # the execution boundary knows the adapter, passing the matching
@@ -206,9 +207,9 @@ module Woods
       # function calls.
       FUNCTION_SCAN_EXCLUDED_KEYWORDS = %w[
         IN EXISTS NOT AND OR VALUES WHERE HAVING ON IS BETWEEN CASE WHEN
-        THEN ELSE END FROM JOIN USING WITH SELECT DISTINCT ALL ANY SOME
-        UNION INTERSECT EXCEPT ORDER GROUP BY ASC DESC LIMIT OFFSET AS INTO
-        OVER PARTITION FILTER WITHIN RETURNING EXPLAIN
+        THEN ELSE FROM JOIN USING SELECT DISTINCT ALL ANY SOME
+        UNION INTERSECT EXCEPT ORDER GROUP BY LIMIT OFFSET AS INTO
+        OVER FILTER RETURNING EXPLAIN
       ].freeze
 
       # EXPLAIN is a statement leader, so `EXPLAIN (FORMAT JSON) SELECT` is an
@@ -318,6 +319,7 @@ module Woods
 
         return validate_dialect_variants!(sql) if unknown_grammar?
 
+        SqliteReadGuard.validate!(sql) if dialect == :sqlite
         normalized = sql.strip
 
         # Reject multiple statements (semicolons not inside string literals)
@@ -582,10 +584,9 @@ module Woods
           match = Regexp.last_match
           quoted = !(match[1] || match[2]).nil?
           identifier = match[1] || match[2] || match[3]
-          # A bare keyword before `(` is grammar (IN/EXISTS/…), not a call; a
-          # quoted name before `(` is always a call, so keyword exclusion
-          # applies only to the bare form.
-          next if !quoted && FUNCTION_SCAN_EXCLUDED_KEYWORDS.include?(identifier.upcase)
+          # Some engines also allow keyword spellings as function names.
+          # Only exempt those names in their supported grammar positions.
+          next if !quoted && function_keyword_grammar?(identifier.upcase, stripped[0...match.begin(0)])
           next if ALLOWED_FUNCTIONS.include?(identifier.downcase)
 
           raise SqlValidationError,
@@ -594,6 +595,45 @@ module Woods
         end
       end
 
+      # Recognize keyword grammar without exempting identically named calls.
+      # The prefix is already stripped of comments and literal contents.
+      #
+      # @param keyword [String] Uppercase bare identifier before `(`.
+      # @param prefix [String] SQL preceding the identifier.
+      # @return [Boolean]
+      def function_keyword_grammar?(keyword, prefix)
+        return false unless FUNCTION_SCAN_EXCLUDED_KEYWORDS.include?(keyword)
+
+        case keyword
+        when 'EXPLAIN' then prefix.strip.empty?
+        when 'BY' then prefix.match?(/\b(?:ORDER|GROUP|PARTITION)\s*\z/i)
+        when 'OVER', 'FILTER' then prefix.match?(/\)\s*\z/)
+        when 'ANY', 'SOME' then comparison_quantifier?(prefix)
+        when 'OFFSET' then offset_grammar?(prefix)
+        else true
+        end
+      end
+
+      # SQLite has no quantified ANY/SOME comparison grammar; both spellings
+      # can instead invoke application-defined functions after an operator.
+      def comparison_quantifier?(prefix)
+        return true if dialect == :postgres # Reserved quantifiers, including LIKE ANY.
+
+        dialect != :sqlite && prefix.match?(/[=<>]\s*\z/)
+      end
+
+      # PostgreSQL reserves bare OFFSET (it cannot name a function), so its
+      # standalone OFFSET clause remains supported. Other dialects only accept
+      # parenthesized offsets after a completed LIMIT operand. Calls
+      # at an expression's start or after an operator are never offset syntax.
+      # Keep the supported operand boundary conservative: literals, bind
+      # numbers, parenthesized expressions, or PostgreSQL's LIMIT ALL.
+      def offset_grammar?(prefix)
+        return true if dialect == :postgres
+
+        prefix.match?(/(?:[0-9)'"`]|\bLIMIT\s+ALL)\s*\z/i)
+      end
+
       # Check if the SQL contains a forbidden keyword at a statement-leader
       # position after stripping comments and string literals. This catches
       # comment-hidden injections like "SELECT 1 --;\nDELETE FROM users",

data/lib/woods/console/sqlite_read_guard.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'woods/console/sql_noise_stripper'
+require 'woods/console/sql_table_scanner'
+
+module Woods
+  module Console
+    # Refuse SQLite-specific syntax outside the scanner's supported grammar.
+    # This is a conservative read boundary, not a general-purpose SQL parser.
+    module SqliteReadGuard
+      SIMPLE_IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/
+      UNSUPPORTED_SYNTAX = /[^\x00-\x7F]|\$|\[|''\s*\(|\b(?:FROM|JOIN)(?=['"`(])/i
+      QUOTED_IDENTIFIER = /"(?:[^"]|"")*"|`(?:[^`]|``)*`/
+
+      # @param sql [String] SQL to execute on SQLite
+      # @raise [SqlValidationError] when an identifier or table factor cannot be checked
+      # @return [void]
+      def self.validate!(sql)
+        view = SqlNoiseStripper.strip_noise(sql, dialect: :sqlite)
+        refuse! if view.match?(UNSUPPORTED_SYNTAX)
+        view.scan(QUOTED_IDENTIFIER) { |quoted| refuse! unless quoted[1...-1].match?(SIMPLE_IDENTIFIER) }
+        SqlTableScanner.relation_factors(view).each do |factor|
+          refuse! unless supported_factor?(factor.strip)
+        end
+      end
+
+      # A SELECT/WITH subquery has its own independently scanned table factors.
+      # Parenthesized table groups and string-quoted names are refused rather
+      # than silently omitted from the blocked-table scan.
+      def self.supported_factor?(factor)
+        return true if factor.match?(/\A\(\s*(?:SELECT|WITH)\b/i)
+
+        match = SqlTableScanner::LEAD_IDENT.match(factor)
+        match && factor[match.end(0)..].match?(/\A(?:\s|,|\)|\z)/)
+      end
+      private_class_method :supported_factor?
+
+      def self.refuse!
+        raise SqlValidationError,
+              'Rejected: unsupported SQLite identifier or table-reference syntax. ' \
+              'Use simple bare or double-quoted identifiers and SELECT subqueries.'
+      end
+      private_class_method :refuse!
+    end
+  end
+end

data/lib/woods/coordination/pipeline_lock.rb

@@ -30,8 +30,9 @@ module Woods
       # The transaction-guard filename for a lock of +name+, exposed so
       # cleanup code that empties a lock directory (woods:clean) can skip the
       # guard during its sweep — deleting a flock'd guard out from under a
-      # contender's critical section would split the flock across two inodes —
-      # and remove it only after the lock is released.
+      # contender's critical section would split the flock across two inodes.
+      # The guard stays after cleanup and lock release: a new contender may
+      # already be holding it before its lock file exists.
       #
       # @param name [String]
       # @return [String]

data/lib/woods/dependency_graph.rb

@@ -32,9 +32,14 @@ module Woods
     # enforce_dependencies: package units (Task 7)
     # commit_count, change_frequency: git enrichment (Task 5)
     NODE_ATTRIBUTE_KEYS = %i[
-      database table foreign_key_tables package enforce_dependencies commit_count change_frequency
+      database table foreign_key_tables package enforce_dependencies commit_count change_frequency kind
     ].freeze
 
+    # These extractors describe a whole source file rather than one constant.
+    # Other types remain unclassified: a path-like identifier is not evidence
+    # that a unit is a profile, and an unmarked unit need not name a constant.
+    FILE_PROFILE_TYPES = %i[caching configuration test_mapping rails_source gem_source].freeze
+
     # Optional edge keys beyond target and via. `through` is the through
     # association name; `through_db` is the through model's resolved
     # database, emitted only when present; `disable_joins` is emitted only
@@ -80,14 +85,14 @@ module Woods
       }.merge(node_attributes_from(unit))
 
       (@edges[unit.identifier] ||= {})[unit.type] =
-        unit.dependencies.map { |d| { target: d[:target], via: d[:via] }.merge(self.class.edge_attributes(d)) }
+        self.class.normalize_edges(unit.dependencies, strict: true)
       (@file_map[unit.file_path] ||= Set.new).add(unit.identifier) if unit.file_path
 
       # Type index for filtering (Set-based for O(1) insert)
       (@type_index[unit.type] ||= Set.new).add(unit.identifier)
 
       # Build reverse edges (Set-based for O(1) insert)
-      unit.dependencies.each do |dep|
+      @edges[unit.identifier][unit.type].each do |dep|
         (@reverse[dep[:target]] ||= Set.new).add(unit.identifier)
         (@reverse_via[[dep[:target], dep[:via]]] ||= Set.new).add(unit.identifier)
       end
@@ -645,7 +650,7 @@ module Woods
     # @return [Hash{Symbol => Object}] normalized, nil-free
     def node_attributes_from(unit)
       metadata = unit.respond_to?(:metadata) && unit.metadata.is_a?(Hash) ? unit.metadata : {}
-      attrs = {}
+      attrs = self.class.file_profile_attributes(unit.type)
       attrs[:database] = metadata[:database] unless metadata[:database].nil?
       attrs[:table] = metadata[:table_name] if unit.type == :model && !metadata[:table_name].nil?
       tables = Array(metadata[:foreign_keys]).filter_map { |fk| fk[:to_table] || fk['to_table'] if fk.is_a?(Hash) }
@@ -729,14 +734,15 @@ module Woods
     # `nodes` and `edges` stay keyed on the bare identifier, holding the
     # **primary** node — the type that sorts first. Identifiers registered
     # under more than one type put their remaining nodes in `variants`, a flat
-    # array that is **omitted entirely when empty**. So the serialized form of
-    # a graph with no collisions is byte-for-byte what it has always been, and
-    # a `dependency_graph.json` written before this change loads through
+    # array that is **omitted entirely when empty**. A
+    # `dependency_graph.json` written before this change loads through
     # {.from_h} unmodified: it simply carries no `variants`.
     #
     # `reverse`, `file_map` and `type_index` need no new shape. They are
     # already identifier-valued sets, so a Scenic view `reports` and a factory
     # `reports` each contribute to their own type bucket and their own path.
+    # The additive `reverse_via` index preserves typed source identities and
+    # complete relationship records without changing the legacy `reverse` map.
     #
     # @return [Hash] Complete graph data
     def to_h
@@ -747,6 +753,7 @@ module Woods
           nodes: key_sorted(self.class.relativize_nodes(primary_nodes, root)),
           edges: key_sorted(primary_edges),
           reverse: key_sorted(@reverse.transform_values { |ids| ids.to_a.sort }),
+          reverse_via: published_reverse_via,
           file_map: key_sorted(self.class.relativize_file_map(@file_map, root)),
           type_index: key_sorted(@type_index.transform_values { |ids| ids.to_a.sort }),
           stats: {
@@ -770,6 +777,23 @@ module Woods
     end
     private :key_sorted
 
+    # Reverse buckets are derived from every typed forward registration, rather
+    # than the bare-name in-memory filter index, which cannot preserve variants
+    # or distinguish association attributes. Legacy nil relationships stay nil.
+    def published_reverse_via
+      buckets = {}
+      @edges.each do |source, by_type|
+        by_type.each do |type, edges|
+          edges.each do |edge|
+            record = { source: source, source_type: type }.merge(edge.except(:target))
+            (buckets[edge[:target]] ||= []) << record
+          end
+        end
+      end
+      key_sorted(buckets.transform_values { |records| records.sort_by { |record| JSON.generate(record) } })
+    end
+    private :published_reverse_via
+
     # A copy whose edge containers are the caller's alone.
     #
     # `dup` copies only the top level, so `to_h[:edges][id]` used to be the
@@ -783,6 +807,9 @@ module Woods
     def detached_snapshot(memo)
       snapshot = memo.dup
       snapshot[:edges] = memo[:edges].transform_values { |list| list.map(&:dup) }
+      snapshot[:reverse_via] = memo[:reverse_via].transform_values do |records|
+        records.map { |record| record.transform_values { |value| value.is_a?(String) ? value.dup : value } }
+      end
       if memo.key?(:variants)
         snapshot[:variants] = memo[:variants].map do |record|
           record.merge(edges: record[:edges].map(&:dup))
@@ -794,7 +821,12 @@ module Woods
 
     # @return [Hash{String => Hash}] identifier => primary node
     def primary_nodes
-      @nodes.transform_values { |nodes| primary_of(nodes) }
+      # Late enrichment and JSON loading can insert optional fields in a
+      # different order. Canonicalize like variant_records so a graph's JSON
+      # fingerprint is stable across full and incremental publication.
+      @nodes.transform_values do |nodes|
+        primary_of(nodes).slice(:type, :file_path, :namespace, *NODE_ATTRIBUTE_KEYS)
+      end
     end
     private :primary_nodes
 
@@ -969,10 +1001,17 @@ module Woods
     # @param node [Hash]
     # @return [Hash{Symbol => Object}]
     def self.persisted_node_attributes(node)
-      NODE_ATTRIBUTE_KEYS.each_with_object({}) do |key, attrs|
+      attributes = NODE_ATTRIBUTE_KEYS.each_with_object({}) do |key, attrs|
         value = node.key?(key) ? node[key] : node[key.to_s]
         attrs[key] = normalize_node_attribute(key, value) unless value.nil?
       end
+      attributes.merge(file_profile_attributes(node[:type] || node['type']))
+    end
+
+    # Backfill the additive marker when an older graph is republished, so
+    # unchanged nodes in an incremental run agree with a fresh full graph.
+    def self.file_profile_attributes(type)
+      FILE_PROFILE_TYPES.include?(type&.to_sym) ? { kind: 'file_profile' } : {}
     end
 
     # Edge attributes present in a dependency or persisted edge hash.
@@ -1032,7 +1071,10 @@ module Woods
       graph.instance_variable_set(:@edges, edges)
 
       raw_reverse = data[:reverse] || data['reverse'] || {}
-      graph.instance_variable_set(:@reverse, raw_reverse.transform_values { |v| v.is_a?(Set) ? v : Set.new(v) })
+      reverse = raw_reverse.each_with_object({}) do |(target, sources), index|
+        (index[target.to_s] ||= Set.new).merge(sources)
+      end
+      graph.instance_variable_set(:@reverse, reverse)
 
       raw_file_map = data[:file_map] || data['file_map'] || {}
       graph.instance_variable_set(:@file_map, absolutize_file_map(normalize_file_map(raw_file_map), root))
@@ -1126,7 +1168,12 @@ module Woods
       }.merge(persisted_node_attributes(node))
     end
 
-    # Normalize edge data from either old format (bare strings) or new format (hashes).
+    # Normalize fresh dependencies and persisted edges to the same identity:
+    # string targets and symbol relationship labels. Extractors may emit
+    # symbolic external targets (such as :http_api); retaining those symbols
+    # after a JSON-loaded baseline splits reverse indexes on re-registration.
+    # Unit dependency metadata stays unchanged; only graph records normalize.
+    # Also accepts the old bare-string edge format.
     #
     # ROUND-TRIP INVARIANT (do not break when refactoring):
     #   DependencyGraph#to_h -> JSON.generate -> JSON.parse -> DependencyGraph.from_h
@@ -1147,15 +1194,20 @@ module Woods
     # explicit data conversion.
     #
     # @param edges [Array] Edge entries — either strings or hashes
+    # @param strict [Boolean] require hashes for fresh units; legacy loads stay tolerant
     # @return [Array<Hash>] Normalized edges with :target and :via keys
-    def self.normalize_edges(edges)
+    def self.normalize_edges(edges, strict: false)
+      raise ArgumentError, 'Fresh graph dependencies must be an array' if strict && !edges.is_a?(Array)
+
       return [] unless edges.is_a?(Array)
 
       edges.map do |edge|
+        raise ArgumentError, 'Fresh graph dependencies must be hashes' if strict && !edge.is_a?(Hash)
+
         if edge.is_a?(String)
           { target: edge, via: nil }
         elsif edge.is_a?(Hash)
-          { target: edge[:target] || edge['target'], via: (edge[:via] || edge['via'])&.to_sym }
+          { target: (edge[:target] || edge['target'])&.to_s, via: (edge[:via] || edge['via'])&.to_sym }
             .merge(edge_attributes(edge))
         else
           { target: edge.to_s, via: nil }

data/lib/woods/embedding/corpus.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../atomic_file'
+require_relative '../generation'
+
+module Woods
+  class Error < StandardError; end unless defined?(Woods::Error)
+
+  module Embedding
+    # Collect the complete extraction input before an embedding run can mutate
+    # stores. Native publications use their authoritative listings, never a glob
+    # that could mistake a missing/corrupt payload for intentional deletion.
+    class Corpus
+      class Incomplete < Woods::Error; end
+
+      def initialize(output_dir)
+        @output_dir = output_dir.to_s
+      end
+
+      def load
+        native? ? published_units : legacy_units
+      rescue IOError, SystemCallError, JSON::ParserError, EncodingError, ArgumentError => e
+        raise Incomplete, "Embedding input incomplete: #{e.message}. Repair or rebuild extraction before embedding."
+      end
+
+      private
+
+      def native?
+        path = File.join(@output_dir, Generation::FILENAME)
+        if File.exist?(path)
+          marker = JSON.parse(AtomicFile.read(path))
+          raise IOError, 'invalid generation marker' unless marker.is_a?(Hash)
+        end
+
+        if !marker || marker['payload'].nil?
+          if File.directory?(File.join(@output_dir, 'payloads'))
+            raise IOError, 'missing publication pointer beside payloads'
+          end
+
+          return false
+        end
+
+        validate_marker(marker)
+        true
+      end
+
+      def validate_marker(marker)
+        strings_valid = %w[payload token].all? { |key| marker[key].is_a?(String) && !marker[key].empty? }
+        return if strings_valid && marker['number'].is_a?(Integer) && marker['number'].positive?
+
+        raise IOError, 'invalid published generation marker'
+      end
+
+      def published_units
+        require_relative '../mcp/index_reader'
+
+        reader = MCP::IndexReader.new(@output_dir)
+        reader.with_pinned_generation do
+          # Generation's compatibility fallback to the root is useful for
+          # readers, but cannot prove completeness for destructive reconciliation.
+          if reader.payload_dir.expand_path == Pathname.new(@output_dir).expand_path
+            raise IOError,
+                  'published payload missing or invalid'
+          end
+
+          validate_counts(reader.manifest)
+          reader.each_unit.to_a
+        end
+      end
+
+      def validate_counts(manifest)
+        counts = manifest.is_a?(Hash) && manifest['counts']
+        valid = counts.is_a?(Hash) && counts.all? do |dir, count|
+          MCP::IndexReader::TYPE_DIRS.include?(dir) && count.is_a?(Integer) && count >= 0
+        end
+        raise IOError, 'invalid published manifest counts' unless valid
+      end
+
+      # Pre-pointer indexes may use arbitrary filenames and have no manifest
+      # or authoritative listing. Keep that input contract, but never silently
+      # omit unreadable JSON before reconciling persisted identities.
+      def legacy_units
+        Dir.glob(File.join(@output_dir, '**', '*.json')).filter_map do |path|
+          relative = path.delete_prefix("#{@output_dir}/")
+          next if relative.start_with?('dumps/', 'payloads/') || File.basename(path) == 'checkpoint.json'
+
+          data = JSON.parse(AtomicFile.read(path))
+          data if data.is_a?(Hash) && data.key?('type') && data.key?('identifier')
+        end
+      end
+    end
+  end
+end