woods 1.4.1 → 1.6.0

data/lib/woods/console/sql_noise_stripper.rb

@@ -82,6 +82,119 @@ module Woods
         pattern = dialect == :mysql ? SINGLE_QUOTED_MYSQL : SINGLE_QUOTED_POSTGRES
         out.gsub(pattern, "''")
       end
+
+      # Strip BOTH comments and string literals in a single left-to-right
+      # pass, so a comment marker inside a literal and a quote inside a
+      # comment are each protected by whichever construct opens first.
+      #
+      # Running {.strip_comments} then {.strip_literals} (or vice versa) is
+      # unsafe: `SELECT '-- ' FROM blocked` has its real `FROM blocked`
+      # swallowed as a line comment (the `--` sits inside a string literal),
+      # letting a blocked table slip past {TableGate}; the reverse order
+      # mis-handles an apostrophe inside a `--` comment. Only a combined scan
+      # that tracks literal/comment state correctly resolves both. This
+      # scanner backs security checks (SqlValidator, TableGate) so it must
+      # never under-detect: an unterminated literal is treated as an ordinary
+      # character rather than swallowing the rest of the statement.
+      #
+      # @param sql [String] the SQL string to process
+      # @param dialect [Symbol] `:postgres` (default) or `:mysql` — controls
+      #   single-quote escape rules (see {.strip_literals}).
+      # @return [String] a new string with comments removed and every string
+      #   literal replaced by `''`
+      # @raise [ArgumentError] if an unsupported dialect is provided
+      def self.strip_noise(sql, dialect: :postgres) # rubocop:disable Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize
+        unless SUPPORTED_DIALECTS.include?(dialect)
+          raise ArgumentError, "Unknown dialect #{dialect.inspect}. Supported: #{SUPPORTED_DIALECTS.inspect}"
+        end
+
+        mysql = dialect == :mysql
+        out = +''
+        i = 0
+        len = sql.length
+
+        while i < len
+          ch = sql[i]
+
+          if ch == "'"
+            close = single_quote_end(sql, i, mysql: mysql)
+            if close
+              out << "''"
+              i = close
+            else
+              # Unterminated literal — never under-detect; keep the char.
+              out << ch
+              i += 1
+            end
+          elsif ch == '$' && (tag = dollar_tag_at(sql, i))
+            close = sql.index(tag, i + tag.length)
+            if close
+              out << "''"
+              i = close + tag.length
+            else
+              out << ch
+              i += 1
+            end
+          elsif ch == '-' && sql[i + 1] == '-'
+            nl = sql.index("\n", i)
+            i = nl || len
+          elsif ch == '/' && sql[i + 1] == '*'
+            close = sql.index('*/', i + 2)
+            if close
+              i = close + 2
+            else
+              # Unterminated block comment: never under-detect. Leave it in
+              # place (over-detection is safe; the old regex also required a
+              # closing */ and left an unterminated /* untouched).
+              out << ch
+              i += 1
+            end
+          else
+            out << ch
+            i += 1
+          end
+        end
+
+        out
+      end
+
+      # Regexp matching a PostgreSQL dollar-quote opening tag (`$$` or
+      # `$tag$`) at the start of the given slice.
+      DOLLAR_TAG = /\A\$\w*\$/
+      private_constant :DOLLAR_TAG
+
+      # Return the dollar-quote tag opening at +index+, or nil.
+      #
+      # @api private
+      def self.dollar_tag_at(sql, index)
+        m = DOLLAR_TAG.match(sql[index..])
+        m && m[0]
+      end
+      private_class_method :dollar_tag_at
+
+      # Return the index just past the closing quote of the single-quoted
+      # literal that opens at +start+, honoring `''` (both dialects) and `\'`
+      # (MySQL only) escapes. Returns nil when the literal is unterminated.
+      #
+      # @api private
+      def self.single_quote_end(sql, start, mysql:)
+        i = start + 1
+        len = sql.length
+        while i < len
+          c = sql[i]
+          if mysql && c == '\\'
+            i += 2
+          elsif c == "'"
+            return i + 1 unless sql[i + 1] == "'" # closing quote
+
+            i += 2 # doubled-quote escape — literal continues
+          else
+            i += 1
+          end
+        end
+        nil
+      end
+      private_class_method :single_quote_end
     end
   end
 end

data/lib/woods/console/sql_table_scanner.rb

@@ -109,23 +109,36 @@ module Woods
       # stripped before scanning. Both JOIN-style and ANSI-89 comma-join syntax
       # are handled.
       #
+      # Literals are stripped under BOTH supported dialects and the scans
+      # unioned. This scanner backs TableGate, so it may over-detect but must
+      # never under-detect: stripping with the wrong dialect's escape rules
+      # can swallow a real FROM clause — e.g. MySQL's `\'` escape applied on
+      # a PostgreSQL host (where backslash is literal under
+      # standard_conforming_strings) folds `'x\' FROM blocked WHERE y = '`
+      # into one literal, hiding `blocked` from the gate while PostgreSQL
+      # genuinely reads that table.
+      #
       # @param sql [String, nil] the SQL string to scan
-      # @return [Array<String>] identifiers in the order they were encountered;
-      #   may contain duplicates if the same table is referenced multiple times
+      # @return [Array<String>] identifiers in first-encounter order, deduplicated
       def self.identifiers_in(sql)
         return [] if sql.nil? || sql.empty?
 
-        stripped = strip_noise(sql)
         results = []
-        collect_join_identifiers(stripped, results)
-        collect_from_identifiers(stripped, results)
-        results
+        %i[postgres mysql].each do |dialect|
+          stripped = strip_noise(sql, dialect: dialect)
+          collect_join_identifiers(stripped, results)
+          collect_from_identifiers(stripped, results)
+        end
+        results.uniq
       end
 
       # @api private
-      def self.strip_noise(sql)
-        out = SqlNoiseStripper.strip_comments(sql)
-        SqlNoiseStripper.strip_literals(out, dialect: :mysql)
+      # Comments and literals must be stripped in a single combined pass —
+      # stripping them separately lets a comment marker inside a literal
+      # (`'-- '`) hide a real FROM clause from the gate. See
+      # {SqlNoiseStripper.strip_noise}.
+      def self.strip_noise(sql, dialect:)
+        SqlNoiseStripper.strip_noise(sql, dialect: dialect)
       end
       private_class_method :strip_noise
 

data/lib/woods/console/sql_validator.rb

@@ -152,8 +152,7 @@ module Woods
       # @param sql [String]
       # @return [Boolean]
       def contains_multiple_statements?(sql)
-        stripped = SqlNoiseStripper.strip_comments(sql)
-        stripped = SqlNoiseStripper.strip_literals(stripped)
+        stripped = SqlNoiseStripper.strip_noise(sql)
         stripped.include?(';')
       end
 

data/lib/woods/coordination/pipeline_lock.rb

@@ -2,6 +2,7 @@
 
 require 'fileutils'
 require 'json'
+require 'securerandom'
 
 module Woods
   module Coordination
@@ -38,12 +39,13 @@ module Woods
       def acquire
         FileUtils.mkdir_p(@lock_dir)
 
-        # Check for stale lock first (separate from atomic creation)
         if File.exist?(@lock_path)
           return false unless stale?
-
-          # Remove stale lock
-          FileUtils.rm_f(@lock_path)
+          # Retire the stale lock atomically. A bare rm_f + create here is
+          # a TOCTOU race: two processes passing the stale check together
+          # could each delete-and-create, the second deleting the first's
+          # FRESH lock — both would then "hold" it.
+          return false unless retire_stale_lock
         end
 
         # Atomic lock creation: File::EXCL ensures this fails if file already exists
@@ -58,10 +60,37 @@ module Woods
 
       # Release the lock.
       #
+      # Deletes the lock file only if it still carries this instance's
+      # token — a run that outlived stale_timeout may have been
+      # legitimately taken over, and deleting unconditionally would drop
+      # the new holder's lock.
+      #
       # @return [void]
       def release
-        FileUtils.rm_f(@lock_path) if @held
+        return unless @held
+
+        # Clear @held up front so no later failure can leave this instance
+        # believing it still holds the lock.
         @held = false
+
+        # Rename first, then inspect: a plain read-then-unlink is a TOCTOU —
+        # after we read our own token a takeover could replace the file, and
+        # our unlink would then delete the NEW holder's lock. Renaming
+        # atomically captures whatever is at the path.
+        graveyard = "#{@lock_path}.release.#{Process.pid}.#{SecureRandom.hex(4)}"
+        begin
+          File.rename(@lock_path, graveyard)
+        rescue Errno::ENOENT
+          return # already gone
+        end
+
+        if own_lock?(graveyard)
+          FileUtils.rm_f(graveyard)
+        else
+          # We were legitimately taken over — put the successor's lock back
+          # without clobbering a still-newer holder (see {#restore_lock}).
+          restore_lock(graveyard)
+        end
       end
 
       # Execute a block while holding the lock.
@@ -100,9 +129,85 @@ module Woods
         true
       end
 
-      # @return [String] Lock file content (JSON with PID and timestamp)
+      # Atomically retire a stale lock file via rename. Rename is atomic on
+      # POSIX: of any processes racing to take over the same stale lock,
+      # exactly one rename succeeds; the losers get ENOENT and back off.
+      # Winning the rename does NOT guarantee winning the lock — another
+      # process may O_EXCL-create between our rename and our create, which
+      # the caller's EEXIST rescue handles.
+      #
+      # Rename alone is not enough, though: a competitor that already passed
+      # `stale?` on the SAME original file may have retired it and created a
+      # FRESH lock before we run. Our rename would then move that fresh lock
+      # aside — and both processes would "hold" the lock. So after winning
+      # the rename we re-check the retired file's age; if it turns out to be
+      # fresh (someone beat us to the takeover), we put it back and lose the
+      # race instead of clobbering a live holder.
+      #
+      # @return [Boolean] true if this process retired a genuinely stale lock
+      def retire_stale_lock
+        graveyard = "#{@lock_path}.stale.#{Process.pid}.#{SecureRandom.hex(4)}"
+        File.rename(@lock_path, graveyard)
+
+        unless stale_file?(graveyard)
+          # We grabbed a lock that is no longer stale — a competitor already
+          # took over. Restore it (without clobbering a still-newer holder)
+          # and back off.
+          restore_lock(graveyard)
+          return false
+        end
+
+        FileUtils.rm_f(graveyard)
+        true
+      rescue Errno::ENOENT
+        false
+      end
+
+      # Whether the lock file at +path+ carries this instance's token.
+      #
+      # @param path [String]
+      # @return [Boolean] true when the token matches, or the file is corrupt
+      #   (an unparseable lock we already renamed aside is treated as ours to
+      #   discard rather than restore).
+      def own_lock?(path)
+        JSON.parse(File.read(path))['token'] == @token
+      rescue JSON::ParserError
+        true
+      end
+
+      # Put a lock file we renamed aside back at @lock_path WITHOUT clobbering
+      # a lock another process may have O_EXCL-created in the meantime.
+      # `File.link` is atomic and fails with EEXIST if @lock_path already
+      # exists, so a newer holder always wins; the aside copy is discarded
+      # either way. A plain `File.rename` back would overwrite that newer
+      # holder's lock — reintroducing a double-hold.
+      #
+      # @param graveyard [String] path of the renamed-aside lock file
+      # @return [void]
+      def restore_lock(graveyard)
+        File.link(graveyard, @lock_path)
+      rescue Errno::EEXIST
+        # A newer holder already claimed the path — our copy is obsolete.
+        nil
+      ensure
+        FileUtils.rm_f(graveyard)
+      end
+
+      # Whether the file at +path+ is older than the stale timeout.
+      #
+      # @param path [String]
+      # @return [Boolean]
+      def stale_file?(path)
+        Time.now - File.mtime(path) > @stale_timeout
+      rescue Errno::ENOENT
+        false
+      end
+
+      # @return [String] Lock file content (JSON with PID, timestamp, and
+      #   an ownership token release verifies before deleting)
       def lock_content
-        JSON.generate(pid: Process.pid, locked_at: Time.now.iso8601, name: @name)
+        @token = SecureRandom.hex(8)
+        JSON.generate(pid: Process.pid, locked_at: Time.now.iso8601, name: @name, token: @token)
       end
     end
   end

data/lib/woods/dependency_graph.rb

@@ -31,12 +31,20 @@ module Woods
       @to_h = nil
     end
 
-    # Register a unit in the graph
+    # Register a unit in the graph.
+    #
+    # Re-registering an identifier (incremental extraction registers into a
+    # graph loaded from disk) first removes the previous registration's
+    # reverse edges, file-map entry, and type-index entry — otherwise stale
+    # dependents accumulate across incremental runs and get persisted back
+    # to dependency_graph.json.
     #
     # @param unit [ExtractedUnit] The unit to register
     def register(unit)
       @to_h = nil
 
+      unregister(unit.identifier) if @nodes.key?(unit.identifier)
+
       @nodes[unit.identifier] = {
         type: unit.type,
         file_path: unit.file_path,
@@ -56,6 +64,35 @@ module Woods
       end
     end
 
+    # Remove an identifier's registration side effects: its contribution to
+    # the reverse indexes (derived from its recorded forward edges), its
+    # file-map entry, and its type-index entry. Forward node/edge data is
+    # overwritten by the caller (register), so it is not cleared here.
+    #
+    # @param identifier [String] Previously-registered unit identifier
+    # @return [void]
+    def unregister(identifier)
+      (@edges[identifier] || []).each do |edge|
+        if (set = @reverse[edge[:target]])
+          set.delete(identifier)
+          @reverse.delete(edge[:target]) if set.empty?
+        end
+
+        via_key = [edge[:target], edge[:via]]
+        next unless (set = @reverse_via[via_key])
+
+        set.delete(identifier)
+        @reverse_via.delete(via_key) if set.empty?
+      end
+
+      old_node = @nodes[identifier]
+      return unless old_node
+
+      old_path = old_node[:file_path]
+      @file_map.delete(old_path) if old_path && @file_map[old_path] == identifier
+      @type_index[old_node[:type]]&.delete(identifier)
+    end
+
     # Find all units affected by changes to given files
     # Uses BFS to find transitive dependents
     #

data/lib/woods/embedding/text_preparer.rb

@@ -95,7 +95,12 @@ module Woods
       def append_dependency_line(lines, dependencies)
         return unless dependencies&.any?
 
-        dep_names = dependencies.map { |d| d[:target] }.compact.first(10)
+        # Dependency hashes arrive symbol-keyed from the extractor's
+        # in-memory units but string-keyed from the indexer (Indexer#build_unit
+        # reads JSON and does not symbolize dependency keys, unlike chunks).
+        # Read both forms or the whole "dependencies:" prefix silently
+        # vanishes from every embedded document on the indexing path.
+        dep_names = dependencies.filter_map { |d| d[:target] || d['target'] }.first(10)
         lines << "dependencies: #{dep_names.join(', ')}" if dep_names.any?
       end
 

data/lib/woods/export/unit_facts.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Woods
+  # Shared building blocks for the export formatters (Unblocked DocumentBuilder,
+  # Obsidian NoteBuilder).
+  module Export
+    # Format-agnostic extraction of facts from a unit's metadata.
+    #
+    # Both the Unblocked and Obsidian exporters need the same *facts* about a
+    # unit — its associations grouped by macro, its schema highlights — but
+    # render them differently (plain text + GitHub URIs vs. wikilinks +
+    # frontmatter). This value object owns the one definition of "what a unit's
+    # metadata says"; each exporter is a thin formatter over it. The returned
+    # structures are deliberately raw (no truncation, no ordering imposed beyond
+    # the source array): formatting decisions — sort order, top-N caps — belong
+    # to each formatter so neither exporter's output is constrained by the other.
+    class UnitFacts
+      # @param unit [Hash] parsed unit JSON
+      def initialize(unit)
+        @meta = (unit && unit['metadata']) || {}
+      end
+
+      # Associations grouped by macro, each as a structured entry. Group keys are
+      # whatever macros are present (insertion order from the source array); the
+      # formatter picks the display order.
+      #
+      # @return [Hash{String=>Array<Hash>}] macro => [{ target:, dependent: }]
+      def associations_by_type
+        associations.group_by { |assoc| assoc['type'] }.transform_values do |items|
+          items.map { |a| { target: a['target'] || a['name'], dependent: a.dig('options', 'dependent') } }
+        end
+      end
+
+      # @return [Integer] total association count across all macros
+      def association_count
+        associations.size
+      end
+
+      # @return [Hash] { enums:, scopes:, concerns:, callbacks: } — raw, untruncated
+      def schema_highlights
+        { enums: enums, scopes: scopes, concerns: concerns, callbacks: callbacks }
+      end
+
+      def enums
+        @meta['enums'].is_a?(Hash) ? @meta['enums'] : {}
+      end
+
+      def scopes
+        @meta['scopes'].is_a?(Array) ? @meta['scopes'].filter_map { |s| s['name'] } : []
+      end
+
+      def concerns
+        @meta['inlined_concerns'].is_a?(Array) ? @meta['inlined_concerns'] : []
+      end
+
+      def callbacks
+        return [] unless @meta['callbacks'].is_a?(Array)
+
+        @meta['callbacks'].map { |cb| { type: cb['type'], filter: cb['filter'] } }
+      end
+
+      # Blast-radius bucket for a dependent count.
+      #
+      # @param dependent_count [Integer]
+      # @return [Symbol] :high (>50), :moderate (>20), or :none
+      def self.blast_radius(dependent_count)
+        return :high if dependent_count > 50
+        return :moderate if dependent_count > 20
+
+        :none
+      end
+
+      private
+
+      def associations
+        @meta['associations'].is_a?(Array) ? @meta['associations'] : []
+      end
+    end
+  end
+end