woods 1.6.1 → 2.0.0.beta2

data/lib/woods/temporal/json_snapshot_store.rb

@@ -3,6 +3,9 @@
 require 'json'
 require 'time'
 require 'digest'
+require 'fileutils'
+require_relative '../atomic_file'
+require_relative '../payload_store'
 
 module Woods
   module Temporal
@@ -12,8 +15,16 @@ module Woods
     # of the index output directory. Each file is named by git SHA and contains
     # manifest metadata plus per-unit content hashes.
     #
+    # Retention matches the payload store (audit P8): capture prunes the
+    # oldest snapshots beyond the configured count, keyed off the same
+    # WOODS_PAYLOAD_RETENTION env var. Old histories answering `diff` and
+    # `unit_history` beyond the retention window degrade to empty.
+    #
     # Implements the same public interface as SnapshotStore so the MCP server
     # tools work identically.
+    # Malformed retained snapshots are warned about and treated as absent:
+    # +find+ returns nil, +diff+ returns an empty result, and history/list scans
+    # omit the corrupt file.
     #
     # @example
     #   store = JsonSnapshotStore.new(dir: '/app/tmp/woods')
@@ -22,8 +33,9 @@ module Woods
     #   store.diff("abc123", "def456") # => { added: [...], modified: [...], deleted: [...] }
     #
     class JsonSnapshotStore # rubocop:disable Metrics/ClassLength
-      def initialize(dir:)
+      def initialize(dir:, retention: nil)
         @dir = File.join(dir, 'snapshots')
+        @retention = resolve_retention(retention)
         FileUtils.mkdir_p(@dir)
       end
 
@@ -34,7 +46,11 @@ module Woods
         # reach snapshot_path, which raises ArgumentError on a non-hex SHA.
         return nil unless git_sha.is_a?(String) && git_sha.match?(/\A[0-9a-f]+\z/i)
 
-        previous = find_latest
+        # Exclude the SHA being captured, mirroring the SQLite store: a
+        # re-capture at an unchanged HEAD must diff against the latest
+        # distinct snapshot, not whatever this SHA's own file used to hold
+        # (L20).
+        previous = find_latest(exclude_sha: git_sha)
         snapshot = build_snapshot(manifest, git_sha, unit_hashes)
 
         if previous
@@ -45,6 +61,7 @@ module Woods
         end
 
         write_snapshot(git_sha, snapshot)
+        prune_snapshots(protect: git_sha)
         snapshot.except(:units)
       end
 
@@ -58,7 +75,9 @@ module Woods
         path = snapshot_path(git_sha)
         return nil unless File.exist?(path)
 
-        data = JSON.parse(File.read(path))
+        data = read_snapshot(path)
+        return nil unless data
+
         symbolize_snapshot(data).except(:units)
       end
 
@@ -77,26 +96,69 @@ module Woods
                     .reverse
                     .first(limit)
 
-        entries = snapshots.filter_map do |snap|
-          unit = snap[:units]&.[](identifier)
-          next unless unit
-
-          {
-            git_sha: snap[:git_sha],
-            extracted_at: snap[:extracted_at],
-            git_branch: snap[:git_branch],
-            unit_type: unit[:unit_type],
-            source_hash: unit[:source_hash],
-            metadata_hash: unit[:metadata_hash],
-            dependencies_hash: unit[:dependencies_hash]
-          }
-        end
+        entries = snapshots.flat_map do |snap|
+          snap[:units].values.select { |unit| unit[:identifier] == identifier }.map do |unit|
+            unit.except(:identifier).merge(
+              git_sha: snap[:git_sha], extracted_at: snap[:extracted_at], git_branch: snap[:git_branch]
+            )
+          end
+        end.first(limit)
 
         mark_changed_entries(entries)
       end
 
       private
 
+      # Resolve the retention count: an explicit positive value wins, then
+      # the WOODS_PAYLOAD_RETENTION env var the payload store honors, then
+      # the payload store's default.
+      #
+      # @param retention [Integer, nil]
+      # @return [Integer]
+      def resolve_retention(retention)
+        return retention if retention.is_a?(Integer) && retention.positive?
+
+        value = ENV.fetch('WOODS_PAYLOAD_RETENTION', nil).to_i
+        value.positive? ? value : PayloadStore::DEFAULT_RETENTION
+      end
+
+      # Delete snapshots beyond the retention count, oldest by extracted_at
+      # first. +protect+ names the snapshot just captured; it is the newest
+      # anyway, and a tie on extracted_at must not delete it.
+      #
+      # Failures are non-fatal: retention is housekeeping, and a failed
+      # prune leaves the store growing as it did before rather than
+      # breaking capture.
+      #
+      # @param protect [String] git SHA of the just-captured snapshot
+      # @return [void]
+      def prune_snapshots(protect:)
+        summaries = load_all_summaries
+        overflow = summaries.size - @retention
+        return unless overflow.positive?
+
+        retention_victims(summaries, overflow, protect).each { |sha| FileUtils.rm_f(snapshot_path(sha)) }
+      rescue StandardError => e
+        warn "[Woods] Snapshot retention failed: #{e.message}"
+      end
+
+      # Oldest `overflow` snapshots by extracted_at, never the protected one.
+      # The protected SHA is rejected before sorting and slicing: it is
+      # captured last, so its extracted_at can tie or precede older entries,
+      # and letting it occupy the victim slice would leave the store one file
+      # over its bound with no replacement victim selected.
+      #
+      # @param summaries [Array<Hash>]
+      # @param overflow [Integer]
+      # @param protect [String] git SHA of the just-captured snapshot
+      # @return [Array<String>]
+      def retention_victims(summaries, overflow, protect)
+        summaries.reject { |summary| summary[:git_sha] == protect }
+                 .sort_by { |summary| summary[:extracted_at] || '' }
+                 .first(overflow)
+                 .map { |summary| summary[:git_sha] }
+      end
+
       def mget(hash, key)
         hash[key] || hash[key.to_sym]
       end
@@ -124,7 +186,8 @@ module Woods
           id = mget(uh, 'identifier')
           next if id.nil?
 
-          [id, {
+          [JSON.generate([id, mget(uh, 'type').to_s]), {
+            identifier: id,
             unit_type: mget(uh, 'type').to_s,
             source_hash: mget(uh, 'source_hash'),
             metadata_hash: mget(uh, 'metadata_hash'),
@@ -144,27 +207,26 @@ module Woods
             if data_a[:source_hash] != data_b[:source_hash] ||
                data_a[:metadata_hash] != data_b[:metadata_hash] ||
                data_a[:dependencies_hash] != data_b[:dependencies_hash]
-              modified << { identifier: identifier, unit_type: data_b[:unit_type] }
+              modified << { identifier: data_b[:identifier], unit_type: data_b[:unit_type] }
             end
           else
-            added << { identifier: identifier, unit_type: data_b[:unit_type] }
+            added << { identifier: data_b[:identifier], unit_type: data_b[:unit_type] }
           end
         end
 
         units_a.each do |identifier, data_a|
-          deleted << { identifier: identifier, unit_type: data_a[:unit_type] } unless units_b.key?(identifier)
+          deleted << { identifier: data_a[:identifier], unit_type: data_a[:unit_type] } unless units_b.key?(identifier)
         end
 
         { added: added, modified: modified, deleted: deleted }
       end
 
       def mark_changed_entries(entries)
-        entries.each_with_index do |entry, i|
-          entry[:changed] = if i == entries.size - 1
-                              true
-                            else
-                              entry[:source_hash] != entries[i + 1][:source_hash]
-                            end
+        entries.group_by { |entry| entry[:unit_type] }.each_value do |versions|
+          versions.each_with_index do |entry, index|
+            older = versions[index + 1]
+            entry[:changed] = older.nil? || entry[:source_hash] != older[:source_hash]
+          end
         end
         entries
       end
@@ -176,37 +238,43 @@ module Woods
       end
 
       def write_snapshot(git_sha, data)
-        File.write(snapshot_path(git_sha), JSON.pretty_generate(data))
+        AtomicFile.write(snapshot_path(git_sha), JSON.pretty_generate(data))
       end
 
       def load_snapshot_with_units(git_sha)
         path = snapshot_path(git_sha)
         return nil unless File.exist?(path)
 
-        symbolize_snapshot(JSON.parse(File.read(path)))
+        data = read_snapshot(path)
+        data && symbolize_snapshot(data)
       end
 
       def load_all_summaries
         Dir.glob(File.join(@dir, '*.json')).filter_map do |path|
-          data = JSON.parse(File.read(path))
-          symbolize_snapshot(data).except(:units)
-        rescue JSON::ParserError => e
-          warn "[Woods] Skipping corrupt snapshot #{File.basename(path)}: #{e.message}"
-          nil
+          data = read_snapshot(path)
+          symbolize_snapshot(data).except(:units) if data
         end
       end
 
       def load_all_with_units
         Dir.glob(File.join(@dir, '*.json')).filter_map do |path|
-          symbolize_snapshot(JSON.parse(File.read(path)))
-        rescue JSON::ParserError => e
-          warn "[Woods] Skipping corrupt snapshot #{File.basename(path)}: #{e.message}"
-          nil
+          data = read_snapshot(path)
+          symbolize_snapshot(data) if data
         end
       end
 
-      def find_latest
+      def read_snapshot(path)
+        JSON.parse(AtomicFile.read(path))
+      rescue JSON::ParserError => e
+        warn "[Woods] Skipping corrupt snapshot #{File.basename(path)}: #{e.message}"
+        nil
+      end
+
+      # @param exclude_sha [String, nil] SHA to leave out of the result
+      # @return [Hash, nil]
+      def find_latest(exclude_sha: nil)
         snapshots = load_all_summaries
+        snapshots.reject! { |s| s[:git_sha] == exclude_sha } if exclude_sha
         return nil if snapshots.empty?
 
         latest = snapshots.max_by { |s| s[:extracted_at] || '' }
@@ -234,13 +302,15 @@ module Woods
       def symbolize_units(units)
         return {} unless units
 
-        units.transform_values do |v|
-          {
+        units.to_h do |key, v|
+          identifier = v['identifier'] || key
+          [JSON.generate([identifier, v['unit_type']]), {
+            identifier: identifier,
             unit_type: v['unit_type'],
             source_hash: v['source_hash'],
             metadata_hash: v['metadata_hash'],
             dependencies_hash: v['dependencies_hash']
-          }
+          }]
         end
       end
     end

data/lib/woods/temporal/snapshot_store.rb

@@ -29,6 +29,7 @@ module Woods
       #   construct the store with a bare mock.
       def initialize(connection:, validate_schema: true)
         @db = connection
+        @db.busy_timeout = 5_000 if @db.respond_to?(:busy_timeout=)
         validate_schema! if validate_schema
       end
 
@@ -69,8 +70,10 @@ module Woods
       def schema_error_message(detail)
         'SnapshotStore requires the `woods_snapshots` and ' \
           '`woods_snapshot_units` tables (migrations 004 + 005 under ' \
-          '`lib/woods/db/migrations/`). Run `rake woods:migrate` on the ' \
-          "metadata DB and retry. Underlying error: #{detail}"
+          '`lib/woods/db/migrations/`). These run automatically on woods-mcp ' \
+          'boot (Bootstrapper.build_snapshot_store auto-migrates), or run them ' \
+          'directly with `Woods::Db::Migrator.new(connection: db).migrate!`. ' \
+          "Underlying error: #{detail}"
       end
 
       public
@@ -80,6 +83,13 @@ module Woods
       # Stores the manifest metadata and per-unit content hashes.
       # Computes diff stats vs. the most recent previous snapshot.
       #
+      # Re-capturing at an unchanged HEAD updates the snapshot row **in
+      # place** — the row id is stable across captures (#206), so the
+      # per-unit DELETE below genuinely clears the previous capture's unit
+      # set instead of leaking it. Unit rows orphaned by the pre-#206
+      # INSERT OR REPLACE (which minted a fresh id per capture) are swept
+      # on every capture so historical leaks self-heal.
+      #
       # @param manifest [Hash] The manifest data (string or symbol keys)
       # @param unit_hashes [Array<Hash>] Per-unit content hashes
       # @return [Hash] Snapshot record with diff stats
@@ -90,15 +100,53 @@ module Woods
         # collide a snapshot row. Mirrors JsonSnapshotStore's path validation.
         return nil unless git_sha.is_a?(String) && git_sha.match?(/\A[0-9a-f]+\z/i)
 
-        previous = find_latest
-        upsert_snapshot(manifest, git_sha, unit_hashes.size)
+        captured = nil
+        with_lock_retry do
+          @db.transaction(:immediate) do
+            # Exclude the SHA being captured: find_latest before the upsert
+            # otherwise resolves "previous" to this very row, and
+            # update_diff_stats then compares the freshly written unit set
+            # against itself — every stat zeroes out silently (L20).
+            previous = find_latest(exclude_sha: git_sha)
+            upsert_snapshot(manifest, git_sha, unit_hashes.size)
+
+            snapshot_id = fetch_snapshot_id(git_sha)
+            @db.execute('DELETE FROM woods_snapshot_units WHERE snapshot_id = ?', [snapshot_id])
+            prune_orphaned_units
+            insert_unit_hashes(snapshot_id, unit_hashes)
+
+            update_diff_stats(snapshot_id, previous)
+            captured = find(git_sha)
+          end
+        end
+        captured
+      end
 
-        snapshot_id = fetch_snapshot_id(git_sha)
-        @db.execute('DELETE FROM woods_snapshot_units WHERE snapshot_id = ?', [snapshot_id])
-        insert_unit_hashes(snapshot_id, unit_hashes)
+      # Attempts for {#with_lock_retry}, including the first.
+      LOCK_ATTEMPTS = 3
+
+      # SQLite skips the busy handler when it detects a lock-order deadlock
+      # (this connection holds SHARED and asks for RESERVED while another
+      # already holds RESERVED), so `BEGIN IMMEDIATE` can raise "database is
+      # locked" at once despite busy_timeout. Two extractions capturing into
+      # one metadata database hit that on CI. A short bounded retry lets the
+      # other writer finish; anything else, or a non-SQLite connection,
+      # raises straight through.
+      def with_lock_retry
+        attempt = 0
+        begin
+          attempt += 1
+          yield
+        rescue StandardError => e
+          raise unless sqlite_busy?(e) && attempt < LOCK_ATTEMPTS
+
+          sleep(0.05 * attempt)
+          retry
+        end
+      end
 
-        update_diff_stats(snapshot_id, previous)
-        find(git_sha)
+      def sqlite_busy?(error)
+        defined?(SQLite3::BusyException) && error.is_a?(SQLite3::BusyException)
       end
 
       # List snapshots, optionally filtered by branch.
@@ -193,12 +241,11 @@ module Woods
       # @param entries [Array<Hash>]
       # @return [Array<Hash>]
       def mark_changed_entries(entries)
-        entries.each_with_index do |entry, i|
-          entry[:changed] = if i == entries.size - 1
-                              true # Oldest version is always "changed" (first appearance)
-                            else
-                              entry[:source_hash] != entries[i + 1][:source_hash]
-                            end
+        entries.group_by { |entry| entry[:unit_type] }.each_value do |versions|
+          versions.each_with_index do |entry, index|
+            older = versions[index + 1]
+            entry[:changed] = older.nil? || entry[:source_hash] != older[:source_hash]
+          end
         end
         entries
       end
@@ -212,7 +259,37 @@ module Woods
         hash[key] || hash[key.to_sym]
       end
 
-      # Insert or replace the snapshot row from manifest data.
+      # Upsert SQL for the snapshot row, keyed on UNIQUE(git_sha).
+      #
+      # Deliberately `ON CONFLICT ... DO UPDATE` (SQLite >= 3.24 — the same
+      # syntax `Storage::MetadataStore` already relies on), **not**
+      # `INSERT OR REPLACE`: REPLACE resolves the conflict by deleting the
+      # old row and inserting a fresh one with a new AUTOINCREMENT id, so
+      # {#capture}'s subsequent `DELETE ... WHERE snapshot_id = ?` matched
+      # nothing and every re-extraction at an unchanged HEAD leaked a full
+      # unit-hash set — unboundedly, since foreign keys are never enabled on
+      # this connection (#206).
+      UPSERT_SNAPSHOT_SQL = <<~SQL
+        INSERT INTO woods_snapshots
+          (git_sha, git_branch, extracted_at, rails_version, ruby_version,
+           total_units, unit_counts, gemfile_lock_sha, schema_sha)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        ON CONFLICT(git_sha) DO UPDATE SET
+          git_branch = excluded.git_branch,
+          extracted_at = excluded.extracted_at,
+          rails_version = excluded.rails_version,
+          ruby_version = excluded.ruby_version,
+          total_units = excluded.total_units,
+          unit_counts = excluded.unit_counts,
+          gemfile_lock_sha = excluded.gemfile_lock_sha,
+          schema_sha = excluded.schema_sha
+      SQL
+
+      private_constant :UPSERT_SNAPSHOT_SQL
+
+      # Insert the snapshot row, or update it in place when this git SHA was
+      # captured before. The row id is preserved either way (see
+      # {UPSERT_SNAPSHOT_SQL}).
       #
       # @param manifest [Hash]
       # @param git_sha [String]
@@ -230,11 +307,22 @@ module Woods
           mget(manifest, 'gemfile_lock_sha'),
           mget(manifest, 'schema_sha')
         ]
-        @db.execute(<<~SQL, params)
-          INSERT OR REPLACE INTO woods_snapshots
-            (git_sha, git_branch, extracted_at, rails_version, ruby_version,
-             total_units, unit_counts, gemfile_lock_sha, schema_sha)
-          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        @db.execute(UPSERT_SNAPSHOT_SQL, params)
+      end
+
+      # Delete unit rows whose snapshot row no longer exists.
+      #
+      # Before #206, every re-capture of an unchanged SHA stranded the
+      # previous capture's unit rows under a snapshot id REPLACE had deleted
+      # (foreign keys are never enabled on this connection, so nothing else
+      # cleans them up). One cheap sweep per capture lets databases that
+      # accumulated those leaks self-heal.
+      #
+      # @return [void]
+      def prune_orphaned_units
+        @db.execute(<<~SQL)
+          DELETE FROM woods_snapshot_units
+          WHERE snapshot_id NOT IN (SELECT id FROM woods_snapshots)
         SQL
       end
 
@@ -251,11 +339,23 @@ module Woods
         )
       end
 
-      # Find the most recent snapshot.
+      # Find the most recent snapshot, optionally excluding one SHA.
       #
+      # {#capture} always excludes the SHA it is about to upsert — see the
+      # note at the call site (L20).
+      #
+      # @param exclude_sha [String, nil] SHA to leave out of the result
       # @return [Hash, nil]
-      def find_latest
-        row = @db.get_first_row('SELECT * FROM woods_snapshots ORDER BY extracted_at DESC LIMIT 1')
+      def find_latest(exclude_sha: nil)
+        row =
+          if exclude_sha
+            @db.get_first_row(
+              'SELECT * FROM woods_snapshots WHERE git_sha != ? ORDER BY extracted_at DESC LIMIT 1',
+              [exclude_sha]
+            )
+          else
+            @db.get_first_row('SELECT * FROM woods_snapshots ORDER BY extracted_at DESC LIMIT 1')
+          end
         return nil unless row
 
         row_to_hash(row)
@@ -281,20 +381,16 @@ module Woods
           VALUES (?, ?, ?, ?, ?, ?)
         SQL
 
-        # Wrap in a transaction to batch all inserts into a single commit,
-        # reducing per-row fsync overhead from O(n) to O(1).
-        @db.transaction do
-          unit_hashes.each do |uh|
-            params = [
-              snapshot_id,
-              mget(uh, 'identifier'),
-              mget(uh, 'type').to_s,
-              mget(uh, 'source_hash'),
-              mget(uh, 'metadata_hash'),
-              mget(uh, 'dependencies_hash')
-            ]
-            @db.execute(sql, params)
-          end
+        unit_hashes.each do |uh|
+          params = [
+            snapshot_id,
+            mget(uh, 'identifier'),
+            mget(uh, 'type').to_s,
+            mget(uh, 'source_hash'),
+            mget(uh, 'metadata_hash'),
+            mget(uh, 'dependencies_hash')
+          ]
+          @db.execute(sql, params)
         end
       end
 
@@ -310,7 +406,8 @@ module Woods
         rows = @db.execute(sql, [snapshot_id])
 
         rows.to_h do |row|
-          [row['identifier'], {
+          [[row['identifier'], row['unit_type']], {
+            identifier: row['identifier'],
             unit_type: row['unit_type'],
             source_hash: row['source_hash'],
             metadata_hash: row['metadata_hash'],
@@ -337,16 +434,16 @@ module Woods
             if data_a[:source_hash] != data_b[:source_hash] ||
                data_a[:metadata_hash] != data_b[:metadata_hash] ||
                data_a[:dependencies_hash] != data_b[:dependencies_hash]
-              modified << { identifier: identifier, unit_type: data_b[:unit_type] }
+              modified << { identifier: data_b[:identifier], unit_type: data_b[:unit_type] }
             end
           else
-            added << { identifier: identifier, unit_type: data_b[:unit_type] }
+            added << { identifier: data_b[:identifier], unit_type: data_b[:unit_type] }
           end
         end
 
         # Units in A but not B → deleted
         units_a.each do |identifier, data_a|
-          deleted << { identifier: identifier, unit_type: data_a[:unit_type] } unless units_b.key?(identifier)
+          deleted << { identifier: data_a[:identifier], unit_type: data_a[:unit_type] } unless units_b.key?(identifier)
         end
 
         { added: added, modified: modified, deleted: deleted }