woods 1.5.0 → 1.6.0

data/lib/woods/obsidian/vault_exporter.rb

@@ -0,0 +1,412 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'json'
+require 'yaml'
+require 'pathname'
+require 'woods'
+require 'woods/atomic_file'
+require_relative 'errors'
+require_relative 'name_mapper'
+require_relative 'note_builder'
+require_relative 'vault_assets'
+
+module Woods
+  module Obsidian
+    # Orchestrates exporting Woods extraction output to a self-contained
+    # Obsidian vault.
+    #
+    # Reads the flat extraction directory via IndexReader (using raw_graph_data,
+    # so string keys + persisted pagerank), then writes:
+    #   - one Markdown note per exported unit (NoteBuilder),
+    #   - per-type `_index.md` MOCs + a root `_Overview.md`,
+    #   - a `_woods/` machine sidecar (id<->path manifest + verbatim graph copies),
+    #   - `.obsidian/` config + `Units.base` (only into a woods-owned vault),
+    #   - a `.woods-vault` ownership sentinel.
+    # Finally it sweeps stale woods-managed notes — guarded so it can never
+    # delete a user's own notes or run against a foreign vault.
+    #
+    # All dependency edges come from one in-memory edge_map (the graph's
+    # edges/reverse); per-unit JSON is read only for note bodies.
+    #
+    # @example
+    #   Woods::Obsidian::VaultExporter.new(
+    #     index_dir: "tmp/woods", vault_path: "tmp/woods/obsidian_vault"
+    #   ).export_all
+    #   # => { exported: 412, indexes: 19, swept: 3, skipped: 0, errors: [] }
+    class VaultExporter
+      MAX_ERRORS = 100
+      PURGE_GUARD_FRACTION = 0.30
+      PURGE_GUARD_MIN_DOCS = 10
+      SENTINEL = '.woods-vault'
+      SIDECAR_DIR = '_woods'
+      FRAMEWORK_TYPES = %w[rails_source gem_source].freeze
+      # Upper bound on frontmatter lines scanned when identifying a managed note
+      # during the sweep — our frontmatter is ~12 lines; this just bounds work on
+      # a malformed file that opens with "---" but never closes the fence.
+      MAX_FRONTMATTER_LINES = 200
+
+      # @param index_dir [String] extraction output directory to read
+      # @param vault_path [String] directory to write the vault into
+      # @param config [Configuration] Woods configuration (default: global)
+      # @param reader [Object, nil] IndexReader (auto-created if nil)
+      # @param include_source [Boolean] embed credential-scrubbed source in notes
+      # @param include_framework [Boolean] include rails_source units
+      # @param force_purge [Boolean] bypass the mass-deletion sweep guard
+      # @param output [IO] progress stream
+      def initialize(index_dir:, vault_path:, config: Woods.configuration, reader: nil,
+                     include_source: false, include_framework: false, force_purge: false, output: $stdout)
+        @config = config
+        @vault = Pathname.new(vault_path.to_s)
+        @reader = reader || build_reader(index_dir)
+        @include_source = include_source
+        @include_framework = include_framework
+        @force_purge = force_purge
+        @output = output
+      end
+
+      # Generate the full vault. Idempotent: re-running against an unchanged
+      # extraction produces byte-identical output.
+      #
+      # @return [Hash] { exported:, indexes:, swept:, skipped:, errors: }
+      def export_all
+        graph = @reader.raw_graph_data || {}
+        @nodes = graph['nodes'] || {}
+        @pagerank = graph['pagerank'] || {}
+        analysis = safe_graph_analysis
+        owned = vault_owned_at_start?
+
+        emitted, skipped = partition_emitted
+        mapper = build_mapper(emitted)
+        edge_map = build_edge_map(emitted.to_set, graph)
+        builder = build_note_builder(mapper, analysis)
+
+        written = Set.new
+        errors = []
+        exported = write_notes(emitted, mapper, edge_map, builder, written, errors)
+        indexes = write_indexes(emitted, mapper, written)
+        write_sidecar(emitted, mapper, graph, analysis, written)
+        write_owned_assets(emitted) if owned
+
+        swept = owned && errors.empty? ? sweep(written) : 0
+        warn_foreign unless owned
+        { exported: exported, indexes: indexes, swept: swept, skipped: skipped, errors: cap_errors(errors) }
+      end
+
+      private
+
+      # ── Selection / mapping ──────────────────────────────────────────
+
+      # The emitted set = graph nodes of an exportable type that have a loadable
+      # unit file. Confirming the file loads (not just that the index lists it)
+      # guarantees every wikilink target is backed by a real note — a node listed
+      # in the index but with no readable unit is dropped (counted as skipped).
+      #
+      # @return [Array(Array<String>, Integer)] [sorted emitted ids, skipped count]
+      def partition_emitted
+        known = known_ids
+        candidates = @nodes.keys.select do |id|
+          type = @nodes[id]['type']
+          next false if !@include_framework && FRAMEWORK_TYPES.include?(type)
+
+          known.include?(id)
+        end.sort
+
+        # Load each candidate once and keep it: confirms the unit is readable
+        # (a corrupt/missing file is skipped, not fatal) and lets Pass 2 render
+        # from this cache instead of re-reading every file past the reader's
+        # small LRU — avoiding ~2x disk reads + JSON parses on a large vault.
+        # Holding the units for one batch export is an acceptable memory trade.
+        @units = {}
+        skipped = 0
+        candidates.each do |id|
+          unit = load_unit(id)
+          unit ? (@units[id] = unit) : (skipped += 1)
+        end
+        [@units.keys, skipped]
+      end
+
+      # Load a unit, treating an unreadable/corrupt file as absent rather than
+      # letting one bad JSON parse abort the whole export.
+      def load_unit(id)
+        @reader.find_unit(id)
+      rescue StandardError => e
+        log "  skipped #{id}: #{e.class}: #{e.message}"
+        nil
+      end
+
+      def known_ids
+        @known_ids ||= @reader.list_units.to_set { |entry| entry['identifier'] }
+      end
+
+      def build_mapper(emitted)
+        NameMapper.new(emitted.to_h { |id| [id, dir_for(@nodes[id]['type'])] })
+      end
+
+      def dir_for(type)
+        require 'woods/mcp/index_reader'
+        Woods::MCP::IndexReader::TYPE_TO_DIR[type] || "#{type}s"
+      end
+
+      def build_note_builder(mapper, analysis)
+        NoteBuilder.new(name_mapper: mapper, nodes: @nodes, pagerank: @pagerank,
+                        analysis: analysis, include_source: @include_source, scanner: scanner)
+      end
+
+      # The single edge source: per-id forward (depends_on) and reverse (used_by)
+      # edges from the graph, filtered to the emitted set, self-edges dropped.
+      def build_edge_map(emitted, graph)
+        edges = graph['edges'] || {}
+        reverse = graph['reverse'] || {}
+        emitted.each_with_object({}) do |id, map|
+          deps = Array(edges[id]).filter_map { |edge| forward_edge(edge, id, emitted) }
+                                 .uniq { |edge| [edge[:target], edge[:via]] }
+          used = Array(reverse[id]).select { |sid| sid != id && emitted.include?(sid) }.uniq
+          map[id] = { depends_on: deps, used_by: used }
+        end
+      end
+
+      def forward_edge(edge, source_id, emitted)
+        edge = { 'target' => edge } unless edge.is_a?(Hash)
+        target = edge['target']
+        return nil if target == source_id || !emitted.include?(target)
+
+        { target: target, via: edge['via'] }
+      end
+
+      # ── Pass 2: notes ────────────────────────────────────────────────
+
+      def write_notes(emitted, mapper, edge_map, builder, written, errors)
+        exported = 0
+        emitted.each do |id|
+          unit = @units[id]
+          next unless unit # defensive — partition_emitted already cached loadable units
+
+          edges = edge_map[id] || { depends_on: [], used_by: [] }
+          note = builder.build(id: id, unit: unit, depends_on: edges[:depends_on], used_by: edges[:used_by])
+          next if note.nil?
+
+          rel = mapper.path_for(id)
+          AtomicFile.write(@vault.join(rel), note)
+          written << rel
+          exported += 1
+        rescue StandardError => e
+          errors << "#{id}: #{e.class}: #{e.message}"
+        end
+        exported
+      end
+
+      # ── Pass 3a: MOC + overview ──────────────────────────────────────
+
+      def write_indexes(emitted, mapper, written)
+        by_dir = emitted.group_by { |id| File.dirname(mapper.path_for(id)) }
+        by_dir.sort.each do |dir, ids|
+          write_managed(written, "#{dir}/_index.md", moc_note(dir, ids, mapper))
+        end
+        write_managed(written, '_Overview.md', overview_note(by_dir))
+        by_dir.size + 1
+      end
+
+      def moc_note(dir, ids, mapper)
+        lines = [managed_frontmatter, "# #{dir} (#{ids.size})", '']
+        ids.sort.each { |id| lines << "- #{mapper.wikilink(id)}" }
+        "#{lines.join("\n")}\n"
+      end
+
+      def overview_note(by_dir)
+        total = by_dir.values.sum(&:size)
+        lines = [managed_frontmatter, '# Woods Vault Overview', '',
+                 "#{total} units across #{by_dir.size} types.", '', '## Units by type']
+        by_dir.sort.each { |dir, ids| lines << "- [[#{dir}/_index|#{dir}]] — #{ids.size}" }
+        lines.push('', '## Machine-readable',
+                   '- `_woods/manifest.json` — id ↔ path map + per-note metadata',
+                   '- `_woods/dependency_graph.json` / `_woods/graph_analysis.json` — full topology')
+        "#{lines.join("\n")}\n"
+      end
+
+      def managed_frontmatter
+        "#{{ 'woods_managed' => true, 'tags' => ['woods/moc'] }.to_yaml}---"
+      end
+
+      # ── Pass 3b: machine sidecar ─────────────────────────────────────
+
+      def write_sidecar(emitted, mapper, graph, analysis, written)
+        notes = emitted.to_h do |id|
+          [id, { 'path' => mapper.path_for(id), 'type' => @nodes[id]['type'],
+                 'pagerank' => @pagerank[id]&.round(6) }.compact]
+        end
+        manifest = { 'schema_version' => 1, 'notes' => sort_hash(notes), 'paths' => sort_hash(mapper.paths_to_ids) }
+        write_json(written, "#{SIDECAR_DIR}/manifest.json", manifest)
+        write_json(written, "#{SIDECAR_DIR}/dependency_graph.json", graph)
+        write_json(written, "#{SIDECAR_DIR}/graph_analysis.json", analysis) if analysis
+      end
+
+      # ── Pass 3c: owned-only assets (.obsidian config, Units.base, sentinel) ──
+
+      def write_owned_assets(emitted)
+        if can_write_obsidian_config?
+          AtomicFile.write(@vault.join('.obsidian', 'app.json'), VaultAssets.app_json)
+          AtomicFile.write(@vault.join('.obsidian', 'types.json'), VaultAssets.types_json)
+          AtomicFile.write(@vault.join('.obsidian', 'graph.json'), VaultAssets.graph_json(present_types(emitted)))
+          AtomicFile.write(@vault.join('Units.base'), VaultAssets.units_base)
+        end
+        AtomicFile.write(@vault.join(SENTINEL),
+                         "woods-managed vault — regenerated by `rake woods:obsidian`. Safe to delete.\n")
+      end
+
+      def can_write_obsidian_config?
+        obsidian = @vault.join('.obsidian')
+        return true unless obsidian.exist?
+        return true if @vault.join(SENTINEL).exist?
+
+        log '  existing .obsidian/ left untouched (not a woods-owned vault)'
+        false
+      end
+
+      def present_types(emitted)
+        emitted.map { |id| @nodes[id]['type'] }.uniq
+      end
+
+      # ── Pass 4: guarded sweep ────────────────────────────────────────
+
+      def sweep(written)
+        managed = managed_notes
+        stale = managed.reject { |rel| written.include?(rel) }
+        return 0 if stale.empty?
+        return 0 if guard_blocks_purge?(stale, managed)
+
+        stale.sort.count do |rel|
+          path = @vault.join(rel)
+          next false unless safe_inside_vault?(path) && path.file?
+
+          File.delete(path)
+          true
+        end
+      end
+
+      def managed_notes
+        Dir.glob(@vault.join('**', '*.md')).filter_map do |abs|
+          next unless managed_marker?(abs)
+
+          Pathname.new(abs).relative_path_from(@vault).to_s
+        end
+      end
+
+      # Only our notes carry this frontmatter flag. Reads just the frontmatter
+      # head (notes can be large when include_source is on) rather than the whole
+      # file, and bails on anything that isn't a leading "---" fence.
+      def managed_marker?(abs)
+        fm = frontmatter_head(abs)
+        fm.is_a?(Hash) && fm['woods_managed'] == true
+      rescue StandardError
+        false
+      end
+
+      # Recognizes only a canonical fence: the file must open with exactly "---\n"
+      # and close with exactly "---\n" (what woods emits). Anything looser — a
+      # fence with trailing characters, CRLF, or no close within the cap — reads
+      # as unmanaged. Deliberately conservative: the sweep deletes managed notes,
+      # so when in doubt we must not claim a file as ours.
+      def frontmatter_head(abs)
+        File.open(abs, 'r:UTF-8') do |file|
+          return nil unless file.gets == "---\n"
+
+          lines = []
+          MAX_FRONTMATTER_LINES.times do
+            line = file.gets
+            break if line.nil?
+            return YAML.safe_load(lines.join) if line == "---\n"
+
+            lines << line
+          end
+          nil
+        end
+      end
+
+      def guard_blocks_purge?(stale, managed)
+        return false if @force_purge
+
+        total = managed.size
+        return false if total < PURGE_GUARD_MIN_DOCS
+
+        fraction = stale.size.to_f / total
+        return false unless fraction > PURGE_GUARD_FRACTION
+
+        log "  WARNING: refusing to delete #{stale.size}/#{total} managed notes " \
+            "(#{(fraction * 100).round}% > #{(PURGE_GUARD_FRACTION * 100).to_i}%). " \
+            'Set WOODS_OBSIDIAN_FORCE_PURGE=1 to override.'
+        true
+      end
+
+      def safe_inside_vault?(path)
+        root = @vault.realpath.to_s
+        target = path.exist? ? path.realpath.to_s : path.expand_path.to_s
+        target == root || target.start_with?(root + File::SEPARATOR)
+      rescue StandardError
+        false
+      end
+
+      # ── Ownership ────────────────────────────────────────────────────
+
+      def vault_owned_at_start?
+        return true unless @vault.exist?
+        return true if @vault.join(SENTINEL).exist?
+
+        (@vault.children.map { |c| c.basename.to_s } - [SENTINEL]).empty?
+      end
+
+      def warn_foreign
+        log "  WARNING: #{@vault} is not a woods-owned vault (no #{SENTINEL} sentinel and it " \
+            'already has content). Wrote notes additively; skipped config and the stale-note sweep.'
+      end
+
+      # ── Helpers ──────────────────────────────────────────────────────
+
+      def write_managed(written, rel, content)
+        AtomicFile.write(@vault.join(rel), content)
+        written << rel
+      end
+
+      def write_json(written, rel, data)
+        AtomicFile.write(@vault.join(rel), "#{JSON.pretty_generate(data)}\n")
+        written << rel
+      end
+
+      def sort_hash(hash)
+        hash.sort.to_h
+      end
+
+      def scanner
+        return nil unless @include_source
+
+        @scanner ||= begin
+          require 'woods/console/credential_scanner'
+          Woods::Console::CredentialScanner.new
+        end
+      end
+
+      def safe_graph_analysis
+        @reader.graph_analysis
+      rescue StandardError
+        nil
+      end
+
+      def build_reader(index_dir)
+        require 'woods/mcp/index_reader'
+        Woods::MCP::IndexReader.new(index_dir)
+      rescue ArgumentError => e
+        raise ExportError, "#{e.message} — run `bundle exec rake woods:extract` first"
+      end
+
+      def cap_errors(errors)
+        return errors if errors.size <= MAX_ERRORS
+
+        errors.first(MAX_ERRORS) + ["... and #{errors.size - MAX_ERRORS} more errors"]
+      end
+
+      def log(message)
+        @output&.puts(message)
+      end
+    end
+  end
+end

data/lib/woods/operator/error_escalator.rb

@@ -37,22 +37,33 @@ module Woods
 
       # Classify an error by severity and suggest remediation.
       #
+      # Most patterns key on the error's class name, so a caller that only
+      # has the class name as a string (e.g. the pipeline_diagnose MCP tool,
+      # which receives error_class over the wire and can't reconstruct the
+      # real exception class) must pass +class_name:+ — otherwise the
+      # synthesized StandardError makes every class-keyed pattern miss and
+      # the error falls through to :unknown.
+      #
       # @param error [StandardError] The error to classify
+      # @param class_name [String, nil] Override for the class name used in
+      #   pattern matching and reported as :error_class (defaults to
+      #   error.class.name)
       # @return [Hash] :severity (:transient or :permanent), :category, :remediation, :error_class, :message
-      def classify(error)
-        error_string = "#{error.class} #{error.message}"
+      def classify(error, class_name: nil)
+        resolved_class = class_name || error.class.name
+        error_string = "#{resolved_class} #{error.message}"
 
         match = find_match(error_string, TRANSIENT_PATTERNS, :transient) ||
                 find_match(error_string, PERMANENT_PATTERNS, :permanent)
 
         if match
-          match.merge(error_class: error.class.name, message: error.message)
+          match.merge(error_class: resolved_class, message: error.message)
         else
           {
             severity: :unknown,
             category: 'unclassified',
             remediation: 'Investigate error details and check logs',
-            error_class: error.class.name,
+            error_class: resolved_class,
             message: error.message
           }
         end

data/lib/woods/resilience/circuit_breaker.rb

@@ -36,50 +36,93 @@ module Woods
 
       # @param threshold [Integer] Number of consecutive failures before opening the circuit
       # @param reset_timeout [Numeric] Seconds to wait before transitioning from open to half_open
-      def initialize(threshold: 5, reset_timeout: 60)
+      # @param success_threshold [Integer] Number of consecutive successful probes required to
+      #   close the circuit from half_open (default 1). Higher values reduce flapping when a
+      #   recovering service is still intermittently failing.
+      def initialize(threshold: 5, reset_timeout: 60, success_threshold: 1)
         @threshold = threshold
         @reset_timeout = reset_timeout
+        @success_threshold = success_threshold
         @state = :closed
         @failure_count = 0
+        @success_count = 0
         @last_failure_time = nil
+        @half_open_probe_in_flight = false
         @mutex = Mutex.new
       end
 
       # Execute a block through the circuit breaker.
       #
+      # In half_open, only a SINGLE probe is admitted at a time: concurrent
+      # calls are rejected with {CircuitOpenError} until the probe resolves.
+      # This prevents a thundering herd of probes against a still-recovering
+      # service, and — because probes can't overlap — removes the race where a
+      # slow probe's success would wipe failures recorded by a concurrent one.
+      #
       # @yield The block to execute
       # @return [Object] The return value of the block
-      # @raise [CircuitOpenError] if the circuit is open and the timeout has not elapsed
+      # @raise [CircuitOpenError] if the circuit is open, or half_open with a probe already in flight
       # @raise [StandardError] re-raises any error from the block
       def call(&block)
-        # Phase 1: Check state under mutex
+        probing = admit_call!
+
+        begin
+          result = block.call
+        rescue CircuitOpenError
+          # A nested breaker tripped — release our probe slot but don't count
+          # it as this breaker's own failure.
+          @mutex.synchronize { @half_open_probe_in_flight = false if probing }
+          raise
+        rescue StandardError => e
+          @mutex.synchronize do
+            @half_open_probe_in_flight = false if probing
+            record_failure(probing)
+          end
+          raise e
+        end
+
+        @mutex.synchronize do
+          @half_open_probe_in_flight = false if probing
+          record_success(probing)
+        end
+        result
+      end
+
+      private
+
+      # Decide whether this call may proceed, transitioning open→half_open when
+      # the reset timeout has elapsed. Runs entirely under the mutex.
+      #
+      # @return [Boolean] true when this call is the half_open probe (so the
+      #   caller knows to release the probe slot when it finishes)
+      # @raise [CircuitOpenError] when the circuit is open, or half_open with a
+      #   probe already in flight
+      def admit_call!
         @mutex.synchronize do
           case @state
           when :open
-            unless monotonic_now - @last_failure_time >= @reset_timeout
-              raise CircuitOpenError, "Circuit breaker is open (#{@failure_count} failures)"
-            end
+            raise CircuitOpenError, "Circuit breaker is open (#{@failure_count} failures)" unless reset_timeout_elapsed?
 
+            # First caller after the timeout becomes the single half_open probe.
             @state = :half_open
+            @half_open_probe_in_flight = true
+            true
+          when :half_open
+            raise CircuitOpenError, 'Circuit breaker is half-open (probe in flight)' if @half_open_probe_in_flight
+
+            @half_open_probe_in_flight = true
+            true
+          else
+            false
           end
         end
-
-        # Phase 2: Execute outside mutex
-        result = block.call
-
-        # Phase 3: Record success under mutex
-        @mutex.synchronize { reset! }
-
-        result
-      rescue CircuitOpenError
-        raise
-      rescue StandardError => e
-        # Phase 4: Record failure under mutex
-        @mutex.synchronize { record_failure }
-        raise e
       end
 
-      private
+      # @return [Boolean] whether enough time has passed since the last failure
+      #   to attempt a recovery probe
+      def reset_timeout_elapsed?
+        @last_failure_time && (monotonic_now - @last_failure_time >= @reset_timeout)
+      end
 
       # Monotonic clock reading — immune to NTP slews and DST adjustments.
       #
@@ -88,18 +131,49 @@ module Woods
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
-      # Record a failure and potentially open the circuit.
-      def record_failure
+      # Record a failure and potentially open the circuit. A failure while
+      # half_open reopens immediately, regardless of the failure threshold —
+      # but only when it is the genuine in-flight probe. A stale call
+      # admitted while the circuit was still closed, completing after the
+      # transition to half_open, is not the probe and must not decide
+      # recovery (its outcome reflects the pre-outage attempt, not the
+      # current one).
+      #
+      # @param probing [Boolean] whether this call was admitted as the probe
+      def record_failure(probing)
+        return if @state == :half_open && !probing
+
         @failure_count += 1
         @last_failure_time = monotonic_now
-        @state = :open if @failure_count >= @threshold
+        @success_count = 0
+        @state = :open if @state == :half_open || @failure_count >= @threshold
+      end
+
+      # Record a success. In half_open, only the in-flight probe advances
+      # recovery — a stale non-probe success (a call admitted while closed
+      # that resolves after the circuit went half_open) must not count toward
+      # the success threshold or close the circuit. In closed, a success
+      # clears accumulated failures.
+      #
+      # @param probing [Boolean] whether this call was admitted as the probe
+      def record_success(probing)
+        if @state == :half_open
+          return unless probing
+
+          @success_count += 1
+          reset! if @success_count >= @success_threshold
+        elsif @state == :closed
+          @failure_count = 0
+        end
       end
 
       # Reset the circuit breaker to closed state with zero failures.
       def reset!
         @state = :closed
         @failure_count = 0
+        @success_count = 0
         @last_failure_time = nil
+        @half_open_probe_in_flight = false
       end
     end
   end

data/lib/woods/resolved_config.rb

@@ -203,7 +203,10 @@ module Woods
         obj.each { |v| deep_freeze(v) }
         obj.frozen? ? obj : obj.freeze
       when String
-        obj.frozen? ? obj : obj.dup.freeze
+        # Freeze in place — returning a frozen dup would be discarded by the
+        # Hash/Array branches above (they recurse for side effects only),
+        # leaving every nested string mutable.
+        obj.frozen? ? obj : obj.freeze
       else
         obj
       end