directory_watcher 1.4.1 → 1.5.1

data/lib/directory_watcher/collector.rb

@@ -0,0 +1,283 @@
+# Collector reads items from a collection Queue and processes them to see if
+# FileEvents should be put onto the notification Queue.
+#
+class DirectoryWatcher::Collector
+  include DirectoryWatcher::Threaded
+  include DirectoryWatcher::Logable
+
+  # Create a new StatCollector from the given Configuration, and an optional
+  # Scan.
+  #
+  # configuration - The Collector uses from Configuration:
+  #   collection_queue   - The Queue to read items from the Scanner on
+  #   notification_queue - The Queue to submit the Events to the Notifier on
+  #   stable             - The number of times we see a file hasn't changed before
+  #                        emitting a stable event
+  #   sort_by            - the method used to sort events during on_scan results
+  #   order_by           - The method used to order events from call to on_scan
+  #
+  # pre_load_scan - A Scan to use to load our internal state from before. No
+  #                 events will be emitted for the FileStat's in this scan.
+  #
+  #def initialize( notification_queue, collection_queue, options = {} )
+  def initialize( config )
+    @stats = Hash.new
+    @stable_counts = Hash.new
+    @config = config
+    on_scan( DirectoryWatcher::Scan.new( config.glob ), false ) if config.pre_load?
+    self.interval = 0.01 # yes this is a fast loop
+  end
+
+  # The number of times we see a file hasn't changed before emitting a stable
+  # count. See Configuration#stable
+  def stable_threshold
+    @config.stable
+  end
+
+  # How to sort Scan results. See Configuration.
+  #
+  def sort_by
+    @config.sort_by
+  end
+
+  # How to order Scan results. See Configuration.
+  #
+  def order_by
+    @config.order_by
+  end
+
+  # The queue from which to read items from the scanners. See Configuration.
+  #
+  def collection_queue
+    @config.collection_queue
+  end
+
+  # The queue to write Events for the Notifier. See Configuration.
+  #
+  def notification_queue
+    @config.notification_queue
+  end
+
+  # Given the scan, update the set of stats with the results from the Scan and
+  # emit events to the notification queue as appropriate.
+  #
+  # scan        - The Scan containing all the new FileStat items
+  # emit_events - Should events be emitted for the events in the scan
+  #               (default: true)
+  #
+  # There is one odd thing that happens here. Scanners that are EventableScanners
+  # use on_stat to emit removed events, and the standard threaded Scanner only
+  # uses Scans. So we make sure and only emit removed events in this method if
+  # the scanner that gave us the scan was the basic threaded Scanner.
+  #
+  # TODO: Possibly fix this through another abstraction in the Scanners.
+  # No idea about what that would be yet.
+  #
+  # Returns nothing.
+  #
+  def on_scan( scan, emit_events = true )
+    seen_paths = Set.new
+    logger.debug "Sorting by #{sort_by} #{order_by}"
+    sorted_stats( scan.run ).each do |stat|
+      on_stat(stat, emit_events)
+      seen_paths << stat.path
+    end
+    emit_removed_events(seen_paths) if @config.scanner.nil?
+  end
+
+  # Process a single stat and emit an event if necessary.
+  #
+  # stat       - The new FileStat to process and see if an event should
+  #              be emitted
+  # emit_event - Whether or not an event should be emitted.
+  #
+  # Returns nothing
+  def on_stat( stat, emit_event = true )
+    orig_stat = update_stat( stat )
+    logger.debug "Emitting event for on_stat #{stat}"
+    emit_event_for( orig_stat, stat ) if emit_event
+  end
+
+  # Remove one item from the collection queue and process it.
+  #
+  # This method is required by the Threaded API
+  #
+  # Returns nothing
+  def run
+    case thing = collection_queue.deq
+    when ::DirectoryWatcher::Scan
+      on_scan(thing)
+    when ::DirectoryWatcher::FileStat
+      on_stat(thing)
+    else
+      raise "Unknown item in the queue: #{thing}"
+    end
+  end
+
+  # Write the current stats to the given IO object as a YAML document.
+  #
+  # io - The IO object to write the document to.
+  #
+  # Returns nothing.
+  def dump_stats( io )
+    YAML.dump(@stats, io)
+  end
+
+  # Read the current stats from the given IO object. Any existing stats in the
+  # Collector will be overwritten
+  #
+  # io - The IO object from which to read the document.
+  #
+  # Returns nothing.
+  def load_stats( io )
+    @stats = YAML.load(io)
+  end
+
+  #######
+  private
+  #######
+
+  # Sort the stats by +sort_by+ and +order_by+ returning the results
+  #
+  def sorted_stats( stats )
+    sorted = stats.sort_by{ |stat| stat.send(sort_by) }
+    sorted = sorted.reverse if order_by == :descending
+    return sorted
+  end
+
+  # Update the stats Hash with the new_stat information, return the old data
+  # that is being replaced.
+  #
+  def update_stat( new_stat )
+    old_stat = @stats.delete(new_stat.path)
+    @stats.store(new_stat.path, new_stat) unless new_stat.removed?
+    return old_stat
+  end
+
+  # Look for removed files and emit removed events for all of them.
+  #
+  # seen_paths - the list of files that we know currently exist
+  #
+  # Return nothing
+  def emit_removed_events( seen_paths )
+    @stats.keys.each do |existing_path|
+      next if seen_paths.include?(existing_path)
+      old_stat = @stats.delete(existing_path)
+      emit_event_for(old_stat, ::DirectoryWatcher::FileStat.for_removed_path(existing_path))
+    end
+  end
+
+  # Determine what type of event to emit, and put that event onto the
+  # notification queue.
+  #
+  # old_stat - The old FileStat
+  # new_stat - The new FileStat
+  #
+  # Returns nothing
+  def emit_event_for( old_stat, new_stat )
+    event = DirectoryWatcher::Event.from_stats( old_stat, new_stat )
+    if should_emit?(event) then
+      logger.debug "Sending event #{event.object_id} to notifcation queue"
+      notification_queue.enq( event )
+    else
+      logger.debug "Emitting of event #{event.object_id} cancelled"
+    end
+  end
+
+  # Should the event given actually be emitted.
+  #
+  # If the event passed in is NOT a stable event, return true
+  # If there is a stable_threshold, then check to see if the stable count for
+  # this event's path has crossed the stable threshold.
+  #
+  # This method has the side effect of updating the stable count of the path of
+  # the event. If we are going to return true for the stable event, then we
+  # reset the stable count of that event to 0.
+  #
+  # event - any event
+  #
+  # Returns whether or not to emit the event based upon its stability
+  def should_emit?( event )
+    if event.stable? then
+      if emitting_stable_events? and valid_for_stable_event?( event.path )then
+        increment_stable_count( event.path )
+        if should_emit_stable?( event.path ) then
+          mark_as_invalid_for_stable_event( event.path )
+          return true
+        end
+      end
+      return false
+    elsif event.removed? then
+      mark_as_invalid_for_stable_event( event.path )
+      return true
+    else
+      mark_as_valid_for_stable_event( event.path )
+      return true
+    end
+  end
+
+  # Is the given path able to have a stable event emitted for it?
+  #
+  # A stable event may only be emitted for a path that has already had an added
+  # or modified event already sent. Also, once a stable event has been emitted
+  # for a path, another stable event may not be emitted until it has been
+  # modified, or added again.
+  #
+  # path - the path of the file to check
+  #
+  # Returns whether or not the path may have a stable event emitted for it.
+  def valid_for_stable_event?( path )
+    @stable_counts.has_key?( path )
+  end
+
+  # Let it be known that the given path can now have a stable event emitted for
+  # it.
+  #
+  # path - the path to mark as ready
+  #
+  # Returns nothing
+  def mark_as_valid_for_stable_event( path )
+    logger.debug "#{path} marked as valid for stable"
+    @stable_counts[path] = 0
+  end
+
+  # Mark that the given path is invalid for having a stable event emitted for
+  # it.
+  #
+  # path - the path to mark
+  #
+  # Returns nothing
+  def mark_as_invalid_for_stable_event( path )
+    logger.debug "#{path} marked as invalid for stable"
+    @stable_counts.delete( path )
+  end
+
+  # Increment the stable count for the given path
+  #
+  # path - the path of the file to increment its stable count
+  #
+  # Returns nothing
+  def increment_stable_count( path )
+    @stable_counts[path] += 1
+  end
+
+  # Is the given path ready to have a stable event emitted?
+  #
+  # path - the path to report on
+  #
+  # Returns whether to emit a stable event or not
+  def should_emit_stable?( path )
+    @stable_counts[path] >= stable_threshold
+  end
+
+  # Is it legal for us to emit stable events at all. This checks the config to
+  # see if that is the case.
+  #
+  # In the @config if the stable threshold is set then we are emitting stable
+  # events.
+  #
+  # Returns whether it is legal to propogate stable events
+  def emitting_stable_events?
+    stable_threshold
+  end
+end

data/lib/directory_watcher/configuration.rb

@@ -0,0 +1,228 @@
+#
+# The top level configuration options used by DirectoryWatcher are used by many
+# of the sub components for a variety of purposes. The Configuration represents
+# all those options and other global like instances.
+#
+# The top level DirectoryWatcher class allows the configs to be changed during
+# execution, so all of the dependent classes need to be informed when their
+# options have changed. This class allows that.
+#
+class DirectoryWatcher::Configuration
+  # The directory to monitor for events. The glob's will be used in conjunction
+  # with this directory to find the full list of globs available.
+  attr_reader :dir
+
+  # The glob of files to monitor. This is an Array of file matching globs
+  # be aware that changing the :glob value after watching has started has the
+  # potential to cause spurious events if the new globs do not match the old,
+  # files will appear to have been deleted.
+  #
+  # The default is '*'
+  attr_reader :glob
+
+  # The interval at which to do a full scan using the +glob+ to determine Events
+  # to send.
+  #
+  # The default is 30.0 seconds
+  attr_reader :interval
+
+  # Controls the number of intervals a file must remain unchanged before it is
+  # considered "stable". When this condition is met, a stable event is
+  # generated for the file. If stable is set to +nil+ then stable events
+  # will not be generated.
+  #
+  # The default is nil, indicating no stable events are to be emitted.
+  attr_reader :stable
+
+  # pre_load says if an initial scan using the globs should be done to pre
+  # populate the state of the system before sending any events.
+  #
+  # The default is false
+  attr_reader :pre_load
+
+  # The filename to persist the state of the DirectoryWatcher too upon calling
+  # *stop*.
+  #
+  # The default is nil, indicating that no state is to be persisted.
+  attr_reader :persist
+
+  # The back end scanner to use. The available options are:
+  #
+  #   nil     => Use the default, pure ruby Threaded scanner
+  #   :em     => Use the EventMachine based scanner. This requires that the
+  #              'eventmachine' gem be installed.
+  #   :coolio => Use the Cool.io based scanner. This requires that the
+  #              'cool.io' gem be installed.
+  #   :rev    => Use the Rev based scanner. This requires that the 'rev' gem be
+  #              installed.
+  #
+  # The default is nil, indicating the pure ruby threaded scanner will be used.
+  # This option may not be changed once the DirectoryWatcher is allocated.
+  #
+  attr_reader :scanner
+
+  # The sorting method to use when emitting a set of Events after a Scan has
+  # happened. Since a Scan may produce a number of events, if those Events should
+  # be emitted in a particular order, use +sort_by+ to pick which field to sort
+  # the events, and +order_by+ to say if those events are to be emitted in
+  # :ascending or :descending order.
+  #
+  # Available options:
+  #
+  #   :path   => The default, they will be sorted by full pathname
+  #   :mtime  => Last modified time. They will be sorted by their FileStat mtime
+  #   :size   => The number of bytes in the file.
+  #
+  attr_accessor :sort_by
+
+  # When sorting you may pick if the order should be:
+  #
+  #   :ascending  => The default, from lowest to highest
+  #   :descending => from highest to lowest.
+  #
+  attr_accessor :order_by
+
+  # The Queue through which the Scanner will send data to the Collector
+  #
+  attr_reader :collection_queue
+
+  # The Queue through which the Collector will send data to the Notifier
+  #
+  attr_reader :notification_queue
+
+  # The logger through wich every one will log
+  #
+  attr_reader :logger
+
+  # Return a Hash of all the default options
+  #
+  def self.default_options
+    {
+      :dir           => '.',
+      :glob          => '*',
+      :interval      => 30.0,
+      :stable        => nil,
+      :pre_load      => false,
+      :persist       => nil,
+      :scanner       => nil,
+      :sort_by       => :path,
+      :order_by      => :ascending,
+      :logger        => nil,
+    }
+  end
+
+  # Create a new Configuration by blending the passed in items with the defaults
+  #
+  def initialize( options = {} )
+    o = self.class.default_options.merge( options )
+    @dir      = o[:dir]
+    @pre_load = o[:pre_load]
+    @scanner  = o[:scanner]
+    @sort_by  = o[:sort_by]
+    @order_by = o[:order_by]
+
+    # These have validation rules
+    self.persist = o[:persist]
+    self.interval = o[:interval]
+    self.glob = o[:glob]
+    self.stable = o[:stable]
+    self.logger = o[:logger]
+
+    @notification_queue = Queue.new
+    @collection_queue = Queue.new
+  end
+
+  # Is pre_load set or not
+  #
+  def pre_load?
+    @pre_load
+  end
+
+  # The class of the scanner
+  #
+  def scanner_class
+    class_name = scanner.to_s.capitalize + 'Scanner'
+    DirectoryWatcher.const_get( class_name ) rescue DirectoryWatcher::Scanner
+  end
+
+  # call-seq:
+  #    glob = '*'
+  #    glob = ['lib/**/*.rb', 'test/**/*.rb']
+  #
+  # Sets the glob pattern that will be used when scanning the directory for
+  # files. A single glob pattern can be given or an array of glob patterns.
+  #
+  def glob=( val )
+    glob = case val
+           when String; [File.join(@dir, val)]
+           when Array; val.flatten.map! {|g| File.join(@dir, g)}
+           else
+             raise(ArgumentError,
+                   'expecting a glob pattern or an array of glob patterns')
+           end
+    glob.uniq!
+    @glob = glob
+  end
+
+  # Sets the directory scan interval. The directory will be scanned every
+  # _interval_ seconds for changes to files matching the glob pattern.
+  # Raises +ArgumentError+ if the interval is zero or negative.
+  #
+  def interval=( val )
+    val = Float(val)
+    raise ArgumentError, "interval must be greater than zero" if val <= 0
+    @interval = val
+  end
+
+  # Sets the logger instance. This will be used by all classes for logging
+  #
+  def logger=( val )
+    if val then
+      if %w[ debug info warn error fatal ].all? { |meth| val.respond_to?( meth ) } then
+        @logger = val
+      end
+    else
+      @logger = ::DirectoryWatcher::Logable.default_logger
+    end
+  end
+
+  # Sets the number of intervals a file must remain unchanged before it is
+  # considered "stable". When this condition is met, a stable event is
+  # generated for the file. If stable is set to +nil+ then stable events
+  # will not be generated.
+  #
+  # A stable event will be generated once for a file. Another stable event
+  # will only be generated after the file has been modified and then remains
+  # unchanged for _stable_ intervals.
+  #
+  # Example:
+  #
+  #     dw = DirectoryWatcher.new( '/tmp', :glob => 'swap.*' )
+  #     dw.interval = 15.0
+  #     dw.stable = 4
+  #
+  # In this example, a directory watcher is configured to look for swap files
+  # in the /tmp directory. Stable events will be generated every 4 scan
+  # intervals iff a swap remains unchanged for that time. In this case the
+  # time is 60 seconds (15.0 * 4).
+  #
+  def stable=( val )
+    if val.nil?
+      @stable = nil
+    else
+      val = Integer(val)
+      raise ArgumentError, "stable must be greater than zero" if val <= 0
+      @stable = val
+    end
+    return @stable
+  end
+
+  # Sets the name of the file to which the directory watcher state will be
+  # persisted when it is stopped. Setting the persist filename to +nil+ will
+  # disable this feature.
+  #
+  def persist=( filename )
+    @persist = filename ? filename.to_s : nil
+  end
+end
+