listen 2.10.1 → 3.0.0

data/lib/listen/event/processor.rb

@@ -0,0 +1,122 @@
+module Listen
+  module Event
+    class Processor
+      def initialize(config, reasons)
+        @config = config
+        @reasons = reasons
+        _reset_no_unprocessed_events
+      end
+
+      # TODO: implement this properly instead of checking the state at arbitrary
+      # points in time
+      def loop_for(latency)
+        @latency = latency
+
+        loop do
+          _wait_until_events
+          _wait_until_events_calm_down
+          _wait_until_no_longer_paused
+          _process_changes
+        end
+      rescue Stopped
+        Listen::Logger.debug('Processing stopped')
+      end
+
+      private
+
+      class Stopped < RuntimeError
+      end
+
+      def _wait_until_events_calm_down
+        loop do
+          now = _timestamp
+
+          # Assure there's at least latency between callbacks to allow
+          # for accumulating changes
+          diff = _deadline - now
+          break if diff <= 0
+
+          # give events a bit of time to accumulate so they can be
+          # compressed/optimized
+          _sleep(:waiting_until_latency, diff)
+        end
+      end
+
+      def _wait_until_no_longer_paused
+        # TODO: may not be a good idea?
+        _sleep(:waiting_for_unpause) while config.paused?
+      end
+
+      def _check_stopped
+        return unless config.stopped?
+
+        _flush_wakeup_reasons
+        raise Stopped
+      end
+
+      def _sleep(_local_reason, *args)
+        _check_stopped
+        sleep_duration = config.sleep(*args)
+        _check_stopped
+
+        _flush_wakeup_reasons do |reason|
+          next unless reason == :event
+          _remember_time_of_first_unprocessed_event unless config.paused?
+        end
+
+        sleep_duration
+      end
+
+      def _remember_time_of_first_unprocessed_event
+        @first_unprocessed_event_time ||= _timestamp
+      end
+
+      def _reset_no_unprocessed_events
+        @first_unprocessed_event_time = nil
+      end
+
+      def _deadline
+        @first_unprocessed_event_time + @latency
+      end
+
+      def _wait_until_events
+        # TODO: long sleep may not be a good idea?
+        _sleep(:waiting_for_events) while config.event_queue.empty?
+        @first_unprocessed_event_time ||= _timestamp
+      end
+
+      def _flush_wakeup_reasons
+        reasons = @reasons
+        until reasons.empty?
+          reason = reasons.pop
+          yield reason if block_given?
+        end
+      end
+
+      def _timestamp
+        config.timestamp
+      end
+
+      # for easier testing without sleep loop
+      def _process_changes
+        _reset_no_unprocessed_events
+
+        changes = []
+        changes << config.event_queue.pop until config.event_queue.empty?
+
+        callable = config.callable?
+        return unless callable
+
+        hash = config.optimize_changes(changes)
+        result = [hash[:modified], hash[:added], hash[:removed]]
+        return if result.all?(&:empty?)
+
+        block_start = _timestamp
+        config.call(*result)
+        Listen::Logger.debug "Callback took #{_timestamp - block_start} sec"
+      end
+
+      attr_reader :config
+    end
+  end
+end

data/lib/listen/event/queue.rb

@@ -0,0 +1,54 @@
+module Listen
+  module Event
+    class Queue
+      class Config
+        def initialize(relative)
+          @relative = relative
+        end
+
+        def relative?
+          @relative
+        end
+      end
+
+      def initialize(config, &block)
+        @event_queue = Thread::Queue.new
+        @block = block
+        @config = config
+      end
+
+      def <<(args)
+        type, change, dir, path, options = *args
+        fail "Invalid type: #{type.inspect}" unless [:dir, :file].include? type
+        fail "Invalid change: #{change.inspect}" unless change.is_a?(Symbol)
+        fail "Invalid path: #{path.inspect}" unless path.is_a?(String)
+
+        dir = _safe_relative_from_cwd(dir)
+        event_queue.public_send(:<<, [type, change, dir, path, options])
+
+        block.call(args) if block
+      end
+
+      def empty?
+        event_queue.empty?
+      end
+
+      def pop
+        event_queue.pop
+      end
+
+      private
+
+      attr_reader :event_queue
+      attr_reader :block
+      attr_reader :config
+
+      def _safe_relative_from_cwd(dir)
+        return dir unless config.relative?
+        dir.relative_path_from(Pathname.pwd)
+      rescue ArgumentError
+        dir
+      end
+    end
+  end
+end

data/lib/listen/file.rb

@@ -1,25 +1,25 @@
 module Listen
   class File
-    def self.change(record, dir, rel_path)
-      path = dir + rel_path
+    def self.change(record, rel_path)
+      path = Pathname.new(record.root) + rel_path
       lstat = path.lstat
 
       data = { mtime: lstat.mtime.to_f, mode: lstat.mode }
 
-      record_data = record.file_data(dir, rel_path)
+      record_data = record.file_data(rel_path)
 
       if record_data.empty?
-        record.async.update_file(dir, rel_path, data)
+        record.update_file(rel_path, data)
         return :added
       end
 
       if data[:mode] != record_data[:mode]
-        record.async.update_file(dir, rel_path, data)
+        record.update_file(rel_path, data)
         return :modified
       end
 
       if data[:mtime] != record_data[:mtime]
-        record.async.update_file(dir, rel_path, data)
+        record.update_file(rel_path, data)
         return :modified
       end
 
@@ -56,13 +56,13 @@ module Listen
       return if data[:mtime].to_i + 2 <= Time.now.to_f
 
       md5 = Digest::MD5.file(path).digest
-      record.async.update_file(dir, rel_path, data.merge(md5: md5))
+      record.update_file(rel_path, data.merge(md5: md5))
       :modified if record_data[:md5] && md5 != record_data[:md5]
     rescue SystemCallError
-      record.async.unset_path(dir, rel_path)
+      record.unset_path(rel_path)
       :removed
     rescue
-      Celluloid::Logger.debug "lstat failed for: #{rel_path} (#{$ERROR_INFO})"
+      Listen::Logger.debug "lstat failed for: #{rel_path} (#{$ERROR_INFO})"
       raise
     end
 

data/lib/listen/fsm.rb

@@ -0,0 +1,131 @@
+# Code copied from https://github.com/celluloid/celluloid-fsm
+module Listen
+  module FSM
+    DEFAULT_STATE = :default # Default state name unless one is explicitly set
+
+    # Included hook to extend class methods
+    def self.included(klass)
+      klass.send :extend, ClassMethods
+    end
+
+    module ClassMethods
+      # Obtain or set the default state
+      # Passing a state name sets the default state
+      def default_state(new_default = nil)
+        if new_default
+          @default_state = new_default.to_sym
+        else
+          defined?(@default_state) ? @default_state : DEFAULT_STATE
+        end
+      end
+
+      # Obtain the valid states for this FSM
+      def states
+        @states ||= {}
+      end
+
+      # Declare an FSM state and optionally provide a callback block to fire
+      # Options:
+      # * to: a state or array of states this state can transition to
+      def state(*args, &block)
+        if args.last.is_a? Hash
+          # Stringify keys :/
+          options = args.pop.each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+        else
+          options = {}
+        end
+
+        args.each do |name|
+          name = name.to_sym
+          default_state name if options['default']
+          states[name] = State.new(name, options['to'], &block)
+        end
+      end
+    end
+
+    # Be kind and call super if you must redefine initialize
+    def initialize
+      @state = self.class.default_state
+    end
+
+    # Obtain the current state of the FSM
+    attr_reader :state
+
+    def transition(state_name)
+      new_state = validate_and_sanitize_new_state(state_name)
+      return unless new_state
+      transition_with_callbacks!(new_state)
+    end
+
+    # Immediate state transition with no checks, or callbacks. "Dangerous!"
+    def transition!(state_name)
+      @state = state_name
+    end
+
+    protected
+
+    def validate_and_sanitize_new_state(state_name)
+      state_name = state_name.to_sym
+
+      return if current_state_name == state_name
+
+      if current_state && !current_state.valid_transition?(state_name)
+        valid = current_state.transitions.map(&:to_s).join(', ')
+        msg = "#{self.class} can't change state from '#{@state}'"\
+          " to '#{state_name}', only to: #{valid}"
+        fail ArgumentError, msg
+      end
+
+      new_state = states[state_name]
+
+      unless new_state
+        return if state_name == default_state
+        fail ArgumentError, "invalid state for #{self.class}: #{state_name}"
+      end
+
+      new_state
+    end
+
+    def transition_with_callbacks!(state_name)
+      transition! state_name.name
+      state_name.call(self)
+    end
+
+    def states
+      self.class.states
+    end
+
+    def default_state
+      self.class.default_state
+    end
+
+    def current_state
+      states[@state]
+    end
+
+    def current_state_name
+      current_state && current_state.name || ''
+    end
+
+    class State
+      attr_reader :name, :transitions
+
+      def initialize(name, transitions = nil, &block)
+        @name, @block = name, block
+        @transitions = nil
+        @transitions = Array(transitions).map(&:to_sym) if transitions
+      end
+
+      def call(obj)
+        obj.instance_eval(&@block) if @block
+      end
+
+      def valid_transition?(new_state)
+        # All transitions are allowed unless expressly
+        return true unless @transitions
+
+        @transitions.include? new_state.to_sym
+      end
+    end
+  end
+end

data/lib/listen/internals/thread_pool.rb

@@ -1,7 +1,6 @@
 module Listen
   # @private api
   module Internals
-    # Just a wrapper for tests to avoid interfereing with Celluloid's threads
     module ThreadPool
       def self.add(&block)
         Thread.new { block.call }.tap do |th|
@@ -11,6 +10,7 @@ module Listen
 
       def self.stop
         return unless @threads ||= nil
+        return if @threads.empty? # return to avoid using possibly stubbed Queue
 
         killed = Queue.new
         killed << @threads.pop.kill until @threads.empty?

data/lib/listen/listener.rb

@@ -1,31 +1,25 @@
-require 'pathname'
+require 'English'
 
 require 'listen/version'
-require 'listen/adapter'
-require 'listen/change'
-require 'listen/record'
-require 'listen/silencer'
-require 'listen/queue_optimizer'
-require 'English'
 
-require 'listen/internals/logging'
+require 'listen/backend'
 
-module Listen
-  class Listener
-    include Celluloid::FSM
-    include QueueOptimizer
+require 'listen/silencer'
+require 'listen/silencer/controller'
 
-    attr_accessor :block
+require 'listen/queue_optimizer'
 
-    attr_reader :silencer
+require 'listen/fsm'
 
-    # TODO: deprecate
-    attr_reader :options, :directories
-    attr_reader :registry, :supervisor
+require 'listen/event/loop'
+require 'listen/event/queue'
+require 'listen/event/config'
 
-    # TODO: deprecate
-    # NOTE: these are VERY confusing (broadcast + recipient modes)
-    attr_reader :host, :port
+require 'listen/listener/config'
+
+module Listen
+  class Listener
+    include Listen::FSM
 
     # Initializes the directories listener.
     #
@@ -37,73 +31,69 @@ module Listen
     # @yieldparam [Array<String>] added the list of added files
     # @yieldparam [Array<String>] removed the list of removed files
     #
-    def initialize(*args, &block)
-      @options = _init_options(args.last.is_a?(Hash) ? args.pop : {})
-
-      # Setup logging first
-      if Celluloid.logger
-        Celluloid.logger.level = _debug_level
-        _info "Celluloid loglevel set to: #{Celluloid.logger.level}"
-        _info "Listen version: #{Listen::VERSION}"
-      end
-
-      @silencer = Silencer.new
-      _reconfigure_silencer({})
-
-      @tcp_mode = nil
-      if [:recipient, :broadcaster].include? args[1]
-        target = args.shift
-        @tcp_mode = args.shift
-        _init_tcp_options(target)
-      end
-
-      @directories = args.flatten.map { |path| Pathname.new(path).realpath }
-      @queue = Queue.new
-      @block = block
-      @registry = Celluloid::Registry.new
+    def initialize(*dirs, &block)
+      options = dirs.last.is_a?(Hash) ? dirs.pop : {}
 
-      transition :stopped
+      @config = Config.new(options)
+
+      eq_config = Event::Queue::Config.new(@config.relative?)
+      queue = Event::Queue.new(eq_config) { @processor.wakeup_on_event }
+
+      silencer = Silencer.new
+      rules = @config.silencer_rules
+      @silencer_controller = Silencer::Controller.new(silencer, rules)
+
+      @backend = Backend.new(dirs, queue, silencer, @config)
+
+      optimizer_config = QueueOptimizer::Config.new(@backend, silencer)
+
+      pconfig = Event::Config.new(
+        self,
+        queue,
+        QueueOptimizer.new(optimizer_config),
+        @backend.min_delay_between_events,
+        &block)
+
+      @processor = Event::Loop.new(pconfig)
+
+      super() # FSM
     end
 
     default_state :initializing
 
-    state :initializing, to: :stopped
-    state :paused, to: [:processing, :stopped]
+    state :initializing, to: :backend_started
 
-    state :stopped, to: [:processing] do
-      _stop_wait_thread
-      if @supervisor
-        @supervisor.terminate
-        @supervisor = nil
-      end
+    state :backend_started, to: [:frontend_ready] do
+      backend.start
     end
 
-    state :processing, to: [:paused, :stopped] do
-      if wait_thread # means - was paused
-        _wakeup_wait_thread
-      else
-        @last_queue_event_time = nil
-        _start_wait_thread
-        _init_actors
+    state :frontend_ready, to: [:processing_events] do
+      processor.setup
+    end
 
-        # Note: make sure building is finished before starting adapter (for
-        # consistent results both in specs and normal usage)
-        sync(:record).build
+    state :processing_events, to: [:paused, :stopped] do
+      processor.resume
+    end
+
+    state :paused, to: [:processing_events, :stopped] do
+      processor.pause
+    end
 
-        _start_adapter
-      end
+    state :stopped, to: [:backend_started] do
+      backend.stop # should be before processor.teardown to halt events ASAP
+      processor.teardown
     end
 
     # Starts processing events and starts adapters
     # or resumes invoking callbacks if paused
     def start
-      transition :processing
+      transition :backend_started if state == :initializing
+      transition :frontend_ready if state == :backend_started
+      transition :processing_events if state == :frontend_ready
+      transition :processing_events if state == :paused
     end
 
-    # TODO: depreciate
-    alias_method :unpause, :start
-
-    # Stops processing and terminates all actors
+    # Stops both listening for events and processing them
     def stop
       transition :stopped
     end
@@ -115,257 +105,28 @@ module Listen
 
     # processing means callbacks are called
     def processing?
-      state == :processing
+      state == :processing_events
     end
 
     def paused?
       state == :paused
     end
 
-    # TODO: deprecate
-    alias_method :listen?, :processing?
-
-    # TODO: deprecate
-    def paused=(value)
-      transition value ? :paused : :processing
-    end
-
-    # TODO: deprecate
-    alias_method :paused, :paused?
-
-    # Add files and dirs to ignore on top of defaults
-    #
-    # (@see Listen::Silencer for default ignored files and dirs)
-    #
     def ignore(regexps)
-      _reconfigure_silencer(ignore: [options[:ignore], regexps])
+      @silencer_controller.append_ignores(regexps)
     end
 
-    # Replace default ignore patterns with provided regexp
     def ignore!(regexps)
-      _reconfigure_silencer(ignore: [], ignore!: regexps)
+      @silencer_controller.replace_with_bang_ignores(regexps)
     end
 
-    # Listen only to files and dirs matching regexp
     def only(regexps)
-      _reconfigure_silencer(only: regexps)
-    end
-
-    def async(type)
-      proxy = sync(type)
-      proxy ? proxy.async : nil
-    end
-
-    def sync(type)
-      @registry[type]
-    end
-
-    def queue(type, change, dir, path, options = {})
-      fail "Invalid type: #{type.inspect}" unless [:dir, :file].include? type
-      fail "Invalid change: #{change.inspect}" unless change.is_a?(Symbol)
-      fail "Invalid path: #{path.inspect}" unless path.is_a?(String)
-      if @options[:relative]
-        dir = begin
-                cwd = Pathname.pwd
-                dir.relative_path_from(cwd)
-              rescue ArgumentError
-                dir
-              end
-      end
-      @queue << [type, change, dir, path, options]
-
-      @last_queue_event_time = Time.now.to_f
-      _wakeup_wait_thread unless state == :paused
-
-      return unless @tcp_mode == :broadcaster
-
-      message = TCP::Message.new(type, change, dir, path, options)
-      registry[:broadcaster].async.broadcast(message.payload)
+      @silencer_controller.replace_with_only(regexps)
     end
 
     private
 
-    include Internals::Logging
-
-    def _init_options(options = {})
-      {
-        # Listener options
-        debug: false,
-        wait_for_delay: 0.1,
-        relative: false,
-
-        # Backend selecting options
-        force_polling: false,
-        polling_fallback_message: nil,
-
-      }.merge(options)
-    end
-
-    def _debug_level
-      debugging = ENV['LISTEN_GEM_DEBUGGING'] || options[:debug]
-      case debugging.to_s
-      when /2/
-        Logger::DEBUG
-      when /true|yes|1/i
-        Logger::INFO
-      else
-        Logger::ERROR
-      end
-    end
-
-    def _init_actors
-      adapter_options = { mq: self, directories: directories }
-
-      @supervisor = Celluloid::SupervisionGroup.run!(registry)
-      supervisor.add(Record, as: :record, args: self)
-      supervisor.pool(Change, as: :change_pool, args: self)
-
-      # TODO: broadcaster should be a separate plugin
-      if @tcp_mode == :broadcaster
-        require 'listen/tcp/broadcaster'
-
-        # TODO: pass a TCP::Config class to make sure host and port are properly
-        # passed, even when nil
-        supervisor.add(TCP::Broadcaster, as: :broadcaster, args: [@host, @port])
-
-        # TODO: should be auto started, because if it crashes
-        # a new instance is spawned by supervisor, but it's 'start' isn't
-        # called
-        registry[:broadcaster].start
-      elsif @tcp_mode == :recipient
-        # TODO: adapter options should be configured in Listen.{on/to}
-        adapter_options.merge!(host: @host, port: @port)
-      end
-
-      # TODO: refactor
-      valid_adapter_options = _adapter_class.const_get(:DEFAULTS).keys
-      valid_adapter_options.each do |key|
-        adapter_options.merge!(key => options[key]) if options.key?(key)
-      end
-
-      supervisor.add(_adapter_class, as: :adapter, args: [adapter_options])
-    end
-
-    def _wait_for_changes
-      latency = options[:wait_for_delay]
-
-      loop do
-        break if state == :stopped
-
-        if state == :paused || @queue.empty?
-          sleep
-          break if state == :stopped
-        end
-
-        # Assure there's at least latency between callbacks to allow
-        # for accumulating changes
-        now = Time.now.to_f
-        diff = latency + (@last_queue_event_time || now) - now
-        if diff > 0
-          sleep diff
-          next
-        end
-
-        _process_changes unless state == :paused
-      end
-    rescue RuntimeError
-      Kernel.warn _format_error('exception while processing events: %s %s')
-    end
-
-    def _silenced?(path, type)
-      @silencer.silenced?(path, type)
-    end
-
-    def _start_adapter
-      # Don't run async, because configuration has to finish first
-      adapter = sync(:adapter)
-      adapter.start
-    end
-
-    def _adapter_class
-      @adapter_class ||= Adapter.select(options)
-    end
-
-    # for easier testing without sleep loop
-    def _process_changes
-      return if @queue.empty?
-
-      @last_queue_event_time = nil
-
-      changes = []
-      changes << @queue.pop until @queue.empty?
-
-      return if block.nil?
-
-      hash = _smoosh_changes(changes)
-      result = [hash[:modified], hash[:added], hash[:removed]]
-
-      block_start = Time.now.to_f
-      # TODO: condition not tested, but too complex to test ATM
-      block.call(*result) unless result.all?(&:empty?)
-      _debug "Callback took #{Time.now.to_f - block_start} seconds"
-    end
-
-    attr_reader :wait_thread
-
-    def _init_tcp_options(target)
-      # Handle TCP options here
-      require 'listen/tcp'
-      fail ArgumentError, 'missing host/port for TCP' unless target
-
-      if @tcp_mode == :recipient
-        @host = 'localhost'
-        @options[:force_tcp] = true
-      end
-
-      if target.is_a? Fixnum
-        @port = target
-      else
-        @host, port = target.split(':')
-        @port = port.to_i
-      end
-    end
-
-    def _reconfigure_silencer(extra_options)
-      @options.merge!(extra_options)
-
-      # TODO: this should be directory specific
-      rules = [:only, :ignore, :ignore!].map do |option|
-        [option, @options[option]] if @options.key? option
-      end
-
-      @silencer.configure(Hash[rules.compact])
-    end
-
-    def _start_wait_thread
-      @wait_thread = Thread.new { _wait_for_changes }
-    end
-
-    def _wakeup_wait_thread
-      wait_thread.wakeup if wait_thread && wait_thread.alive?
-    end
-
-    def _stop_wait_thread
-      return unless wait_thread
-      if wait_thread.alive?
-        wait_thread.wakeup
-        wait_thread.join
-      end
-      @wait_thread = nil
-    end
-
-    def _queue_raw_change(type, dir, rel_path, options)
-      _debug { "raw queue: #{[type, dir, rel_path, options].inspect}" }
-
-      unless (worker = async(:change_pool))
-        _warn 'Failed to allocate worker from change pool'
-        return
-      end
-
-      worker.change(type, dir, rel_path, options)
-    rescue RuntimeError
-      _error_exception "_queue_raw_change exception %s:\n%s:\n"
-      raise
-    end
+    attr_reader :processor
+    attr_reader :backend
   end
 end