evesync 1.0.2

data/lib/evesync/sync.rb

@@ -0,0 +1,209 @@
+require 'socket'
+require 'full_dup'
+require 'evesync/discover'
+require 'evesync/log'
+require 'evesync/config'
+require 'evesync/ipc/client'
+require 'evesync/ipc/data/utils'
+require 'evesync/utils'
+
+module Evesync
+  class Sync
+    def initialize
+      @discovery = Discover.new
+      @monitor = IPC::Client.new(
+        port: :evemond
+      )
+      @database = IPC::Client.new(
+        port: :evedatad
+      )
+      @handler = IPC::Client.new(
+        port: :evehand
+      )
+    end
+
+    ##
+    # Starting Synchronization between nodes that are
+    # found. Checking if all events are synchronized and
+    # synchronizing missing events.
+    #
+    # TODO:
+    #   * Catch the time when an event is sent while
+    #     synchronizing
+
+    def synchronize
+      Log.debug('Synchronizing starting...')
+      apply_events fetch_events missed_events
+      Log.debug('Synchronizing done!')
+    end
+
+    ##
+    # Sending a discovery message to broadcast.
+    # Local UDP socket listens and responses for handling answers.
+
+    def discover
+      @discovery.send_discovery_message
+    end
+
+    def apply_events(events)
+      events.each do |_, message|
+        message.values.each do |json|
+          ipc_message = IPC::Data.from_json(json)
+          @handler.handle(ipc_message)
+        end
+      end
+    end
+
+    # Diffs missed of `v1' that `v2' contain
+    def self.diff_missed(params)
+      v1 = params[:v1]
+      v2 = params[:v2]
+
+      # Fully missed objects
+      fully_missed = v2.reject { |k| v1.include?(k) }
+
+      # Included in both, but may be missed in `v1'
+      maybe_missed = v2.select { |k| v1.include?(k) }
+
+      not_relevant = maybe_missed.select do |k, v|
+        v.max > v1[k].max
+      end
+
+      partially_missed = not_relevant.map do |k, v|
+        [k, v.select { |tms| tms > v1[k].max }]
+      end.to_h
+
+      fully_missed.merge(partially_missed)
+    end
+
+    private
+
+    # We only recieve, dont push events to synchronize.
+    # This is because some node may be setted not to
+    # synchronize, so we don't want to make them synching.
+    def missed_events
+      remote_events = {}
+      remote_handlers = @monitor.remote_handlers
+
+      return {} unless remote_handlers.respond_to? :each
+
+      remote_handlers.each do |handler|
+        begin
+          Log.debug('Synchronizing with host (IP):', handler.ip)
+          remote_events[handler.ip] = handler.events || {}
+        rescue
+          next
+        end
+      end
+
+      return {} if remote_events.empty?
+
+      local_events = @database.events
+
+      events_diff(
+        local: local_events,
+        remote: remote_events
+      )
+    end
+
+    # Using Longest common subsequence problem solution
+    # we find timestamps that are absent in our database.
+    #
+    # Order doesn't matter because we sort events
+    #
+    # May be consider using any existing solution
+    def events_diff(params)
+      # params:
+      #   local = {object [...events]}
+      #   remote = {ip => {object => [...events]}
+      # convert to
+      #   remote = {object => {event => [..ips]}}
+      # then
+      #   use remote part {object => [...events]} and
+      #   compare to local, then get object-event that are
+      #   going to be fetched and apply ips (the choice may be random)
+      #   that can be used to fetch these events
+      local = params[:local]
+      remote = params[:remote]
+      Log.debug('Synchronizing remote objects:', remote)
+      # Transforming data
+      remote_objects = {}
+      remote.each do |ip, objects|
+        objects.each do |object, events|
+          remote_objects[object] ||= {}
+          events.each do |event|
+            remote_objects[object][event] ||= []
+            remote_objects[object][event].push(ip)
+          end
+        end
+      end
+
+      # Applying algorithm
+      diff = self.class.diff_missed(
+        v1: local,
+        v2: remote_objects.map do |k, v|
+          [k, v.keys]
+        end.to_h
+      )
+
+      # Returning duplicate
+      # remote_diff = remote_objects.full_dup
+      # remote_diff
+      diff.map do |obj, tms|
+        [
+          obj,                  # 1
+          tms.map do |t|
+            [t, remote_objects[obj][t]]
+          end.to_h              # 2
+        ]
+      end.to_h
+    end
+
+    # Fetch events from given diff.
+    #   events_diff: {object => {event => [ip..]}}
+    def fetch_events(events_diff)
+      if events_diff.empty?
+        Log.info('Synchronizing no events')
+        return {}
+      end
+
+      # Getting {ip => handler} map
+      handlers = {}
+      @monitor.remote_handlers.each do |handler|
+        handlers[handler.ip] = handler
+      end
+
+      # Mapping events to nodes: {ip => {object => [events...]}}
+      nodes_events = map_nodes_for_events(events_diff, handlers)
+
+      # Fetching
+      messages = {}
+      nodes_events.each do |ip, events|
+        messages.merge! handlers[ip].messages(events)
+      end
+      Log.debug('Synchronizing events fetched:', messages)
+      messages
+    end
+
+    # Map events to appropriate nodes that can be used to
+    # fetch events.
+    # TODO: intellectual choosing the nodes to fetch msgs from.
+    # Now choosing the firs matched one for most of the events.
+    def map_nodes_for_events(events_diff, handlers)
+      nodes_events = {}
+      events_diff.each do |object, events|
+        events.each do |event, nodes|
+          handlers.keys.each do |ip|
+            if nodes.include?(ip)
+              nodes_events[ip] ||= {}
+              nodes_events[ip][object] ||= []
+              nodes_events[ip][object].push(event)
+              break
+            end
+          end
+        end
+      end
+      nodes_events
+    end
+  end
+end

data/lib/evesync/trigger/base.rb

@@ -0,0 +1,56 @@
+require 'timeout'
+require 'evesync/log'
+
+module Evesync
+  class Trigger
+    module Base
+      # db must have a realization of _save_ method
+      def save_to_db(db, message)
+        db.save(message)
+      end
+
+      # Every element in *remotes* must have a realization
+      # of _handle_ method
+      def send_to_remotes(remotes, message)
+        remotes.each do |evehand|
+          begin
+            Timeout.timeout(30) do # FIXME: take from Config
+              evehand.handle(message)
+            end
+          rescue Timeout::Error
+            Log.warn("Trigger remote timeout: server #{evehand.uri} " \
+                     'is not accessible')
+          end
+        end
+      end
+
+      def process(message)
+        if ignore?(message)
+          unignore(message)
+          false
+        else
+          if save_to_db(@db, message)
+            send_to_remotes(@remotes, message)
+            true
+          end
+        end
+      end
+
+      def ignore(ipc_data)
+        @ignore << ipc_data if
+          ipc_data.is_a? data_class
+      end
+
+      def unignore(ipc_data)
+        @ignore.delete_if { |d| d == ipc_data }
+      end
+
+      private
+
+      def ignore?(ipc_data)
+        Log.debug("File ignore aray: #{@ignore}")
+        @ignore.find { |d| d == ipc_data }
+      end
+    end
+  end
+end

data/lib/evesync/trigger/file.rb

@@ -0,0 +1,20 @@
+require 'evesync/trigger/base'
+require 'evesync/ipc/data/file'
+
+module Evesync
+  class Trigger
+    class File
+      include Trigger::Base
+
+      def initialize(params)
+        @ignore = []
+        @db = params[:db]
+        @remotes = params[:remotes]
+      end
+
+      def data_class
+        IPC::Data::File
+      end
+    end
+  end
+end

data/lib/evesync/trigger/package.rb

@@ -0,0 +1,20 @@
+require 'evesync/trigger/base'
+require 'evesync/ipc/data/package'
+
+module Evesync
+  class Trigger
+    class Package
+      include Trigger::Base
+
+      def initialize(params)
+        @ignore = []
+        @db = params[:db]
+        @remotes = params[:remotes]
+      end
+
+      def data_class
+        IPC::Data::Package
+      end
+    end
+  end
+end

data/lib/evesync/trigger.rb

@@ -0,0 +1,106 @@
+require 'evesync/trigger/file'
+require 'evesync/trigger/package'
+require 'evesync/config'
+require 'evesync/log'
+require 'evesync/ipc/client'
+require 'evesync/utils'
+
+module Evesync
+  class Trigger
+    def initialize(watcher_queue)
+      @watcher_queue = watcher_queue
+
+      # Local Data daemon
+      evedatad = IPC::Client.new(port: :evedatad)
+
+      @remote_handlers = Config[:evemond]['remotes'].map do |ip|
+        new_remote_handler(ip)
+      end.compact # remove nils
+
+      # Helper triggers
+      package_trigger = Trigger::Package.new(
+        db: evedatad,
+        remotes: @remote_handlers
+      )
+      file_trigger = Trigger::File.new(
+        db: evedatad,
+        remotes: @remote_handlers
+      )
+
+      @triggers = [package_trigger, file_trigger]
+
+      Log.debug('Trigger initialization done!')
+    end
+
+    def start
+      @thr = Thread.new do
+        loop { biz }
+      end
+      Log.debug('Trigger started')
+    end
+
+    def stop
+      @thr.exit
+      Log.debug('Trigger stopped')
+    end
+
+    def ignore(change)
+      trigger_method(:ignore, change)
+    end
+
+    def unignore(change)
+      trigger_method(:unignore, change)
+    end
+
+    def add_remote_node(ip)
+      unless @remote_handlers.find { |h| h.ip == ip }
+        remote_handler = new_remote_handler(ip)
+        @remote_handlers << remote_handler if remote_handler
+      end
+      Log.debug 'Trigger actual remote nodes:', @remote_handlers.map(&:ip)
+    end
+
+    attr_reader :remote_handlers
+
+    private
+
+    # Main thread business logic goes here
+    def biz
+      change = @watcher_queue.pop
+      Log.info "Trigger dequed event: #{change}"
+      trigger = message_trigger(change)
+      trigger.process(change)
+    end
+
+    # Send a method to target (choose by change class name)
+    def trigger_method(method, change)
+      Log.debug("Trigger calling '#{method}' on '#{change.class.name}'")
+
+      trigger = message_trigger(change)
+
+      if trigger
+        trigger.send(method, change) && true
+      else
+        # TODO: forward somewhere
+        Log.error('Trigger: no watchers will be notified on ' \
+                  "#{change}")
+      end
+    end
+
+    def message_trigger(message)
+      class_last = message.class.name.to_s.split('::')[-1]
+      @triggers.find do |trigger|
+        trigger.to_s.include? class_last
+      end
+    end
+
+    def new_remote_handler(ip)
+      unless Utils.local_ip?(ip)
+        IPC::Client.new(
+          port: :evehand,
+          ip: ip
+        )
+      end
+    end
+  end
+end

data/lib/evesync/utils.rb

@@ -0,0 +1,51 @@
+module Evesync
+  module Utils
+    class << self
+      def local_ip?(ip)
+        ips = `getent hosts #{ip}`
+                .lines
+                .map(&:split)
+                .map(&:first)
+        loc_ips = local_ips
+
+        !(ips & loc_ips).empty?
+      end
+
+      def local_ip
+        local_ips.first
+      end
+
+      def local_ips
+        `ip a`
+          .lines
+          .grep(/inet/)
+          .map(&:split)
+          .map { |lines| lines[1].split('/')[0] }
+      end
+
+    end
+  end
+end
+
+# For ruby < 2.1
+class Array
+  unless defined? to_h
+    def to_h
+      Hash[*flatten(1)]
+    end
+  end
+end
+
+class Hash
+  unless defined? deep_merge
+    def deep_merge(h)
+      self.merge(h) do |_k, a, b|
+        if a.is_a? Hash
+          a.deep_merge(b)
+        else
+          b
+        end
+      end
+    end
+  end
+end

data/lib/evesync/watcher/file.rb

@@ -0,0 +1,156 @@
+require 'date'
+require 'rb-inotify'
+require 'evesync/config'
+require 'evesync/ntp'
+require 'evesync/ipc/data/file'
+require 'evesync/watcher/interface'
+
+module Evesync
+  class Watcher
+
+    ##
+    # Watches the files and directories, defined in
+    # configuration attribute _watch_
+    #
+    # TODO:
+    #   * Test on various cases, make it work properly
+    #   * Find out all possible occasions
+
+    class File < Watcher::Interface
+      def initialize(queue)
+        @queue = queue
+        @watches = Config[:evemond]['watch']
+        @period = Config[:evemond]['watch_interval'].to_i
+        @inotify = INotify::Notifier.new
+        @events = {}
+        @wfiles = []
+        @wdirs = []
+        initialize_watcher
+      end
+
+      def start
+        @inotify_thr = Thread.new { @inotify.run }
+        @main_thr = Thread.new do
+          loop do
+            sleep @period
+            send_events
+          end
+        end
+      end
+
+      def stop
+        @inotify.stop
+        @inotify_thr.exit
+        @main_thr.exit
+      end
+
+      private
+
+      # Send all events from the methods-handlers
+      def send_events
+        @events.each do |file, events|
+          event = guess_event(events)
+          mode = case event
+                 when :delete then nil
+                 else ::File::Stat.new(file).mode
+                 end
+
+          msg = IPC::Data::File.new(
+            name: file,
+            mode: mode,
+            action: event,
+            touched_at: NTP.time.to_s
+          )
+          @queue.push msg
+          Log.debug("Watcher File guessed event #{event} " \
+                    "from #{events} on #{file}")
+        end
+        @events = {}
+      end
+
+      def initialize_watcher
+        @watches.each do |filename|
+          unless ::File.exist? filename
+            Log.error("Watcher File: '#{filename}' absent on system")
+          end
+
+          # TODO: ignore /dev and /sys, /proc directories
+          if ::File.file? filename
+            watch_file filename
+          elsif ::File.directory? filename
+            watch_directory filename
+          else
+            # Seems to be a drive or
+            Log.warn("Watcher File: watching '#{filename}' is not implemented yet")
+          end
+        end
+      end
+
+      def watch_file(filename)
+        # add file modify watch
+        Log.debug("Watcher File watching #{filename}")
+
+        @inotify.watch(filename, :modify) do |e|
+          Log.debug("Watcher File: MODIFIED #{e.absolute_name}")
+          h_file(e.absolute_name, [:modify]) # the only flag we need
+        end
+
+        # Waiting for file to disappear and created again
+        @wfiles << filename unless @wfiles.include?(filename)
+        dirname = ::File.dirname(filename)
+        unless @wdirs.include?(dirname)
+          watch_directory_for_file(dirname)
+          @wdirs << dirname
+        end
+      end
+
+      def watch_directory(dirname)
+        @inotify.watch(dirname, :create, :delete, :moved_to, :moved_from) do |e|
+          Log.debug("Watcher File: DIRECTORY #{e.absolute_name}")
+          h_directory(e.absolute_name, e.flags)
+        end
+      end
+
+      def watch_directory_for_file(dirname)
+        @inotify.watch(dirname, :create, :moved_to) do |e|
+          watch_file(e.absolute_name) if @wfiles.include? e.absolute_name
+        end
+      end
+
+      # Handlers of inotify changes
+
+      def h_file(filename, events)
+        @events[filename] = [] unless @events[filename]
+
+        @events[filename] += events
+        Log.debug("Watcher File added events for file #{filename}")
+      end
+
+      def h_directory(filename, events)
+        if ::File.file? filename
+          watch_file(filename)
+          @watches << filename unless @watches.include? filename
+        elsif ::File.directory? filename
+          watch_directory(filename)
+        end
+
+        # Better pass a file
+        @events[filename] = [] unless @events[filename]
+
+        @events[filename] += events
+        Log.debug("Watcher File added events for directory #{filename}")
+      end
+
+      def guess_event(events)
+        return events.first if events.length == 1
+
+        return :modify if events.include?(:create) && events.include?(:modify)
+
+        return :delete if events.include?(:delete) && (!events.include? :create)
+
+        # TODO: find out more logic
+        events.last
+      end
+    end
+  end
+end

data/lib/evesync/watcher/interface.rb

@@ -0,0 +1,21 @@
+module Evesync
+  class Watcher
+    # Base watcher abstract class with methods for other
+    # watchers to implement
+    class Interface
+      # The class must be initialized with the queue object
+      def initialize(_queue)
+        raise NotImplementedError, "must implement 'initialize'"
+      end
+
+      # The watcher must be able to handle start and stop calls
+      def start
+        raise NotImplementedError, "must implement 'start'"
+      end
+
+      def stop
+        raise NotImplementedError, "must implement 'stop'"
+      end
+    end
+  end
+end

data/lib/evesync/watcher/package.rb

@@ -0,0 +1,8 @@
+require 'evesync/os/linux'
+
+module Evesync
+  class Watcher
+    # Package class is a reference to Distro::PackageWatcher
+    Package = Evesync::OS::PackageWatcher
+  end
+end