evesync 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9a1973a7b7ed425602bbb593496479cfb6ce24e57d5f490ee2d54c52e76375cb
+  data.tar.gz: a593092487578200014b495d75a77c20a9798525d00fe9e9ab94ed26b4eba8fd
+SHA512:
+  metadata.gz: b06baf168fac9f60cc20995b586712b1de0cf1f13ab32be858d916686d4a770ea67eba82f9b0967bee7b6cc1d664f788d5c4717489320d224a1f2e724ac1ad4c
+  data.tar.gz: 7178b8ffc3f5a120438ffd5a0ea1a61fbb3c1a2b9b4b55a90242fadd5b2abcc61379af372bd1575551076b59583d3472faab46214f4c9b3c5687cacf83cfd424

data/LICENSE

@@ -0,0 +1,25 @@
+BSD 2-Clause License
+
+Copyright (c) 2018, Valentine Kiselev
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/Rakefile

@@ -0,0 +1,83 @@
+require 'rspec/core/rake_task'
+require 'find'
+require 'mkmf'
+require_relative 'lib/evesync'
+VERSION = Evesync::VERSION
+GEMSPEC = 'evesync.gemspec'.freeze
+GEMFILE = "evesync-#{VERSION}.gem".freeze
+
+task default: %i[lint build]
+
+task :lint do
+  sh 'rubocop -l lib' if find_executable 'rubocop'
+end
+
+task :rdoc do
+  sh 'rdoc --ri'
+  sh 'rdoc'
+end
+
+task install: [GEMFILE] do
+  sh "gem install --local #{GEMFILE}"
+end
+
+task build: GEMFILE
+file GEMFILE do
+  sh "gem build #{GEMSPEC}"
+end
+
+RSpec::Core::RakeTask.new(:spec)
+
+task test: :spec
+
+task :clean, [:remove_rpm] do |_t, args|
+  args.with_defaults(:remove_rpm => 1)
+  rm_rf('tmp')
+  rm_rf('doc')
+  rm_rf('mkmf.log')
+  rm_rf(Dir['*.zip'])
+  rm_rf('RPM') if args[:remove_rpm] == 1
+  rm_rf("evesync-#{VERSION}")
+  rm_rf(GEMFILE)
+end
+
+task todos: :todo
+task :todo do
+  puts "==== \033[0;31mTODOs\033[0m in code"
+  puts `find bin lib dockerfiles -type f -exec grep --color=always TODO   \{} \+ ||:`
+  puts "==== \033[0;31mFIXMEs\033[0m in code"
+  puts `find bin lib dockerfiles -type f -exec grep --color=always FIXME  \{} \+ ||:`
+end
+
+task :lines do
+  sh 'find . -name "*.rb" -exec grep -v -E "^\s*#|^\s*$" \{} \+ | wc -l'
+end
+
+## Docker related targets
+
+task :docker do
+  sh 'docker-compose build'
+end
+
+task :up do
+  sh 'docker-compose up -d'
+end
+
+task :down do
+  sh 'docker-compose stop || docker-compose kill ||:'
+  sh 'docker-compose rm --force ||:'
+end
+
+task :rpm do
+  sh("rpmbuild -bb "\
+     "--define 'VERSION #{VERSION}' "\
+     "--define 'RELEASE `git rev-list HEAD master --count`' "\
+     "--define '_builddir #{Dir.pwd}' "\
+     "--define '_rpmdir #{Dir.pwd}/RPM/' "\
+     "evesync.spec")
+  sh('chown -R 1000:1000 RPM')
+end
+
+task :'build-rpm' do
+  sh('echo rake rpm | docker-compose run build-centos')
+end

data/bin/evedatad

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+
+require 'evesync/ipc/server'
+require 'evesync/log'
+require 'evesync/database'
+
+Evesync::Log.info('Database daemon starting...')
+
+database = Evesync::Database.new
+
+# Creating an IPC server to accept data
+data_server = Evesync::IPC::Server.new(
+  port: :evedatad,
+  proxy: database
+).start
+
+Signal.trap('TERM') do
+  data_server.stop
+  exit(0)
+end
+
+Evesync::Log.info('Database daemon started!')
+
+begin
+  loop do
+    sleep 3600 # FIXME: 1 hour, rewrite better
+    # zip db if needed, save backup
+  end
+rescue SignalException => e
+  data_server.stop
+  Evesync::Log.warn("Database daemon received signal: #{e.signm}(#{e.signo})")
+  exit(0)
+end

data/bin/evehand

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+
+require 'evesync/log'
+require 'evesync/ipc/server'
+require 'evesync/handler'
+
+Evesync::Log.info('Handle daemon starting...')
+
+all_handler = Evesync::Handler.new
+
+handler_server = Evesync::IPC::Server.new(
+  port: :evehand,
+  proxy: all_handler,
+  ip: '*' # accept remote requests
+).start
+
+Evesync::Log.info('Handle daemon started!')
+
+# Signal handling
+
+Signal.trap('TERM') do
+  handler_server.stop
+  exit(0)
+end
+
+begin
+  loop do
+    sleep 3600 # FIXME: 1 hour, rewrite better
+    # FIXME: know what to do
+  end
+rescue SignalException => e
+  handler_server.stop
+  Evesync::Log.warn("Received signal: #{e.signm}(#{e.signo})")
+  exit(0)
+end

data/bin/evemond

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+
+require 'evesync/log'
+require 'evesync/ipc/server'
+require 'evesync/trigger'
+require 'evesync/watcher'
+
+Evesync::Log.info 'Monitoring daemon starting...'
+
+queue = Queue.new
+trigger = Evesync::Trigger.new(queue)
+watcher = Evesync::Watcher.new(queue)
+
+[trigger, watcher].each(&:start)
+
+evesync_server = Evesync::IPC::Server.new(
+  port: :evemond,
+  proxy: trigger
+).start
+
+Evesync::Log.info 'Monitoring daemon started!'
+
+# Signal handling
+
+Signal.trap('SIGINT') do
+  [watcher, trigger].each(&:stop)
+  evesync_server.stop
+  exit(0)
+end
+
+begin
+  loop do
+    sleep 100
+    # FIXME: know what to do
+  end
+rescue SignalException => e
+  watcher.stop
+  evesync_server.stop
+  Evesync::Log.warn("Monitoring daemon received signal: #{e.signm}(#{e.signo})")
+  exit(0)
+end

data/bin/evesync

@@ -0,0 +1,164 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+
+require 'optparse'
+require 'socket'
+require 'find'
+require 'zip'
+require 'evesync'
+require 'evesync/log'
+require 'evesync/utils'
+require 'evesync/database'
+require 'evesync/config'
+require 'evesync/sync'
+require 'evesync/ipc/client'
+
+DB_TAG = 'db'
+FILES_TAG = 'files'
+
+opts = {}
+$program = File.basename($0)
+#Evesync::Log.level = :warn
+Evesync::Log.simple = true
+
+OptionParser.new do |parser|
+  parser.banner = "#{$program} [OPTIONS...]
+
+"
+  parser.summary_width = 20
+
+  parser.on('-l', '--load FILE', 'Load events from file') do |f|
+    unless File.file? f
+      puts "File is absent: #{f}"
+      exit 1
+    end
+    opts['load_file'] = f
+  end
+
+  parser.on('-z', '--zip [DEST]', 'Zip current database') do |d|
+
+    # Pretty default name
+    hostname = Socket.gethostname
+    hostname = hostname == 'localhost' \
+               ? Evesync::Utils::local_ip : hostname
+    time_now = Time.now.strftime('%Y.%m.%d-%H:%M:%S')
+    opts['zip'] = d || "#{$program}_#{hostname}_#{time_now}.zip"
+
+    # Validation
+    unless File.directory? File.dirname opts['zip']
+      puts "Directory doesn't exist: #{File.dirname opts['zip']}"
+      exit 1
+    end
+  end
+
+  parser.on('-f', '--force', 'Enforce synchronization') do
+    opts['sync'] = true
+  end
+
+  parser.on_tail('-v', '--version', "Version of #{$program}") do
+    puts Evesync::VERSION
+    exit
+  end
+
+  parser.on_tail('-h', '--help', 'Help message') do
+    puts parser
+    exit
+  end
+end.parse!
+
+def force_sync
+  sync_client = Evesync::IPC::Client.new(
+    port: :evesyncd
+  )
+  sync_client.synchronize
+end
+
+def files_in(folder)
+  begin
+    Find.find(folder).collect { |file| file }
+  rescue StandardError
+    puts "No files in #{folder}"
+    []
+  end
+end
+
+def zip(out)
+  databases = {
+    DB_TAG => {
+      Evesync::Config[:evedatad]['db_path'] =>
+      files_in(Evesync::Config[:evedatad]['db_path'])
+    },
+    FILES_TAG => {
+      Evesync::Config[:evedatad]['db_files_path'] =>
+      files_in(Evesync::Config[:evedatad]['db_files_path'])
+    }
+  }
+
+  Zip::File.open(out, Zip::File::CREATE) do |zip|
+    databases.each do |key, entries|
+      entries.each do |base_path, files|
+        files.each do |file|
+          relative_path = Pathname.new(file)
+                            .relative_path_from(Pathname.new(base_path))
+          zip.add(File.join(key, relative_path.to_s), file)
+        end
+      end
+    end
+  end
+end
+
+def load_file(filename)
+
+  Dir.mktmpdir($program) do |tmpdir|
+    # Unpacking...
+    Zip::File.open(filename) do |zip|
+      zip.each do |entry|
+        entry.extract(File.join(tmpdir, entry.name))
+      end
+    end
+
+    db_path = File.join(tmpdir, DB_TAG)
+    db_files_path = File.join(tmpdir, FILES_TAG)
+
+    loaded_db = Evesync::Database.new(
+      db_path,
+      db_files_path
+    )
+
+    local_db = Evesync::IPC::Client.new(
+      port: :evedatad
+    )
+
+    diff = Evesync::Sync::diff_missed(
+      v1: local_db.events,
+      v2: loaded_db.events
+    )
+
+    new_messages = loaded_db.messages(diff)
+
+    sync_client = Evesync::IPC::Client.new(
+      port: :evesyncd
+    )
+
+    sync_client.apply_events(new_messages)
+  end
+
+end
+
+def evesync(opts)
+  if opts['sync']
+    puts 'Synchronization enforced'
+    force_sync
+  end
+
+  if opts['load_file']
+    load_file opts['load_file']
+  end
+
+  if opts['zip']
+    zip(opts['zip'])
+    puts(opts['zip'])
+  end
+end
+
+evesync opts

data/bin/evesyncd

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+
+require 'evesync/log'
+require 'evesync/sync'
+require 'evesync/config'
+require 'evesync/ipc/server'
+
+# Exiting if `auto_discover = false'
+unless Evesync::Config['auto_discover']
+  Evesync::Log.info('Synchronizing not required')
+  exit(0)
+end
+
+Evesync::Log.info('Synchronizing daemon starting...')
+
+sync = Evesync::Sync.new
+
+sync_server = Evesync::IPC::Server.new(
+  port: :evesyncd,
+  proxy: sync
+).start
+
+Signal.trap('TERM') do
+  sync_server.stop
+  exit(0)
+end
+
+Evesync::Log.info('Synchronizing daemon started!')
+
+few_seconds = Evesync::Config['discover_timeout'].to_i
+
+sync.discover                   # discovering nodes
+sleep 3                         # giving 3 seconds to complete
+
+begin
+  loop do
+    sync.synchronize            # updating this node
+    sleep few_seconds           # ... in some interval
+  end
+rescue SignalException
+  sync_server.stop
+  exit(0)
+end

data/bin/start

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+trap 'echo Exiting; \
+	pkill evedatad; \
+	pkill evemond; \
+	pkill evehand; \
+	pkill evesyncd
+	exit 1' 2
+evedatad &
+evemond &
+evehand &
+sleep 3s
+evesyncd &
+
+while true; do
+    sleep 1000
+done

data/config/example.conf

@@ -0,0 +1,58 @@
+# This is a configuration file example
+# It contains all information about daemons that can be changed
+# It's TOML formatted file, so feel free to use this format
+# without any restrictions.
+#
+# For more information about TOML format see:
+#   - https://github.com/toml-lang/toml
+#   - https://en.wikipedia.org/wiki/TOML
+
+# Available log levels:
+# debug info warn error fatal
+loglevel = 'debug'
+# Delay in seconds for discovery package sending
+discover_timeout = 600
+# Automate discovering
+auto_discover = true
+
+ntp = 'us.pool.ntp.org'
+
+[evemond]			# System Monitor daemon
+port = 55443
+
+# A list of remote nodes ips to keep connected with
+remotes = [ ]
+
+# Files to watch and synchronize
+watch = [
+    '/etc/pam.d/login',
+    '/etc/pam.d/other',
+]
+
+# Interval for gathering files' changes
+watch_interval = 3
+
+# TODO:
+ignore_packages = [
+    'ruby',
+]
+
+# TODO:
+ignore_files = [
+    '/etc/hosts',
+]
+
+
+[evehand]			# System Handler daemon
+port = 55432
+
+
+[evedatad]			# System Data daemon
+port = 54321
+
+[evesyncd]
+port = 64653
+broadcast_port = 64564
+
+
+# Copyright (c) Kiselev Valentin 2019

data/lib/evesync/config.rb

@@ -0,0 +1,63 @@
+require 'toml-rb'
+require 'evesync/utils'
+require 'evesync/constants'
+require 'evesync/log'
+
+module Evesync
+  module Config
+    class << self
+
+      DEFAULTS = {
+        'ntp' => '',
+        'evemond' => {
+          'port'           => Constants::MOOND_PORT,
+          'remotes'        => [],
+          'watch'          => [],
+          'watch_interval' => Constants::WATCH_INTERVAL
+        },
+        'evedatad' => {
+          'port'           => Constants::DATAD_PORT,
+          'db_path'        => Constants::DB_PATH,
+          'db_files_path'  => Constants::DB_FILES_PATH
+        },
+        'evehand'  => {
+          'port' => Constants::HAND_PORT
+        },
+        'evesyncd' => {
+          'port' => Constants::SYNC_PORT
+        },
+        'discover_timeout' => Constants::DISCOVER_TIMEOUT
+      }
+
+
+      def [](daemon)
+        read_config if needs_reading
+
+        @@config[daemon.to_s]
+      end
+
+      def reread
+        read_config
+      end
+
+      private
+
+      def read_config
+        config = begin
+                   TomlRB.load_file(Constants::CONFIG_FILE)
+                 rescue StandardError => e
+                   Log.error("Config ERROR: Couldn't parse file #{Constants::CONFIG_FILE}")
+                   Log.error("Config ERROR MESSAGE: #{e}")
+                   Log.error("Config ERROR: Using default configuration")
+                   {}
+                 end
+        @@config = DEFAULTS.deep_merge(config)
+        Log.info("Config initialized!")
+      end
+
+      def needs_reading
+        ! defined? @@config
+      end
+    end
+  end
+end

data/lib/evesync/constants.rb

@@ -0,0 +1,17 @@
+module Evesync
+  module Constants
+    MOOND_PORT = '55443'.freeze
+    DATAD_PORT = '54321'.freeze
+    HAND_PORT  = '55432'.freeze
+    SYNC_PORT  = '64653'.freeze
+
+    CONFIG_FILE   = '/etc/evesync.conf'.freeze
+    DB_PATH       = '/var/lib/evesync/db/'.freeze
+    DB_FILES_PATH = '/var/lib/evesync/files/'.freeze
+
+    DEFAULT_LOGLEVEL = 'debug'.freeze
+
+    DISCOVER_TIMEOUT = 3600
+    WATCH_INTERVAL   = 2
+  end
+end

data/lib/evesync/database.rb

@@ -0,0 +1,107 @@
+require 'fileutils'
+require 'json'
+require 'lmdb'
+require 'evesync/config'
+require 'evesync/constants'
+require 'evesync/ipc/data/package'
+require 'evesync/ipc/data/file'
+require 'evesync/ipc/data/ignore'
+
+module Evesync
+
+  ##
+  # *Database* class is a proxy for *evedatad* daemon
+  # implements at least one method: +save+. Allows
+  # Local +evemond+ save messages about changes
+  #
+  # Messages should be serializable (JSON)
+  #
+  # TODO:
+  #  * Think about how it can be widened
+
+  class Database
+    def initialize(db_path=nil, db_files_path=nil)
+      path = db_path || Config[:evedatad]['db_path']
+
+      unless ::File.exist? path
+        # FIXME: only root. handle exception
+        FileUtils.mkdir_p(path)
+      end
+      @env = LMDB.new(path)
+      @db = @env.database
+      @files_path = db_files_path || Config[:evedatad]['db_files_path']
+    end
+
+    # Save message to database, key is timestamp+object
+    def save(message)
+      Log.debug("Database save: #{message}")
+      db_add_entry(message)
+
+      if message.is_a? Evesync::IPC::Data::File
+        Log.debug("Database save file action: #{message.action}")
+        unless message.action ==
+               IPC::Data::File::Action::DELETE
+          save_file(message)
+        end
+      end
+      true
+    end
+
+    # Events simplified: object => [timestamp...]
+    def events
+      events = {}
+      @db.each do |key, _|
+        timestamp, object = parse_event(key)
+        events[object] ||= []
+        events[object].push(timestamp)
+      end
+      events
+    end
+
+    # Messages for events: object => {timestamp => message}
+    def messages(events)
+      ev_msgs = {}
+      @db.each do |key, message|
+        timestamp, object = parse_event(key)
+        next unless events.include?(object)
+
+        ev_msgs[object] ||= {}
+        if events[object].include?(timestamp)
+          ev_msgs[object][timestamp] = message
+        end
+      end
+      ev_msgs
+    end
+
+    private
+
+    def db_add_entry(message)
+      Log.debug('Database adding entry...')
+      key = create_key(message)
+      value = create_value(message)
+      @db[key] = value
+      Log.debug('Database adding entry done!')
+    end
+
+    def create_key(message)
+      "#{message.timestamp}_#{message.name}"
+    end
+
+    def create_value(message)
+      message.to_hash.to_json
+    end
+
+    def parse_event(key)
+      key.split('_', 2)
+    end
+
+    def save_file(file)
+      Log.debug('Database saving file...')
+      fulldest = File.join(@files_path,
+                           file.name + ".#{file.timestamp}")
+      FileUtils.mkdir_p(File.dirname(fulldest))
+      FileUtils.cp(file.name, fulldest)
+      Log.debug('Database saving file done!')
+    end
+  end
+end