evesync 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +25 -0
  3. data/Rakefile +83 -0
  4. data/bin/evedatad +34 -0
  5. data/bin/evehand +36 -0
  6. data/bin/evemond +42 -0
  7. data/bin/evesync +164 -0
  8. data/bin/evesyncd +44 -0
  9. data/bin/start +16 -0
  10. data/config/example.conf +58 -0
  11. data/lib/evesync/config.rb +63 -0
  12. data/lib/evesync/constants.rb +17 -0
  13. data/lib/evesync/database.rb +107 -0
  14. data/lib/evesync/discover.rb +77 -0
  15. data/lib/evesync/err.rb +25 -0
  16. data/lib/evesync/handler/file.rb +28 -0
  17. data/lib/evesync/handler/package.rb +30 -0
  18. data/lib/evesync/handler.rb +89 -0
  19. data/lib/evesync/ipc/client.rb +37 -0
  20. data/lib/evesync/ipc/data/file.rb +64 -0
  21. data/lib/evesync/ipc/data/hashable.rb +74 -0
  22. data/lib/evesync/ipc/data/ignore.rb +22 -0
  23. data/lib/evesync/ipc/data/package.rb +58 -0
  24. data/lib/evesync/ipc/data/utils.rb +14 -0
  25. data/lib/evesync/ipc/data.rb +50 -0
  26. data/lib/evesync/ipc/ipc.rb +42 -0
  27. data/lib/evesync/ipc/server.rb +56 -0
  28. data/lib/evesync/log.rb +66 -0
  29. data/lib/evesync/ntp.rb +16 -0
  30. data/lib/evesync/os/linux/arch/package_manager.rb +29 -0
  31. data/lib/evesync/os/linux/arch/package_watcher.rb +59 -0
  32. data/lib/evesync/os/linux/arch.rb +2 -0
  33. data/lib/evesync/os/linux/base_package_manager.rb +80 -0
  34. data/lib/evesync/os/linux/deb/dpkg.rb +30 -0
  35. data/lib/evesync/os/linux/deb/package_manager.rb +38 -0
  36. data/lib/evesync/os/linux/deb/package_watcher.rb +32 -0
  37. data/lib/evesync/os/linux/deb.rb +2 -0
  38. data/lib/evesync/os/linux/rhel/package_manager.rb +42 -0
  39. data/lib/evesync/os/linux/rhel/package_watcher.rb +32 -0
  40. data/lib/evesync/os/linux/rhel/rpm.rb +41 -0
  41. data/lib/evesync/os/linux/rhel.rb +3 -0
  42. data/lib/evesync/os/linux.rb +9 -0
  43. data/lib/evesync/os.rb +2 -0
  44. data/lib/evesync/sync.rb +209 -0
  45. data/lib/evesync/trigger/base.rb +56 -0
  46. data/lib/evesync/trigger/file.rb +20 -0
  47. data/lib/evesync/trigger/package.rb +20 -0
  48. data/lib/evesync/trigger.rb +106 -0
  49. data/lib/evesync/utils.rb +51 -0
  50. data/lib/evesync/watcher/file.rb +156 -0
  51. data/lib/evesync/watcher/interface.rb +21 -0
  52. data/lib/evesync/watcher/package.rb +8 -0
  53. data/lib/evesync/watcher.rb +68 -0
  54. data/lib/evesync.rb +3 -0
  55. metadata +198 -0
@@ -0,0 +1,209 @@
1
+ require 'socket'
2
+ require 'full_dup'
3
+ require 'evesync/discover'
4
+ require 'evesync/log'
5
+ require 'evesync/config'
6
+ require 'evesync/ipc/client'
7
+ require 'evesync/ipc/data/utils'
8
+ require 'evesync/utils'
9
+
10
+ module Evesync
11
+ class Sync
12
+ def initialize
13
+ @discovery = Discover.new
14
+ @monitor = IPC::Client.new(
15
+ port: :evemond
16
+ )
17
+ @database = IPC::Client.new(
18
+ port: :evedatad
19
+ )
20
+ @handler = IPC::Client.new(
21
+ port: :evehand
22
+ )
23
+ end
24
+
25
+ ##
26
+ # Starting Synchronization between nodes that are
27
+ # found. Checking if all events are synchronized and
28
+ # synchronizing missing events.
29
+ #
30
+ # TODO:
31
+ # * Catch the time when an event is sent while
32
+ # synchronizing
33
+
34
+ def synchronize
35
+ Log.debug('Synchronizing starting...')
36
+ apply_events fetch_events missed_events
37
+ Log.debug('Synchronizing done!')
38
+ end
39
+
40
+ ##
41
+ # Sending a discovery message to broadcast.
42
+ # Local UDP socket listens and responses for handling answers.
43
+
44
+ def discover
45
+ @discovery.send_discovery_message
46
+ end
47
+
48
+ def apply_events(events)
49
+ events.each do |_, message|
50
+ message.values.each do |json|
51
+ ipc_message = IPC::Data.from_json(json)
52
+ @handler.handle(ipc_message)
53
+ end
54
+ end
55
+ end
56
+
57
+ # Diffs missed of `v1' that `v2' contain
58
+ def self.diff_missed(params)
59
+ v1 = params[:v1]
60
+ v2 = params[:v2]
61
+
62
+ # Fully missed objects
63
+ fully_missed = v2.reject { |k| v1.include?(k) }
64
+
65
+ # Included in both, but may be missed in `v1'
66
+ maybe_missed = v2.select { |k| v1.include?(k) }
67
+
68
+ not_relevant = maybe_missed.select do |k, v|
69
+ v.max > v1[k].max
70
+ end
71
+
72
+ partially_missed = not_relevant.map do |k, v|
73
+ [k, v.select { |tms| tms > v1[k].max }]
74
+ end.to_h
75
+
76
+ fully_missed.merge(partially_missed)
77
+ end
78
+
79
+ private
80
+
81
+ # We only recieve, dont push events to synchronize.
82
+ # This is because some node may be setted not to
83
+ # synchronize, so we don't want to make them synching.
84
+ def missed_events
85
+ remote_events = {}
86
+ remote_handlers = @monitor.remote_handlers
87
+
88
+ return {} unless remote_handlers.respond_to? :each
89
+
90
+ remote_handlers.each do |handler|
91
+ begin
92
+ Log.debug('Synchronizing with host (IP):', handler.ip)
93
+ remote_events[handler.ip] = handler.events || {}
94
+ rescue
95
+ next
96
+ end
97
+ end
98
+
99
+ return {} if remote_events.empty?
100
+
101
+ local_events = @database.events
102
+
103
+ events_diff(
104
+ local: local_events,
105
+ remote: remote_events
106
+ )
107
+ end
108
+
109
+ # Using Longest common subsequence problem solution
110
+ # we find timestamps that are absent in our database.
111
+ #
112
+ # Order doesn't matter because we sort events
113
+ #
114
+ # May be consider using any existing solution
115
+ def events_diff(params)
116
+ # params:
117
+ # local = {object [...events]}
118
+ # remote = {ip => {object => [...events]}
119
+ # convert to
120
+ # remote = {object => {event => [..ips]}}
121
+ # then
122
+ # use remote part {object => [...events]} and
123
+ # compare to local, then get object-event that are
124
+ # going to be fetched and apply ips (the choice may be random)
125
+ # that can be used to fetch these events
126
+ local = params[:local]
127
+ remote = params[:remote]
128
+ Log.debug('Synchronizing remote objects:', remote)
129
+ # Transforming data
130
+ remote_objects = {}
131
+ remote.each do |ip, objects|
132
+ objects.each do |object, events|
133
+ remote_objects[object] ||= {}
134
+ events.each do |event|
135
+ remote_objects[object][event] ||= []
136
+ remote_objects[object][event].push(ip)
137
+ end
138
+ end
139
+ end
140
+
141
+ # Applying algorithm
142
+ diff = self.class.diff_missed(
143
+ v1: local,
144
+ v2: remote_objects.map do |k, v|
145
+ [k, v.keys]
146
+ end.to_h
147
+ )
148
+
149
+ # Returning duplicate
150
+ # remote_diff = remote_objects.full_dup
151
+ # remote_diff
152
+ diff.map do |obj, tms|
153
+ [
154
+ obj, # 1
155
+ tms.map do |t|
156
+ [t, remote_objects[obj][t]]
157
+ end.to_h # 2
158
+ ]
159
+ end.to_h
160
+ end
161
+
162
+ # Fetch events from given diff.
163
+ # events_diff: {object => {event => [ip..]}}
164
+ def fetch_events(events_diff)
165
+ if events_diff.empty?
166
+ Log.info('Synchronizing no events')
167
+ return {}
168
+ end
169
+
170
+ # Getting {ip => handler} map
171
+ handlers = {}
172
+ @monitor.remote_handlers.each do |handler|
173
+ handlers[handler.ip] = handler
174
+ end
175
+
176
+ # Mapping events to nodes: {ip => {object => [events...]}}
177
+ nodes_events = map_nodes_for_events(events_diff, handlers)
178
+
179
+ # Fetching
180
+ messages = {}
181
+ nodes_events.each do |ip, events|
182
+ messages.merge! handlers[ip].messages(events)
183
+ end
184
+ Log.debug('Synchronizing events fetched:', messages)
185
+ messages
186
+ end
187
+
188
+ # Map events to appropriate nodes that can be used to
189
+ # fetch events.
190
+ # TODO: intellectual choosing the nodes to fetch msgs from.
191
+ # Now choosing the firs matched one for most of the events.
192
+ def map_nodes_for_events(events_diff, handlers)
193
+ nodes_events = {}
194
+ events_diff.each do |object, events|
195
+ events.each do |event, nodes|
196
+ handlers.keys.each do |ip|
197
+ if nodes.include?(ip)
198
+ nodes_events[ip] ||= {}
199
+ nodes_events[ip][object] ||= []
200
+ nodes_events[ip][object].push(event)
201
+ break
202
+ end
203
+ end
204
+ end
205
+ end
206
+ nodes_events
207
+ end
208
+ end
209
+ end
@@ -0,0 +1,56 @@
1
+ require 'timeout'
2
+ require 'evesync/log'
3
+
4
+ module Evesync
5
+ class Trigger
6
+ module Base
7
+ # db must have a realization of _save_ method
8
+ def save_to_db(db, message)
9
+ db.save(message)
10
+ end
11
+
12
+ # Every element in *remotes* must have a realization
13
+ # of _handle_ method
14
+ def send_to_remotes(remotes, message)
15
+ remotes.each do |evehand|
16
+ begin
17
+ Timeout.timeout(30) do # FIXME: take from Config
18
+ evehand.handle(message)
19
+ end
20
+ rescue Timeout::Error
21
+ Log.warn("Trigger remote timeout: server #{evehand.uri} " \
22
+ 'is not accessible')
23
+ end
24
+ end
25
+ end
26
+
27
+ def process(message)
28
+ if ignore?(message)
29
+ unignore(message)
30
+ false
31
+ else
32
+ if save_to_db(@db, message)
33
+ send_to_remotes(@remotes, message)
34
+ true
35
+ end
36
+ end
37
+ end
38
+
39
+ def ignore(ipc_data)
40
+ @ignore << ipc_data if
41
+ ipc_data.is_a? data_class
42
+ end
43
+
44
+ def unignore(ipc_data)
45
+ @ignore.delete_if { |d| d == ipc_data }
46
+ end
47
+
48
+ private
49
+
50
+ def ignore?(ipc_data)
51
+ Log.debug("File ignore aray: #{@ignore}")
52
+ @ignore.find { |d| d == ipc_data }
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,20 @@
1
+ require 'evesync/trigger/base'
2
+ require 'evesync/ipc/data/file'
3
+
4
+ module Evesync
5
+ class Trigger
6
+ class File
7
+ include Trigger::Base
8
+
9
+ def initialize(params)
10
+ @ignore = []
11
+ @db = params[:db]
12
+ @remotes = params[:remotes]
13
+ end
14
+
15
+ def data_class
16
+ IPC::Data::File
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,20 @@
1
+ require 'evesync/trigger/base'
2
+ require 'evesync/ipc/data/package'
3
+
4
+ module Evesync
5
+ class Trigger
6
+ class Package
7
+ include Trigger::Base
8
+
9
+ def initialize(params)
10
+ @ignore = []
11
+ @db = params[:db]
12
+ @remotes = params[:remotes]
13
+ end
14
+
15
+ def data_class
16
+ IPC::Data::Package
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,106 @@
1
+ require 'evesync/trigger/file'
2
+ require 'evesync/trigger/package'
3
+ require 'evesync/config'
4
+ require 'evesync/log'
5
+ require 'evesync/ipc/client'
6
+ require 'evesync/utils'
7
+
8
+ module Evesync
9
+ class Trigger
10
+ def initialize(watcher_queue)
11
+ @watcher_queue = watcher_queue
12
+
13
+ # Local Data daemon
14
+ evedatad = IPC::Client.new(port: :evedatad)
15
+
16
+ @remote_handlers = Config[:evemond]['remotes'].map do |ip|
17
+ new_remote_handler(ip)
18
+ end.compact # remove nils
19
+
20
+ # Helper triggers
21
+ package_trigger = Trigger::Package.new(
22
+ db: evedatad,
23
+ remotes: @remote_handlers
24
+ )
25
+ file_trigger = Trigger::File.new(
26
+ db: evedatad,
27
+ remotes: @remote_handlers
28
+ )
29
+
30
+ @triggers = [package_trigger, file_trigger]
31
+
32
+ Log.debug('Trigger initialization done!')
33
+ end
34
+
35
+ def start
36
+ @thr = Thread.new do
37
+ loop { biz }
38
+ end
39
+ Log.debug('Trigger started')
40
+ end
41
+
42
+ def stop
43
+ @thr.exit
44
+ Log.debug('Trigger stopped')
45
+ end
46
+
47
+ def ignore(change)
48
+ trigger_method(:ignore, change)
49
+ end
50
+
51
+ def unignore(change)
52
+ trigger_method(:unignore, change)
53
+ end
54
+
55
+ def add_remote_node(ip)
56
+ unless @remote_handlers.find { |h| h.ip == ip }
57
+ remote_handler = new_remote_handler(ip)
58
+ @remote_handlers << remote_handler if remote_handler
59
+ end
60
+ Log.debug 'Trigger actual remote nodes:', @remote_handlers.map(&:ip)
61
+ end
62
+
63
+ attr_reader :remote_handlers
64
+
65
+ private
66
+
67
+ # Main thread business logic goes here
68
+ def biz
69
+ change = @watcher_queue.pop
70
+ Log.info "Trigger dequed event: #{change}"
71
+ trigger = message_trigger(change)
72
+ trigger.process(change)
73
+ end
74
+
75
+ # Send a method to target (choose by change class name)
76
+ def trigger_method(method, change)
77
+ Log.debug("Trigger calling '#{method}' on '#{change.class.name}'")
78
+
79
+ trigger = message_trigger(change)
80
+
81
+ if trigger
82
+ trigger.send(method, change) && true
83
+ else
84
+ # TODO: forward somewhere
85
+ Log.error('Trigger: no watchers will be notified on ' \
86
+ "#{change}")
87
+ end
88
+ end
89
+
90
+ def message_trigger(message)
91
+ class_last = message.class.name.to_s.split('::')[-1]
92
+ @triggers.find do |trigger|
93
+ trigger.to_s.include? class_last
94
+ end
95
+ end
96
+
97
+ def new_remote_handler(ip)
98
+ unless Utils.local_ip?(ip)
99
+ IPC::Client.new(
100
+ port: :evehand,
101
+ ip: ip
102
+ )
103
+ end
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,51 @@
1
+ module Evesync
2
+ module Utils
3
+ class << self
4
+ def local_ip?(ip)
5
+ ips = `getent hosts #{ip}`
6
+ .lines
7
+ .map(&:split)
8
+ .map(&:first)
9
+ loc_ips = local_ips
10
+
11
+ !(ips & loc_ips).empty?
12
+ end
13
+
14
+ def local_ip
15
+ local_ips.first
16
+ end
17
+
18
+ def local_ips
19
+ `ip a`
20
+ .lines
21
+ .grep(/inet/)
22
+ .map(&:split)
23
+ .map { |lines| lines[1].split('/')[0] }
24
+ end
25
+
26
+ end
27
+ end
28
+ end
29
+
30
+ # For ruby < 2.1
31
+ class Array
32
+ unless defined? to_h
33
+ def to_h
34
+ Hash[*flatten(1)]
35
+ end
36
+ end
37
+ end
38
+
39
+ class Hash
40
+ unless defined? deep_merge
41
+ def deep_merge(h)
42
+ self.merge(h) do |_k, a, b|
43
+ if a.is_a? Hash
44
+ a.deep_merge(b)
45
+ else
46
+ b
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,156 @@
1
+ require 'date'
2
+ require 'rb-inotify'
3
+ require 'evesync/config'
4
+ require 'evesync/ntp'
5
+ require 'evesync/ipc/data/file'
6
+ require 'evesync/watcher/interface'
7
+
8
+ module Evesync
9
+ class Watcher
10
+
11
+ ##
12
+ # Watches the files and directories, defined in
13
+ # configuration attribute _watch_
14
+ #
15
+ # TODO:
16
+ # * Test on various cases, make it work properly
17
+ # * Find out all possible occasions
18
+
19
+ class File < Watcher::Interface
20
+ def initialize(queue)
21
+ @queue = queue
22
+ @watches = Config[:evemond]['watch']
23
+ @period = Config[:evemond]['watch_interval'].to_i
24
+ @inotify = INotify::Notifier.new
25
+ @events = {}
26
+ @wfiles = []
27
+ @wdirs = []
28
+ initialize_watcher
29
+ end
30
+
31
+ def start
32
+ @inotify_thr = Thread.new { @inotify.run }
33
+ @main_thr = Thread.new do
34
+ loop do
35
+ sleep @period
36
+ send_events
37
+ end
38
+ end
39
+ end
40
+
41
+ def stop
42
+ @inotify.stop
43
+ @inotify_thr.exit
44
+ @main_thr.exit
45
+ end
46
+
47
+ private
48
+
49
+ # Send all events from the methods-handlers
50
+ def send_events
51
+ @events.each do |file, events|
52
+ event = guess_event(events)
53
+ mode = case event
54
+ when :delete then nil
55
+ else ::File::Stat.new(file).mode
56
+ end
57
+
58
+ msg = IPC::Data::File.new(
59
+ name: file,
60
+ mode: mode,
61
+ action: event,
62
+ touched_at: NTP.time.to_s
63
+ )
64
+ @queue.push msg
65
+ Log.debug("Watcher File guessed event #{event} " \
66
+ "from #{events} on #{file}")
67
+ end
68
+ @events = {}
69
+ end
70
+
71
+ def initialize_watcher
72
+ @watches.each do |filename|
73
+ unless ::File.exist? filename
74
+ Log.error("Watcher File: '#{filename}' absent on system")
75
+ end
76
+
77
+ # TODO: ignore /dev and /sys, /proc directories
78
+ if ::File.file? filename
79
+ watch_file filename
80
+ elsif ::File.directory? filename
81
+ watch_directory filename
82
+ else
83
+ # Seems to be a drive or
84
+ Log.warn("Watcher File: watching '#{filename}' is not implemented yet")
85
+ end
86
+ end
87
+ end
88
+
89
+ def watch_file(filename)
90
+ # add file modify watch
91
+ Log.debug("Watcher File watching #{filename}")
92
+
93
+ @inotify.watch(filename, :modify) do |e|
94
+ Log.debug("Watcher File: MODIFIED #{e.absolute_name}")
95
+ h_file(e.absolute_name, [:modify]) # the only flag we need
96
+ end
97
+
98
+ # Waiting for file to disappear and created again
99
+ @wfiles << filename unless @wfiles.include?(filename)
100
+ dirname = ::File.dirname(filename)
101
+ unless @wdirs.include?(dirname)
102
+ watch_directory_for_file(dirname)
103
+ @wdirs << dirname
104
+ end
105
+ end
106
+
107
+ def watch_directory(dirname)
108
+ @inotify.watch(dirname, :create, :delete, :moved_to, :moved_from) do |e|
109
+ Log.debug("Watcher File: DIRECTORY #{e.absolute_name}")
110
+ h_directory(e.absolute_name, e.flags)
111
+ end
112
+ end
113
+
114
+ def watch_directory_for_file(dirname)
115
+ @inotify.watch(dirname, :create, :moved_to) do |e|
116
+ watch_file(e.absolute_name) if @wfiles.include? e.absolute_name
117
+ end
118
+ end
119
+
120
+ # Handlers of inotify changes
121
+
122
+ def h_file(filename, events)
123
+ @events[filename] = [] unless @events[filename]
124
+
125
+ @events[filename] += events
126
+ Log.debug("Watcher File added events for file #{filename}")
127
+ end
128
+
129
+ def h_directory(filename, events)
130
+ if ::File.file? filename
131
+ watch_file(filename)
132
+ @watches << filename unless @watches.include? filename
133
+ elsif ::File.directory? filename
134
+ watch_directory(filename)
135
+ end
136
+
137
+ # Better pass a file
138
+ @events[filename] = [] unless @events[filename]
139
+
140
+ @events[filename] += events
141
+ Log.debug("Watcher File added events for directory #{filename}")
142
+ end
143
+
144
+ def guess_event(events)
145
+ return events.first if events.length == 1
146
+
147
+ return :modify if events.include?(:create) && events.include?(:modify)
148
+
149
+ return :delete if events.include?(:delete) && (!events.include? :create)
150
+
151
+ # TODO: find out more logic
152
+ events.last
153
+ end
154
+ end
155
+ end
156
+ end
@@ -0,0 +1,21 @@
1
+ module Evesync
2
+ class Watcher
3
+ # Base watcher abstract class with methods for other
4
+ # watchers to implement
5
+ class Interface
6
+ # The class must be initialized with the queue object
7
+ def initialize(_queue)
8
+ raise NotImplementedError, "must implement 'initialize'"
9
+ end
10
+
11
+ # The watcher must be able to handle start and stop calls
12
+ def start
13
+ raise NotImplementedError, "must implement 'start'"
14
+ end
15
+
16
+ def stop
17
+ raise NotImplementedError, "must implement 'stop'"
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,8 @@
1
+ require 'evesync/os/linux'
2
+
3
+ module Evesync
4
+ class Watcher
5
+ # Package class is a reference to Distro::PackageWatcher
6
+ Package = Evesync::OS::PackageWatcher
7
+ end
8
+ end