ruby-livesync 1.0.0.beta2 → 1.0.0.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d19adaa07cc52746ac91f99dc954b5b85c3d1150b41ffb4e789c453efe404ae
-  data.tar.gz: bf4db87f65060754f52063a41dfc27094027fd8165e4b722f051e6af069dd6cf
+  metadata.gz: 1e77356b7dfd5070935875d4842a16129c5d1eeaaec1e049d2b4e4824032a2c0
+  data.tar.gz: f8350e318d1f3e31dc5925981c1526b74d404795c4300419a2db489ff38e0f2a
 SHA512:
-  metadata.gz: 4caca9a0bb7a879d9133d0aadf74768f2252866f6b5729eb45935ee5d975bb7029a77a40d2e9ada50026ee196bf2ca1a54e5aa4169ca0e87892670a8dc818587
-  data.tar.gz: 50b8a7101b60a5f4b29906f38b92a1cfc1671ebac8cd7aba1fe78ae879cda7d4f2a441ec248e68a7201959f356e12dc0e5583c8cee6e9ea234ba8290b0385e7a
+  metadata.gz: 9989027e61b51d31e4248e71c7d24e3d1a8c9cfaf74ab532cd99fcc6f25ae7d9e23c12790cc31a57e306395213c846e132e3cdc24c6d14d548f74d525860515d
+  data.tar.gz: 43ac4cf5d5ff1cc2926bb13ba60f4efbafa07fd54d940d8bdb55cbd183b6a608c6d644a4a7173c2ebdf1d490534eedd37e1940ef9854b98b16201d3461be5009

data/config/sample.rb

@@ -3,15 +3,27 @@
 sync '4tb' do
   enabled = false
 
+  # watchers available:
+  # - :rb (default)
+  # - :py_inotify
+  # - :cmd (inotifywait)
+  # - :py_watchdog
+  #
+  watcher = :rb
+
   # fork to user below, usually associated with private keys 
   user = :root
 
   delay = 5
 
-  source = '/mnt/4tb/'
-  target = 'root@bhavapower:/mnt/extensor/4tb'
+  # event list from inotify
+  # full list at https://manpages.ubuntu.com/manpages/latest/en/man1/inotifywait.1.html#events
+  modes = %i[create modify]
 
-  rsync.opts = '-ax --partial' # default
+  source = '/mnt/4tb/'
+  target rsync: 'root@bhavapower:/mnt/extensor/4tb' do
+    opts = '-ax --partial' # default
+  end
 
   # possible values are: true, false, :initial, :watched
   delete = true

data/lib/ruby-livesync/live_sync/cmd_watcher.rb

@@ -0,0 +1,56 @@
+module LiveSync
+  class CmdWatcher < Watcher
+
+    class_attribute :base_cmd
+    self.base_cmd = 'bash -s'
+
+    class_attribute :script
+    self.script = <<-HEREDOC
+      tp=$(mktemp -u)
+      mkfifo $tp
+      inotifywait %{opts} -m -e %{events} --format "%%e %%w%%f" %{path} %{excludes} > $tp 2>/dev/null &
+      ipid=$!
+      trap "kill $ipid; rm -f $tp; exit" INT TERM EXIT
+      while true; do
+        timeout %{delay} cat < $tp | sort | uniq
+      done
+    HEREDOC
+
+    def watch path, *modes, **params
+      modes  = DEFAULT_MODES if modes.blank?
+      script = self.script % parsed_params(path, *modes, **params)
+      stdin, stdout, stderr, @wait_thr = Open3.popen3 base_cmd
+      stdin.write script
+      stdin.close
+
+      Thread.new do
+        stdout.sync = true
+        lines = stdout.each_line.each do |line|
+          file,events = parse line
+          yield [OpenStruct.new(absolute_name: file, flags: events)]
+        end
+      end
+      Thread.new do
+        STDERR.puts stderr.read
+      end
+    end
+
+    def parsed_params path, *modes, recursive: true, delay: 1, excludes: [], **params
+      opts  = '-r' if recursive
+      {
+        path:      path,
+        opts:      opts,
+        events:    modes.join(','),
+        delay:     delay,
+        excludes:  excludes.map{ |e| "'@#{e}'" }.join(' '),
+      }
+    end
+
+    def parse line
+      events,file = line.split ' ', 2
+      events = events.split(',').map(&:downcase).map(&:to_sym)
+      [file.chomp, events]
+    end
+
+  end
+end

data/lib/ruby-livesync/live_sync/daemon.rb

@@ -22,8 +22,7 @@ module LiveSync
     end
 
     def sync name_or_path, &block
-      s = Sync.new name_or_path
-      s.dsl_apply(&block)
+      s = Sync.new name_or_path, &block
       @syncs[s.user] << s
     end
 

data/lib/ruby-livesync/live_sync/dsl.rb

@@ -10,24 +10,27 @@ module LiveSync
       class_attribute :attrs
       self.attrs = []
 
-      def self.dsl attr, default: nil, enum: nil, type: nil, &block
+      def self.dsl attr, default: nil, enum: nil,
+        type: nil, skip_set: false, &block
         self.attrs << attr
-        define_method attr do |sv=nil|
+
+        define_method attr do |sv=nil, opts={}, &ablock|
+          sv = default if opts[:init]
           (v = instance_variable_get("@#{attr}"); return(if v.nil? then default else v end)) if sv.nil?
 
-          raise "#{ctx}/#{attr}: incorrect type" if type and !sv.is_a? type
+          raise "#{ctx}/#{attr}: incorrect type of #{sv.inspect}" if type and !sv.is_a? type
           raise "#{ctx}/#{attr}: value not one of following #{enum}" if enum and !sv.in? enum
 
-          instance_variable_set "@#{attr}", sv
-          instance_exec sv, &block if block
+          instance_variable_set "@#{attr}", sv unless skip_set
+          instance_exec sv, ablock, &block if block
         end
       end
 
       def dsl_apply &block
         if b = binding and bs = block.source.match(/do(.+)end$/m)&.captures&.first
           b.eval bs
-          attrs.each do |a| # read local variables
-            next unless a.in? b.local_variables
+          attrs.each do |a|
+            next send a, nil, {init: true} unless a.in? b.local_variables
             send a, b.local_variable_get(a)
           end
         else

data/lib/ruby-livesync/live_sync/py/inotify.py

@@ -0,0 +1,45 @@
+import pyinotify
+import os
+import time
+
+class EventHandler(pyinotify.ProcessEvent):
+    def my_init(self, excludes, recursive, mask):
+        self.excludes = excludes
+        self.recursive = recursive
+        self.mask = mask
+        self.events = set()
+
+    def process_default(self, event):
+        full_path = event.pathname
+        if not any(exclude in full_path for exclude in self.excludes):
+            event_desc = event.maskname
+            self.events.add((event_desc, full_path))
+
+    def print_events(self):
+        if self.events:
+            for event_desc, full_path in self.events:
+                print(f"{event_desc} {full_path}", flush=True)
+            self.events.clear()
+
+    def add_watch(self, path):
+        wm.add_watch(path, self.mask, rec=self.recursive, auto_add=True)
+
+excludes = [%{excludes}]
+recursive = %{recursive}
+event_mask = pyinotify.IN_CREATE | pyinotify.IN_MODIFY | pyinotify.IN_MOVED_TO | pyinotify.IN_DELETE  # Event mask
+
+wm = pyinotify.WatchManager()
+handler = EventHandler(excludes=excludes, recursive=recursive, mask=event_mask)
+notifier = pyinotify.Notifier(wm, handler)
+handler.add_watch('%{path}')
+
+try:
+    while True:
+        notifier.process_events()
+        if notifier.check_events():
+            notifier.read_events()
+        handler.print_events()
+        time.sleep(%{delay})
+except KeyboardInterrupt:
+    notifier.stop()
+

data/lib/ruby-livesync/live_sync/py/watchdog.py

@@ -0,0 +1,52 @@
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+import time
+
+class MyEventHandler(FileSystemEventHandler):
+    def __init__(self, observer, excludes):
+        self.observer = observer
+        self.excludes = excludes
+        self.events = set()
+
+    def handle_event(self, event):
+        if not any(exclusion in event.src_path for exclusion in self.excludes):
+            self.events.add((event.event_type, event.src_path))
+            if event.event_type == 'created' and os.path.isdir(event.src_path):
+                self.schedule_directory(event.src_path)
+
+    def on_modified(self, event):
+        self.handle_event(event)
+
+    def on_created(self, event):
+        self.handle_event(event)
+
+    def on_deleted(self, event):
+        self.handle_event(event)
+
+    def on_moved(self, event):
+        self.handle_event(event)
+
+    def print_events(self):
+        if self.events:
+            for event_type, src_path in self.events:
+                print(f"{event_type} {src_path}", flush=True)
+            self.events.clear()
+
+    def schedule_directory(self, path):
+        self.observer.schedule(self, path, recursive=%{recursive})
+
+path = "%{path}"
+excludes = [%{excludes}]
+observer = Observer()
+event_handler = MyEventHandler(observer, excludes)
+event_handler.schedule_directory(path)
+
+observer.start()
+try:
+    while True:
+        time.sleep(%{delay})
+        event_handler.print_events()
+except KeyboardInterrupt:
+    observer.stop()
+observer.join()
+

data/lib/ruby-livesync/live_sync/py_inotify_watcher.rb

@@ -0,0 +1,19 @@
+module LiveSync
+  class PyInotifyWatcher < CmdWatcher
+
+    self.base_cmd = 'python'
+
+    self.script = File.read "#{File.dirname __FILE__}/py/inotify.py"
+
+    def parsed_params path, *modes, recursive: true, delay: 1, excludes: [], **params
+      {
+        path:      path,
+        recursive: if recursive then 'True' else 'False' end,
+        events:    modes.join(','),
+        delay:     delay,
+        excludes:  excludes.map{ |e| "'#{e}'" }.join(','),
+      }
+    end
+
+  end
+end

data/lib/ruby-livesync/live_sync/py_watchdog_watcher.rb

@@ -0,0 +1,19 @@
+module LiveSync
+  class PyWatchdogWatcher < CmdWatcher
+
+    self.base_cmd = 'python'
+
+    self.script = File.read "#{File.dirname __FILE__}/py/watchdog.py"
+
+    def parsed_params path, *modes, recursive: true, delay: 1, excludes: [], **params
+      {
+        path:      path,
+        recursive: if recursive then 'True' else 'False' end,
+        events:    modes.join(','),
+        delay:     delay,
+        excludes:  excludes.map{ |e| "'#{e}'" }.join(','),
+      }
+    end
+
+  end
+end

data/lib/ruby-livesync/live_sync/rb_watcher.rb

@@ -0,0 +1,88 @@
+module LiveSync
+  class RbWatcher < Watcher
+
+    def watch path, *modes, excludes: [], samefs: true, delay: 1, recursive: true, **params, &block
+      raise "#{path}: not a directory" unless File.directory? path
+      modes  = DEFAULT_MODES if modes.blank?
+      modes << :delete if sync&.delete&.in? [true, :watched]
+
+      tracker = Tracker.new path, delay, &block
+      rtgts = glob path, recursive: recursive, excludes: excludes
+      track(path, rtgts, *modes,
+        delay:     1,
+        tracker:   tracker,
+        recursive: recursive,
+        excludes:  excludes,
+        samefs:    samefs,
+      )
+      tracker.timeout_check
+
+      self
+    end
+
+    protected
+
+    class Tracker
+      def initialize path, delay, &block
+        @delay    = delay
+        @block    = block
+        @to_sync  = Set.new
+        @notifier = INotify::Notifier.new
+        @pathname = Pathname.new path
+      end
+
+      def watch *args, &block
+        @notifier.watch *args, &block
+      end
+
+      def track event
+        @to_sync << Pathname.new(event.absolute_name).relative_path_from(@pathname).to_s
+      end
+
+      def check
+        @notifier.process # calls track
+        @block.call @to_sync
+        @to_sync.clear
+      ensure
+        timeout_check
+      end
+
+      def timeout_check
+        Thread.new{ sleep @delay; check }
+      end
+    end
+
+    def track path, rtgts, *modes, tracker:, **opts
+      rtgts.each do |rt|
+        t = "#{path}/#{rt}"
+        tracker.watch t, *modes do |event|
+          tracker.track event
+          et = event.absolute_name
+          next unless opts[:recursive] and File.directory?(et) and :create.in? event.flags
+          track et, glob(et, **opts), *modes, tracker: tracker, **opts
+        end
+      rescue => e
+        Log.warning "watcher: #{t}: skipping due to #{e.class}: #{e.message}"
+      end
+    end
+
+    def glob path, recursive: true, samefs: true, excludes: [], **params
+      wcard = if recursive then '{.**,**}/' else '' end
+      tgts  = [path] + Dir.glob("#{path}/#{wcard}")
+
+      dev   = File.stat(path).dev
+      tgts.select!{ |t| dev == File.stat(t).dev } if samefs
+
+      rtgts = tgts.map{ |t| Pathname.new(t).relative_path_from(path).to_s }
+      excs  = excludes.map{ |e| Regexp.new e } || []
+      excs.each do |e|
+        next unless mt = rtgts.find{ |rt| e.match rt }
+        Log.debug "watcher: skipping #{path}/#{mt} with subdirs"
+        rtgts.delete mt
+        rtgts.delete_if{ |rt| rt.start_with? mt } if File.directory? "#{path}/#{mt}"
+      end
+      rtgts
+    end
+
+  end
+end

data/lib/ruby-livesync/live_sync/rsync.rb

@@ -1,12 +1,20 @@
 module LiveSync
-  class Rsync
+  class Rsync < Target
 
-    attr_reader :sync
-    attr_accessor :opts
+    dsl :opts, type: String
 
-    def initialize sync
-      @sync = sync
-      @opts = '-ax --partial'
+    attr_reader :ssh
+
+    def initialize *args, &block
+      super
+      # add trailing slash in case the dir is the same
+      sync.source File.join(sync.source, '') if File.basename(sync.source) == File.basename(@path)
+    end
+
+    def start
+      @ssh  = Ssh.connect userhost
+      sleep 1 and log.warning 'waiting for ssh' while !@ssh.available?
+      true
     end
 
     def running?
@@ -31,21 +39,21 @@ module LiveSync
     protected
 
     def run type, *args, loglevel: :debug
-      cmd = "rsync -e '#{rsh}' #{opts} #{sync.source} #{sync.target} #{args.join ' '}"
+      cmd = "rsync -e '#{rsh}' #{opts} #{sync.source} #{dest} #{args.join ' '}"
       sync.excludes.each{ |e| cmd << " --exclude='#{e}'" }
 
-      sync.log.send loglevel, "#{type}: starting with cmd: #{cmd}"
+      log.send loglevel, "#{type}: starting with cmd: #{cmd}"
       stdin, stdout, stderr, @wait_thr = Open3.popen3 cmd
       yield stdin, stdout, stderr if block_given?
       Thread.new do
         Process.wait @wait_thr.pid rescue Errno::ECHILD; nil
         @wait_thr = nil
-        sync.log.send loglevel, "#{type}: finished"
+        log.send loglevel, "#{type}: finished"
       end
     end
 
     def rsh
-      "ssh -o ControlPath=#{sync.ssh.cpath}"
+      "ssh -o ControlPath=#{ssh.cpath}"
     end
 
   end

data/lib/ruby-livesync/live_sync/sync.rb

@@ -4,17 +4,14 @@ module LiveSync
     include DSL
 
     attr_reader :name
-    attr_reader :scheduler
     attr_reader :log
     attr_reader :watcher
-    attr_reader :ssh
-    attr_reader :rsync
+    attr_reader :target
 
-    def initialize name = nil
+    def initialize name = nil, &block
       fill_name name
       source name if File.exist? name
-      @to_sync = Set.new
-      @rsync   = Rsync.new self
+      dsl_apply(&block)
     end
 
     def fill_name name
@@ -25,21 +22,26 @@ module LiveSync
 
     dsl :enabled, default: true
 
+    dsl :watcher, skip_set: true, default: :rb do |name|
+      klass = :"#{name.to_s.camelize}Watcher"
+      @watcher_class = LiveSync.const_get klass
+    end
+
     dsl :user, default: :root
     dsl :source do |source|
-      raise "#{ctx}: source not found" unless File.exist? source
+      raise "#{ctx}: source isn't a directory" unless File.directory? source
       fill_name source
       @pathname = Pathname.new source
     end
 
-    dsl :target do |target|
-      @userhost, @target_path = target.split(':')
-      raise "#{ctx}: missing target path" unless @target_path
-      source File.join(@source, '') if File.basename(@source) == File.basename(@target_path)
+    dsl :target, skip_set: true do |opts, &block|
+      @target = Rsync.new self, opts[:rsync], &block if opts[:rsync]
     end
 
     dsl :delay, default: 5, type: Integer
 
+    dsl :modes, default: %i[create modify], type: Array
+
     dsl :delete, default: false, enum: [true,false] + %i[initial watched]
 
     dsl :excludes, default: []
@@ -47,40 +49,27 @@ module LiveSync
     def start
       return log.warning('skipping disabled sync') && false unless enabled
       raise "#{ctx}: missing target" unless @target
-      @ssh = Ssh.connect @userhost
-      sleep 1 and log.warning 'waiting for ssh' while !@ssh.available?
-      true
+      target.start
     end
 
     def guard
       fork do
         Process.setproctitle "livesync: sync #{ctx}"
-        @watcher   = Watcher.new self
-        @scheduler = Rufus::Scheduler.new
+        @watcher = @watcher_class.new self
 
-        @watcher.dir_rwatch source, &method(:track)
-        @rsync.initial
-        schedule
+        watch source
+        target.initial
         sleep 1.day while true
       end
     end
 
-    def track event
-      @to_sync << Pathname.new(event.absolute_name).relative_path_from(@pathname).to_s
-    end
-
-    def schedule
-      @scheduler.in "#{delay}s", &method(:check)
+    def watch dir
+      @watcher.watch dir, *modes, excludes: excludes, delay: delay, &method(:sync)
     end
 
-    def check
-      return if @rsync.running?
-      @watcher.process # calls #track
-      return if @to_sync.blank?
-      @rsync.partial @to_sync
-      @to_sync.clear
-    ensure
-      schedule
+    def sync paths
+      return if target.running?
+      target.partial paths
     end
 
     protected

data/lib/ruby-livesync/live_sync/target.rb

@@ -0,0 +1,36 @@
+module LiveSync
+  class Target
+
+    include DSL
+
+    attr_reader :sync
+    delegate :log, to: :sync
+
+    attr_reader :dest, :path, :userhost
+
+    def initialize sync, dest, &block
+      @sync = sync
+      @dest = dest
+      @userhost, @path = dest.split ':'
+      raise "#{sync.ctx}: missing target path" unless @path
+      dsl_apply &block if block
+    end
+
+    def start
+      false
+    end
+
+    def running?
+      false
+    end
+
+    def initial
+      raise 'not implemented'
+    end
+
+    def partial paths
+      raise 'not implemented'
+    end
+
+  end
+end

data/lib/ruby-livesync/live_sync/version.rb

@@ -1,5 +1,5 @@
 module LiveSync
 
-  VERSION = '1.0.0.beta2'
+  VERSION = '1.0.0.beta4'
 
 end

data/lib/ruby-livesync/live_sync/watcher.rb

@@ -1,50 +1,15 @@
 module LiveSync
   class Watcher
 
-    attr_reader :notifier
-    delegate_missing_to :notifier
+    DEFAULT_MODES = %i[create modify]
 
     attr_reader :sync
 
     def initialize sync=nil
-      @sync     = sync
-      @notifier = INotify::Notifier.new
+      @sync = sync
     end
 
-    def watch path, *modes
-      Log.debug "#{path}: watching for #{modes.join ','}"
-      modes  = %i[all_events] if modes.blank?
-
-      notifier.watch path, *modes do |event|
-        yield event
-      end
-      self
-    end
-
-    def dir_rwatch path, *modes
-      raise "#{path}: not a directory" unless File.directory? path
-      modes  = %i[create modify] if modes.blank?
-      modes << :delete if sync&.delete&.in? [true, :watched]
-
-      excs  = sync&.excludes.map{ |e| Regexp.new e } || []
-      tgts  = [path] + Dir.glob("#{path}/{.**,**}/")
-      rtgts = tgts.map{ |t| Pathname.new(t).relative_path_from(path).to_s }
-      excs.each do |e|
-        next unless mt = rtgts.find{ |rt| e.match rt }
-        Log.debug "watcher: skipping #{path}/#{mt} with subdirs"
-        rtgts.delete mt
-        rtgts.delete_if{ |rt| rt.start_with? mt } if File.directory? "#{path}/#{mt}"
-      end
-
-      rtgts.each do |rt|
-        t = "#{path}/#{rt}"
-        notifier.watch t, *modes do |event|
-          yield event
-        end
-      rescue => e
-        Log.warning "watcher: #{t}: skipping due to #{e.class}: #{e.message}"
-      end
-      self
+    def watch path, *modes, delay: 1, excludes: [], &block
     end
 
   end

data/lib/ruby-livesync/live_sync.rb

@@ -7,8 +7,13 @@ require_relative 'live_sync/dsl'
 require_relative 'live_sync/log'
 require_relative 'live_sync/user'
 require_relative 'live_sync/ssh'
+require_relative 'live_sync/target'
 require_relative 'live_sync/rsync'
 require_relative 'live_sync/watcher'
+require_relative 'live_sync/rb_watcher'
+require_relative 'live_sync/cmd_watcher'
+require_relative 'live_sync/py_inotify_watcher'
+require_relative 'live_sync/py_watchdog_watcher'
 require_relative 'live_sync/sync'
 require_relative 'live_sync/daemon'
 

data/lib/ruby-livesync.rb

@@ -2,7 +2,6 @@ require 'pry'
 require 'active_support/all'
 
 require 'rb-inotify'
-require 'rufus-scheduler'
 
 require_relative 'ruby-livesync/live_sync'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-livesync
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta2
+  version: 1.0.0.beta4
 platform: ruby
 authors:
 - Braulio Oliveira
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-13 00:00:00.000000000 Z
+date: 2024-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -52,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rufus-scheduler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description:
 email:
 - brauliobo@gmail.com
@@ -78,12 +64,19 @@ files:
 - config/sample.rb
 - lib/ruby-livesync.rb
 - lib/ruby-livesync/live_sync.rb
+- lib/ruby-livesync/live_sync/cmd_watcher.rb
 - lib/ruby-livesync/live_sync/daemon.rb
 - lib/ruby-livesync/live_sync/dsl.rb
 - lib/ruby-livesync/live_sync/log.rb
+- lib/ruby-livesync/live_sync/py/inotify.py
+- lib/ruby-livesync/live_sync/py/watchdog.py
+- lib/ruby-livesync/live_sync/py_inotify_watcher.rb
+- lib/ruby-livesync/live_sync/py_watchdog_watcher.rb
+- lib/ruby-livesync/live_sync/rb_watcher.rb
 - lib/ruby-livesync/live_sync/rsync.rb
 - lib/ruby-livesync/live_sync/ssh.rb
 - lib/ruby-livesync/live_sync/sync.rb
+- lib/ruby-livesync/live_sync/target.rb
 - lib/ruby-livesync/live_sync/user.rb
 - lib/ruby-livesync/live_sync/version.rb
 - lib/ruby-livesync/live_sync/watcher.rb
@@ -103,11 +96,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.3.25
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Lightweight and fast live sync daemon