entangler 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 490b5a2f8672548a4e3a7d15fbd641291f9456fc
-  data.tar.gz: 5750ebccf9775aa3971dc7dfe930527ee7803af4
+  metadata.gz: e910a8be4617a7df89c5a0d5f94b51617a26deb6
+  data.tar.gz: 1d083cf56c15e7f332c703eb36381d31e717b7f6
 SHA512:
-  metadata.gz: 6b74a61f4a3dda5e52b8b10fe8f41e92eb9e9266dfa923bfc015b857f68e24757770c620ee6f937da0fc18e2885c2a0a723a2a329febf9871574ab9128dd223e
-  data.tar.gz: 4df993ced868041b7f96012b425bf2b0a1d569a1d16074a852c63bf4e9d1b96f19b2ffd1ff362af09f00888bb7a8ca55ce9471848db37d19f76b63b56aa4c642
+  metadata.gz: 829fd8b9cc788a049e1daab942a3eff9d911351f580b71f276610c0c48b758638698868066ceb39e3bac63a6bebc7e295915d19fd95d7f93e4a66a0bfe057b65
+  data.tar.gz: b08f880f5e249658f83b906e3789adf84b999d9922b068dc6eec037aa069475a62434212287f3a3898f56661a1a01c43911eab4c9b70fd6c59a12f153edd92ef

data/lib/entangler/entangled_file.rb

@@ -28,6 +28,23 @@ module Entangler
       File.exists?(full_path)
     end
 
+    def export
+      raise "Delta file doesn't exist when creaing patched file" unless delta_exists?
+      tempfile = Tempfile.new('final_file')
+      if File.exists?(full_path)
+        LibRubyDiff.patch(full_path, delta_file.path, tempfile.path)
+      else
+        temp_empty_file = Tempfile.new('empty_file')
+        LibRubyDiff.patch(temp_empty_file.path, delta_file.path, tempfile.path)
+      end
+      tempfile.rewind
+      File.open(full_path, 'w'){|f| f.write(tempfile.read)}
+      tempfile.close
+      tempfile.unlink
+      File.utime(File.atime(full_path), @desired_modtime, full_path)
+    end
+
+    private
     def signature_exists?
       defined?(@signature_tempfile)
     end
@@ -79,22 +96,6 @@ module Entangler
       delta_file.read
     end
 
-    def export
-      raise "Delta file doesn't exist when creaing patched file" unless delta_exists?
-      tempfile = Tempfile.new('final_file')
-      if File.exists?(full_path)
-        LibRubyDiff.patch(full_path, delta_file.path, tempfile.path)
-      else
-        temp_empty_file = Tempfile.new('empty_file')
-        LibRubyDiff.patch(temp_empty_file.path, delta_file.path, tempfile.path)
-      end
-      tempfile.rewind
-      File.open(full_path, 'w'){|f| f.write(tempfile.read)}
-      tempfile.close
-      tempfile.unlink
-      File.utime(File.atime(full_path), @desired_modtime, full_path)
-    end
-
     def close_and_unlink_files
       if signature_exists?
         @signature_tempfile.close

data/lib/entangler/executor/background/base.rb

@@ -0,0 +1,108 @@
+module Entangler
+  module Executor
+    module Background
+      module Base
+        protected
+        def wait_for_threads
+          @consumer_thread.join
+          @remote_io_thread.join
+          @local_io_thread.join
+          Process.wait @notify_daemon_pid
+        end
+
+        def kill_off_threads
+          Process.kill("TERM", @notify_daemon_pid) rescue nil
+          @consumer_thread.terminate
+          @remote_io_thread.terminate
+          @local_io_thread.terminate
+        end
+
+        def start_notify_daemon
+          logger.info('starting notify daemon')
+          r,w = IO.pipe
+          @notify_daemon_pid = spawn(start_notify_daemon_cmd, out: w)
+          w.close
+          @notify_reader = r
+        end
+
+        def start_remote_io
+          logger.info('starting remote IO')
+          @remote_io_thread = Thread.new do
+            begin
+              loop do
+                msg = Marshal.load(@remote_reader)
+                next if msg.nil?
+
+                case msg[:type]
+                when :new_changes
+                  process_new_changes(msg[:content])
+                when :entangled_files
+                  process_entangled_files(msg[:content])
+                end
+              end
+            rescue => e
+              $stderr.puts e.message
+              $stderr.puts e.backtrace.join("\n")
+              kill_off_threads
+            end
+          end
+        end
+
+        def start_local_io
+          logger.info('starting local IO')
+          @local_action_queue = Queue.new
+          @local_io_thread = Thread.new do
+            begin
+              msg = []
+              loop do
+                ready = IO.select([@notify_reader]).first
+                next unless ready && ready.any?
+                break if ready.first.eof?
+                line = ready.first.gets
+                next if line.nil? || line.empty?
+                line = line.strip
+                next if line == '-'
+                @local_action_queue.push line
+              end
+            rescue => e
+              $stderr.puts e.message
+              $stderr.puts e.backtrace.join("\n")
+              kill_off_threads
+            end
+          end
+        end
+
+        def start_local_consumer
+          @consumer_thread = Thread.new do
+            loop do
+              msg = [@local_action_queue.pop]
+              loop do
+                sleep 0.2
+                break if @local_action_queue.empty?
+                while !@local_action_queue.empty?
+                  msg << @local_action_queue.pop
+                end
+              end
+              while Time.now.to_f <= @notify_sleep
+                sleep 0.5
+                while !@local_action_queue.empty?
+                  msg << @local_action_queue.pop
+                end
+              end
+              process_lines(msg.uniq)
+              msg = []
+              sleep 0.5
+            end
+          end
+        end
+
+        def start_notify_daemon_cmd
+          uname = `uname`.strip.downcase
+          raise 'Unsupported OS' unless ['darwin', 'linux'].include?(uname)
+
+          "#{File.join(File.dirname(File.dirname(File.dirname(File.dirname(__FILE__)))), 'notifier', 'bin', uname, 'notify')} #{self.base_dir}"
+        end
+      end
+    end
+  end
+end

data/lib/entangler/executor/background/master.rb

@@ -0,0 +1,24 @@
+module Entangler
+  module Executor
+    module Background
+      module Master
+        protected
+        def start_remote_slave
+          require 'open3'
+          @remote_writer, @remote_reader, remote_err, @remote_thread = Open3.popen3("ssh -q #{@opts[:remote_user]}@#{@opts[:remote_host]} -p #{@opts[:remote_port]} -C \"source ~/.rvm/environments/default && entangler slave #{@opts[:remote_base_dir]}\"")
+          remote_err.close
+        end
+
+        def wait_for_threads
+          super
+          Process.wait @remote_thread[:pid] rescue nil
+        end
+
+        def kill_off_threads
+          Process.kill("INT", @remote_thread[:pid])
+          super
+        end
+      end
+    end
+  end
+end

data/lib/entangler/executor/base.rb

@@ -1,23 +1,32 @@
 require 'logger'
 require 'fileutils'
 require 'thread'
+require_relative 'background/base'
+require_relative 'processing/base'
 
 module Entangler
   module Executor
     class Base
+      include Entangler::Executor::Background::Base, Entangler::Executor::Processing::Base
+
       attr_reader :base_dir
 
       def initialize(base_dir, opts = {})
         @base_dir = File.realpath(File.expand_path(base_dir))
         @notify_sleep = 0
+        @exported_at = 0
         @opts = opts
         @opts[:ignore] = [/^\/\.git.*/, /^\/\.entangler.*/, /^\/\.idea.*/, /^\/log.*/, /^\/tmp.*/] unless @opts.has_key?(:ignore)
         validate_opts
         logger.info("Starting executor")
       end
 
-      def validate_opts
-        raise "Base directory doesn't exist" unless Dir.exists?(self.base_dir)
+      def generate_abs_path(rel_path)
+        File.join(self.base_dir, rel_path)
+      end
+
+      def strip_base_path(path, base_dir = self.base_dir)
+        File.expand_path(path).sub(base_dir, '')
       end
 
       def run
@@ -27,214 +36,18 @@ module Entangler
         start_local_consumer
         logger.debug("NOTIFY PID: #{@notify_daemon_pid}")
         Signal.trap("INT") { kill_off_threads }
-        @consumer_thread.join
-        @remote_io_thread.join
-        @local_io_thread.join
-        Process.wait @notify_daemon_pid
-      end
-
-      def kill_off_threads
-        Process.kill("TERM", @notify_daemon_pid) rescue nil
-        @consumer_thread.terminate
-        @remote_io_thread.terminate
-        @local_io_thread.terminate
-      end
-
-      def start_file_transfers(paths, pipe)
-        Marshal.dump(paths.map{|path| EntangledFile.new(path) }, pipe)
+        wait_for_threads
       end
 
-      def start_notify_daemon
-        logger.info('starting notify daemon')
-        r,w = IO.pipe
-        @notify_daemon_pid = spawn(start_notify_daemon_cmd, out: w)
-        w.close
-        @notify_reader = r
-      end
-
-      def start_local_io
-        logger.info('starting local IO')
-        @local_action_queue = Queue.new
-        @local_io_thread = Thread.new do
-          begin
-            msg = []
-            loop do
-              ready = IO.select([@notify_reader]).first
-              next unless ready && ready.any?
-              break if ready.first.eof?
-              line = ready.first.gets
-              next if line.nil? || line.empty?
-              line = line.strip
-              next if line == '-'
-              @local_action_queue.push line
-            end
-          rescue => e
-            $stderr.puts e.message
-            $stderr.puts e.backtrace.join("\n")
-            kill_off_threads
-          end
-        end
-      end
-
-      def start_local_consumer
-        @consumer_thread = Thread.new do
-          loop do
-            msg = [@local_action_queue.pop]
-            while !@local_action_queue.empty?
-              msg << @local_action_queue.pop
-            end
-            while Time.now.to_i <= @notify_sleep
-              sleep 1
-              while !@local_action_queue.empty?
-                msg << @local_action_queue.pop
-              end
-            end
-            process_lines(msg.uniq)
-            msg = []
-            sleep 1
-          end
-        end
+      protected
+      def validate_opts
+        raise "Base directory doesn't exist" unless Dir.exists?(self.base_dir)
       end
 
       def send_to_remote(msg = {})
         Marshal.dump(msg, @remote_writer)
       end
 
-      def start_remote_io
-        logger.info('starting remote IO')
-        @remote_io_thread = Thread.new do
-          begin
-            loop do
-              msg = Marshal.load(@remote_reader)
-              next if msg.nil?
-
-              case msg[:type]
-              when :new_changes
-                logger.debug("Got #{msg[:content].length} new folder changes from remote")
-
-                created_dirs = []
-                dirs_to_remove = []
-                files_to_remove = []
-                files_to_update = []
-
-                msg[:content].each do |base, changes|
-                  possible_creation_dirs = changes[:dirs].clone
-                  possible_creation_files = changes[:files].keys.clone
-                  full_base_path = generate_abs_path(base)
-
-                  unless File.directory?(full_base_path)
-                    FileUtils::mkdir_p(full_base_path)
-                    @notify_sleep = Time.now.to_i + 60
-                  end
-
-                  Dir.entries(full_base_path).each do |f|
-                    next if ['.', '..'].include? f
-                    full_path = File.join(generate_abs_path(base), f)
-                    if File.directory?(full_path)
-                      possible_creation_dirs -= [f]
-                      dirs_to_remove << full_path unless changes[:dirs].include?(f)
-                    elsif changes[:files].has_key?(f)
-                      possible_creation_files -= [f]
-                      files_to_update << File.join(base, f) unless changes[:files][f] == [File.size(full_path), File.mtime(full_path).to_i]
-                    else
-                      files_to_remove << full_path
-                    end
-                  end
-
-                  dirs_to_create = possible_creation_dirs.map{|d| File.join(generate_abs_path(base), d)}
-                  if dirs_to_create.any?
-                    logger.debug("Creating #{dirs_to_create.length} dirs")
-                    @notify_sleep = Time.now.to_i + 60
-                    FileUtils.mkdir_p dirs_to_create
-                  end
-                  created_dirs += dirs_to_create
-                  files_to_update += possible_creation_files.map{|f| File.join(base, f)}
-                end
-
-                @notify_sleep = Time.now.to_i + 60 if (files_to_remove + created_dirs + dirs_to_remove + files_to_update).any?
-
-                if files_to_remove.any?
-                  logger.debug("Deleting #{files_to_remove.length} files")
-                  FileUtils.rm files_to_remove
-                end
-                if dirs_to_remove.any?
-                  logger.debug("Deleting #{dirs_to_remove.length} dirs")
-                  FileUtils.rm_r dirs_to_remove
-                end
-                if files_to_update.any?
-                  logger.debug("Creating #{files_to_update.length} new entangled files to sync")
-                  send_to_remote(type: :entangled_files, content: files_to_update.map{|f| Entangler::EntangledFile.new(f) })
-                end
-                @notify_sleep = Time.now.to_i + 1 if (files_to_remove + created_dirs + dirs_to_remove + files_to_update).any?
-                @notify_sleep += 60 if files_to_update.any?
-              when :entangled_files
-                logger.debug("Got #{msg[:content].length} entangled files from remote")
-                completed_files, updated_files = msg[:content].partition(&:done?)
-
-                completed_files.each(&:export)
-
-                updated_files = updated_files.find_all{|f| f.state != 1 || f.file_exists? }
-                if updated_files.any?
-                  send_to_remote(type: :entangled_files, content: updated_files)
-                end
-                @notify_sleep = Time.now.to_i + 1 if completed_files.any?
-              end
-            end
-          rescue => e
-            $stderr.puts e.message
-            $stderr.puts e.backtrace.join("\n")
-            kill_off_threads
-          end
-        end
-      end
-
-      def process_lines(lines)
-        to_process = lines.map do |line|
-          path = line[2..-1]
-          stripped_path = strip_base_path(path)
-          next unless @opts[:ignore].nil? || @opts[:ignore].none?{|i| stripped_path.match(i) }
-          next unless File.directory?(path)
-
-          [stripped_path, generate_file_list(path)]
-        end.compact.sort_by(&:first)
-
-        return unless to_process.any?
-        logger.debug("PROCESSING #{to_process.count} folder/s")
-        send_to_remote(type: :new_changes, content: to_process)
-      end
-
-      def generate_file_list(path)
-        dirs = []
-        files = {}
-
-        Dir.entries(path).each do |f|
-          next if ['.', '..'].include? f
-          f_path = File.join(path, f)
-          if File.directory? f_path
-            dirs << f
-          else
-            files[f] = [File.size(f_path), File.mtime(f_path).to_i]
-          end
-        end
-
-        {dirs: dirs, files: files}
-      end
-
-      def generate_abs_path(rel_path)
-        File.join(self.base_dir, rel_path)
-      end
-
-      def strip_base_path(path, base_dir = self.base_dir)
-        File.expand_path(path).sub(base_dir, '')
-      end
-
-      def start_notify_daemon_cmd
-        uname = `uname`.strip.downcase
-        raise 'Unsupported OS' unless ['darwin', 'linux'].include?(uname)
-
-        "#{File.join(File.dirname(File.dirname(File.dirname(__FILE__))), 'notifier', 'bin', uname, 'notify')} #{self.base_dir}"
-      end
-
       def logger
         FileUtils::mkdir_p log_dir
         @logger ||= Logger.new(File.join(log_dir, 'entangler.log'))

data/lib/entangler/executor/master.rb

@@ -1,30 +1,29 @@
+require_relative 'background/master'
+
 module Entangler
   module Executor
     class Master < Base
-      def validate_opts
-        super
-        raise 'Missing remote base dir' unless @opts.keys.include?(:remote_base_dir)
-        raise 'Missing remote user' unless @opts.keys.include?(:remote_user)
-        raise 'Missing remote host' unless @opts.keys.include?(:remote_host)
-        @opts[:remote_port] ||= '22'
-        res = `ssh -q #{@opts[:remote_user]}@#{@opts[:remote_host]} -p #{@opts[:remote_port]} -C "[[ -d '#{@opts[:remote_base_dir]}' ]] && echo 'ok' || echo 'missing'"`
-        raise 'Cannot connect to remote' if res.empty?
-        raise 'Remote base dir invalid' unless res.strip == 'ok'
-      end
+      include Entangler::Executor::Background::Master
 
       def run
         perform_initial_rsync
         sleep 1
         start_remote_slave
         super
-        Process.wait @remote_thread[:pid] rescue nil
         @remote_writer.close
         @remote_reader.close
       end
 
-      def kill_off_threads
-        Process.kill("INT", @remote_thread[:pid])
+      private
+      def validate_opts
         super
+        raise 'Missing remote base dir' unless @opts.keys.include?(:remote_base_dir)
+        raise 'Missing remote user' unless @opts.keys.include?(:remote_user)
+        raise 'Missing remote host' unless @opts.keys.include?(:remote_host)
+        @opts[:remote_port] ||= '22'
+        res = `ssh -q #{@opts[:remote_user]}@#{@opts[:remote_host]} -p #{@opts[:remote_port]} -C "[[ -d '#{@opts[:remote_base_dir]}' ]] && echo 'ok' || echo 'missing'"`
+        raise 'Cannot connect to remote' if res.empty?
+        raise 'Remote base dir invalid' unless res.strip == 'ok'
       end
 
       def perform_initial_rsync
@@ -34,12 +33,6 @@ module Entangler
         end
         logger.debug 'Initial sync complete'
       end
-
-      def start_remote_slave
-        require 'open3'
-        @remote_writer, @remote_reader, remote_err, @remote_thread = Open3.popen3("ssh -q #{@opts[:remote_user]}@#{@opts[:remote_host]} -p #{@opts[:remote_port]} -C \"source ~/.rvm/environments/default && entangler slave #{@opts[:remote_base_dir]}\"")
-        remote_err.close
-      end
     end
   end
 end

data/lib/entangler/executor/processing/base.rb

@@ -0,0 +1,122 @@
+module Entangler
+  module Executor
+    module Processing
+      module Base
+        protected
+        def generate_file_list(path)
+          dirs = []
+          files = {}
+
+          Dir.entries(path).each do |f|
+            next if ['.', '..'].include? f
+            f_path = File.join(path, f)
+            if File.directory? f_path
+              dirs << f
+            else
+              files[f] = [File.size(f_path), File.mtime(f_path).to_i]
+            end
+          end
+
+          {dirs: dirs, files: files}
+        end
+
+        def process_new_changes(content)
+          logger.debug("Got #{content.length} new folder changes from remote")
+
+          created_dirs = []
+          dirs_to_remove = []
+          files_to_remove = []
+          files_to_update = []
+
+          content.each do |base, changes|
+            possible_creation_dirs = changes[:dirs].clone
+            possible_creation_files = changes[:files].keys.clone
+            full_base_path = generate_abs_path(base)
+
+            unless File.directory?(full_base_path)
+              FileUtils::mkdir_p(full_base_path)
+              @notify_sleep = Time.now.to_i + 60
+            end
+
+            Dir.entries(full_base_path).each do |f|
+              next if ['.', '..'].include? f
+              full_path = File.join(generate_abs_path(base), f)
+              if File.directory?(full_path)
+                possible_creation_dirs -= [f]
+                dirs_to_remove << full_path unless changes[:dirs].include?(f)
+              elsif changes[:files].has_key?(f)
+                possible_creation_files -= [f]
+                files_to_update << File.join(base, f) unless changes[:files][f] == [File.size(full_path), File.mtime(full_path).to_i]
+              else
+                files_to_remove << full_path
+              end
+            end
+
+            dirs_to_create = possible_creation_dirs.map{|d| File.join(generate_abs_path(base), d)}
+            if dirs_to_create.any?
+              logger.debug("Creating #{dirs_to_create.length} dirs")
+              @notify_sleep = Time.now.to_i + 60
+              FileUtils.mkdir_p dirs_to_create
+            end
+            created_dirs += dirs_to_create
+            files_to_update += possible_creation_files.map{|f| File.join(base, f)}
+          end
+
+          @notify_sleep = Time.now.to_i + 60 if (files_to_remove + created_dirs + dirs_to_remove + files_to_update).any?
+
+          if files_to_remove.any?
+            logger.debug("Deleting #{files_to_remove.length} files")
+            FileUtils.rm files_to_remove
+          end
+          if dirs_to_remove.any?
+            logger.debug("Deleting #{dirs_to_remove.length} dirs")
+            FileUtils.rm_r dirs_to_remove
+          end
+          if files_to_update.any?
+            logger.debug("Creating #{files_to_update.length} new entangled files to sync")
+            send_to_remote(type: :entangled_files, content: files_to_update.map{|f| Entangler::EntangledFile.new(f) })
+          end
+          @notify_sleep = Time.now.to_f + 0.5 if (files_to_remove + created_dirs + dirs_to_remove + files_to_update).any?
+          @notify_sleep += 60 if files_to_update.any?
+        end
+
+        def process_entangled_files(content)
+          logger.debug("Got #{content.length} entangled files from remote")
+          completed_files, updated_files = content.partition(&:done?)
+
+          if completed_files.any?
+            @exported_at = Time.now.to_f
+            @exported_folders = completed_files.map{|ef| "#{File.dirname(generate_abs_path(ef.path))}/" }.uniq
+            completed_files.each(&:export)
+          end
+
+          updated_files = updated_files.find_all{|f| f.state != 1 || f.file_exists? }
+          if updated_files.any?
+            send_to_remote(type: :entangled_files, content: updated_files)
+          end
+          @notify_sleep = Time.now.to_f + 0.5 if completed_files.any?
+        end
+
+        def process_lines(lines)
+          paths = lines.map{|line| line[2..-1] }
+
+          if @exported_at < Time.now.to_f && Time.now.to_f < @exported_at + 2
+            paths -= @exported_folders
+          end
+
+          to_process = paths.map do |path|
+            stripped_path = strip_base_path(path)
+            next unless @opts[:ignore].nil? || @opts[:ignore].none?{|i| stripped_path.match(i) }
+            next unless File.directory?(path)
+
+            [stripped_path, generate_file_list(path)]
+          end.compact.sort_by(&:first)
+
+          return unless to_process.any?
+          logger.debug("PROCESSING #{to_process.count} folder/s")
+          send_to_remote(type: :new_changes, content: to_process)
+        end
+      end
+    end
+  end
+end

data/lib/entangler/version.rb

@@ -1,3 +1,3 @@
 module Entangler
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/entangler.rb

@@ -1,5 +1,5 @@
-require 'entangler/version'
-require 'entangler/entangled_file'
+require_relative 'entangler/version'
+require_relative 'entangler/entangled_file'
 
 module Entangler
   class << self

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: entangler
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Dave Allie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-24 00:00:00.000000000 Z
+date: 2016-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -88,8 +88,11 @@ files:
 - exe/entangler
 - lib/entangler.rb
 - lib/entangler/entangled_file.rb
+- lib/entangler/executor/background/base.rb
+- lib/entangler/executor/background/master.rb
 - lib/entangler/executor/base.rb
 - lib/entangler/executor/master.rb
+- lib/entangler/executor/processing/base.rb
 - lib/entangler/executor/slave.rb
 - lib/entangler/version.rb
 - lib/notifier/bin/darwin/notify