entangler 0.4.1 → 1.0.0.beta1

data/lib/entangler/executor/background/master.rb

@@ -3,9 +3,10 @@ module Entangler
     module Background
       module Master
         protected
+
         def start_remote_slave
           require 'open3'
-          ignore_opts = @opts[:ignore].map{|regexp| "-i '#{regexp.inspect}'"}.join(' ')
+          ignore_opts = @opts[:ignore].map { |regexp| "-i '#{regexp.inspect}'" }.join(' ')
           entangler_cmd = "entangler slave #{@opts[:remote_base_dir]} #{ignore_opts}"
           ssh_cmd = generate_ssh_command("source ~/.rvm/environments/default && #{entangler_cmd}")
           full_cmd = @opts[:remote_mode] ? ssh_cmd : entangler_cmd
@@ -16,11 +17,15 @@ module Entangler
 
         def wait_for_threads
           super
-          Process.wait @remote_thread[:pid] rescue nil
+          begin
+            Process.wait @remote_thread[:pid]
+          rescue
+            nil
+          end
         end
 
         def kill_off_threads
-          Process.kill("INT", @remote_thread[:pid])
+          Process.kill('INT', @remote_thread[:pid])
           super
         end
       end

data/lib/entangler/executor/base.rb

@@ -2,29 +2,28 @@ require 'logger'
 require 'fileutils'
 require 'thread'
 require_relative 'background/base'
-require_relative 'processing/base'
 
 module Entangler
   module Executor
     class Base
-      include Entangler::Executor::Background::Base, Entangler::Executor::Processing::Base
+      include Entangler::Executor::Background::Base
 
       attr_reader :base_dir
 
       def initialize(base_dir, opts = {})
         @base_dir = File.realpath(File.expand_path(base_dir))
-        @notify_sleep = 0
-        @exported_at = 0
+        @recently_received_paths = []
+        @listener_pauses = [false, false]
         @opts = opts
-        @opts[:ignore] = [/^\/\.git.*/] unless @opts.has_key?(:ignore)
-        @opts[:ignore] << /^\/\.entangler.*/
+        @opts[:ignore] = [/^\.git.*/] unless @opts.key?(:ignore)
+        @opts[:ignore] << /^\.entangler.*/
 
         validate_opts
-        logger.info("Starting executor")
+        logger.info('Starting executor')
       end
 
       def generate_abs_path(rel_path)
-        File.join(self.base_dir, rel_path)
+        File.join(base_dir, rel_path)
       end
 
       def strip_base_path(path, base_dir = self.base_dir)
@@ -32,18 +31,18 @@ module Entangler
       end
 
       def run
-        start_notify_daemon
-        start_local_io
+        start_listener
         start_remote_io
-        start_local_consumer
-        logger.debug("NOTIFY PID: #{@notify_daemon_pid}")
-        Signal.trap("INT") { kill_off_threads }
+        Signal.trap('INT') { kill_off_threads }
         wait_for_threads
+      ensure
+        stop_listener
       end
 
       protected
+
       def validate_opts
-        raise "Base directory doesn't exist" unless Dir.exists?(self.base_dir)
+        raise "Base directory doesn't exist" unless Dir.exist?(base_dir)
       end
 
       def send_to_remote(msg = {})
@@ -51,8 +50,12 @@ module Entangler
       end
 
       def logger
-        FileUtils::mkdir_p log_dir
-        @logger ||= Logger.new(File.join(log_dir, 'entangler.log'))
+        FileUtils.mkdir_p log_dir
+        @logger ||= begin
+          l = Logger.new(File.join(log_dir, 'entangler.log'))
+          l.level = @opts[:verbose] ? Logger::DEBUG : Logger::INFO
+          l
+        end
       end
 
       def log_dir

data/lib/entangler/executor/master.rb

@@ -15,49 +15,99 @@ module Entangler
       end
 
       private
+
       def validate_opts
         super
         if @opts[:remote_mode]
-          raise 'Missing remote base dir' unless @opts.keys.include?(:remote_base_dir)
-          raise 'Missing remote user' unless @opts.keys.include?(:remote_user)
-          raise 'Missing remote host' unless @opts.keys.include?(:remote_host)
           @opts[:remote_port] ||= '22'
-          res = `#{generate_ssh_command("[[ -d '#{@opts[:remote_base_dir]}' ]] && echo 'ok' || echo 'missing'")}`
-          raise 'Cannot connect to remote' if res.empty?
-          raise 'Remote base dir invalid' unless res.strip == 'ok'
+          validate_remote_opts
         else
-          @opts[:remote_base_dir] = File.realpath(File.expand_path(@opts[:remote_base_dir]))
-          raise "Destination directory can't be the same as the base directory" if @opts[:remote_base_dir] == self.base_dir
-          raise "Destination directory doesn't exist" unless Dir.exists?(@opts[:remote_base_dir])
+          validate_local_opts
         end
       end
 
-      def perform_initial_rsync
-        logger.info 'Running initial sync'
-        local_folders = `find #{base_dir} -type d`.split("\n").tap{|a| a.shift(1) }.map{|path| path.sub(base_dir, '') }
+      def validate_local_opts
+        @opts[:remote_base_dir] = File.realpath(File.expand_path(@opts[:remote_base_dir]))
+        raise "Destination directory can't be the same as the base directory" if @opts[:remote_base_dir] == base_dir
+        raise "Destination directory doesn't exist" unless Dir.exist?(@opts[:remote_base_dir])
+      end
 
-        remote_find_cmd = "find #{@opts[:remote_base_dir]} -type d"
-        raw_remote_folders = `#{@opts[:remote_mode] ? generate_ssh_command(remote_find_cmd) : remote_find_cmd}`
-        remote_folders = raw_remote_folders.split("\n").tap{|a| a.shift(1) }.map{|path| path.sub(@opts[:remote_base_dir], '') }
+      def validate_remote_opts
+        keys = @opts.keys
+        raise 'Missing remote base dir' unless keys.include?(:remote_base_dir)
+        raise 'Missing remote user' unless keys.include?(:remote_user)
+        raise 'Missing remote host' unless keys.include?(:remote_host)
+        validate_remote_base_dir
+        validate_remote_entangler_version
+      end
 
-        all_folders = remote_folders | local_folders
-        ignore_matches = all_folders.map{|path| @opts[:ignore].map{|regexp| (regexp.match(path) || [])[0]}.compact.first}.compact.uniq
-        exclude_folders = ignore_matches.map{|path| path[1..-1]}
-        exclude_args = exclude_folders.map{|path| "--exclude #{path}"}.join(' ')
+      def validate_remote_base_dir
+        res = `#{generate_ssh_command("[[ -d '#{@opts[:remote_base_dir]}' ]] && echo 'ok' || echo 'missing'")}`
+        raise 'Cannot connect to remote' if res.empty?
+        raise 'Remote base dir invalid' unless res.strip == 'ok'
+      end
 
-        ssh_settings = @opts[:remote_mode] ? "-e \"ssh -p #{@opts[:remote_port]}\"" : ''
-        remote_path = @opts[:remote_mode] ? "#{@opts[:remote_user]}@#{@opts[:remote_host]}:#{@opts[:remote_base_dir]}/" : "#{@opts[:remote_base_dir]}/"
+      def validate_remote_entangler_version
+        return unless @opts[:remote_mode]
+        res = `#{generate_ssh_command('source ~/.rvm/environments/default && entangler --version')}`
+        remote_version = Gem::Version.new(res.strip)
+        local_version = Gem::Version.new(Entangler::VERSION)
+        return unless major_version_mismatch?(local_version, remote_version)
+        raise 'Entangler version too far apart, please update either local or remote Entangler.' \
+              " Local version is #{local_version} and remote version is #{remote_version}."
+      end
 
-        rsync_cmd = "rsync -azv #{exclude_args} #{ssh_settings} --delete #{base_dir}/ #{remote_path}"
+      def major_version_mismatch?(version1, version2)
+        version1.segments[0] != version2.segments[0] ||
+          (version1.segments[0].zero? && version1 != version2) ||
+          ((version1.prerelease? || version2.prerelease?) && version1 != version2)
+      end
 
-        IO.popen(rsync_cmd).each do |line|
+      def perform_initial_rsync
+        logger.info 'Running initial sync'
+        IO.popen(rsync_cmd_string).each do |line|
           logger.debug line.chomp
         end
         logger.debug 'Initial sync complete'
       end
 
+      def find_all_folders
+        local_folders = process_raw_file_list(`find #{base_dir} -type d`, base_dir)
+
+        remote_find_cmd = "find #{@opts[:remote_base_dir]} -type d"
+        raw_remote_folders = `#{@opts[:remote_mode] ? generate_ssh_command(remote_find_cmd) : remote_find_cmd}`
+        remote_folders = process_raw_file_list(raw_remote_folders, @opts[:remote_base_dir])
+
+        remote_folders | local_folders
+      end
+
+      def process_raw_file_list(output, base)
+        output.split("\n").tap { |a| a.shift(1) }
+              .map { |path| path.sub("#{base}/", '') }
+      end
+
+      def find_rsync_ignore_folders
+        find_all_folders.map do |path|
+          @opts[:ignore].map { |regexp| (regexp.match(path) || [])[0] }.compact.first
+        end.compact.uniq
+      end
+
       def generate_ssh_command(cmd)
-        "ssh -q #{@opts[:remote_user]}@#{@opts[:remote_host]} -p #{@opts[:remote_port]} -C \"#{cmd}\""
+        "ssh -q #{remote_hostname} -p #{@opts[:remote_port]} -C \"#{cmd}\""
+      end
+
+      def remote_hostname
+        "#{@opts[:remote_user]}@#{@opts[:remote_host]}"
+      end
+
+      def rsync_cmd_string
+        exclude_args = find_rsync_ignore_folders.map { |path| "--exclude #{path}" }.join(' ')
+        remote_path = @opts[:remote_mode] ? "#{@opts[:remote_user]}@#{@opts[:remote_host]}:" : ''
+        remote_path += "#{@opts[:remote_base_dir]}/"
+
+        cmd = "rsync -azv #{exclude_args}"
+        cmd += " -e \"ssh -p #{@opts[:remote_port]}\"" if @opts[:remote_mode]
+        cmd + " --delete #{base_dir}/ #{remote_path}"
       end
     end
   end

data/lib/entangler/executor/slave.rb

@@ -10,7 +10,7 @@ module Entangler
 
         @remote_reader = STDIN
         @remote_writer = STDOUT
-        $stderr.reopen(File.join(log_dir, 'entangler.err'), "w")
+        $stderr.reopen(File.join(log_dir, 'entangler.err'), 'w')
       end
     end
   end

data/lib/entangler/version.rb

@@ -1,3 +1,3 @@
 module Entangler
-  VERSION = "0.4.1"
+  VERSION = '1.0.0.beta1'.freeze
 end

data/lib/entangler.rb

@@ -6,7 +6,7 @@ module Entangler
     attr_accessor :executor
 
     def run(base_dir, opts = {})
-      opts = {mode: 'master', remote_mode: true}.merge(opts)
+      opts = { mode: 'master', remote_mode: true }.merge(opts)
 
       require 'entangler/executor/base'
       if opts[:mode] == 'master'
@@ -17,7 +17,7 @@ module Entangler
         self.executor = Entangler::Executor::Slave.new(base_dir, opts)
       end
 
-      self.executor.run
+      executor.run
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: entangler
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 1.0.0.beta1
 platform: ruby
 authors:
 - Dave Allie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-18 00:00:00.000000000 Z
+date: 2016-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,19 +53,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: lib_ruby_diff
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.46'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.46'
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: to_regexp
   requirement: !ruby/object:Gem::Requirement
@@ -80,7 +94,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
-description: Two way file syncer using platform native notify and rdiff syncing.
+description: Two way file syncer using platform native notify.
 email:
 - dave@daveallie.com
 executables:
@@ -90,6 +104,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -106,23 +121,8 @@ files:
 - lib/entangler/executor/background/master.rb
 - lib/entangler/executor/base.rb
 - lib/entangler/executor/master.rb
-- lib/entangler/executor/processing/base.rb
 - lib/entangler/executor/slave.rb
 - lib/entangler/version.rb
-- lib/notifier/README.md
-- lib/notifier/bin/darwin/notify
-- lib/notifier/bin/linux/notify
-- lib/notifier/src/darwin/BUILD
-- lib/notifier/src/darwin/README
-- lib/notifier/src/darwin/notify.c
-- lib/notifier/src/linux/BUILD
-- lib/notifier/src/linux/README
-- lib/notifier/src/linux/notify.c
-- lib/notifier/src/linux/old/BUILD
-- lib/notifier/src/linux/old/README
-- lib/notifier/src/linux/old/kernel-filesystem-monitor-daemon-cat.cpp
-- lib/notifier/src/linux/old/kernel-filesystem-monitor-daemon.cpp
-- lib/notifier/src/linux/old/kernel-filesystem-monitor-daemon.hh
 homepage: https://github.com/daveallie/entangler
 licenses:
 - MIT
@@ -138,13 +138,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
-summary: Two way file syncer using platform native notify and rdiff syncing.
+summary: Two way file syncer using platform native notify.
 test_files: []

data/lib/entangler/executor/processing/base.rb

@@ -1,122 +0,0 @@
-module Entangler
-  module Executor
-    module Processing
-      module Base
-        protected
-        def generate_file_list(path)
-          dirs = []
-          files = {}
-
-          Dir.entries(path).each do |f|
-            next if ['.', '..'].include? f
-            f_path = File.join(path, f)
-            if File.directory? f_path
-              dirs << f
-            else
-              files[f] = [File.size(f_path), File.mtime(f_path).to_i]
-            end
-          end
-
-          {dirs: dirs, files: files}
-        end
-
-        def process_new_changes(content)
-          logger.debug("RECIEVING #{content.length} folder/s from remote:\n#{content.map{|c| "#{c[0][1..-1]}/"}.join("\n")}")
-
-          created_dirs = []
-          dirs_to_remove = []
-          files_to_remove = []
-          files_to_update = []
-
-          content.each do |base, changes|
-            possible_creation_dirs = changes[:dirs].clone
-            possible_creation_files = changes[:files].keys.clone
-            full_base_path = generate_abs_path(base)
-
-            unless File.directory?(full_base_path)
-              FileUtils::mkdir_p(full_base_path)
-              @notify_sleep = Time.now.to_i + 60
-            end
-
-            Dir.entries(full_base_path).each do |f|
-              next if ['.', '..'].include? f
-              full_path = File.join(generate_abs_path(base), f)
-              if File.directory?(full_path)
-                possible_creation_dirs -= [f]
-                dirs_to_remove << full_path unless changes[:dirs].include?(f)
-              elsif changes[:files].has_key?(f)
-                possible_creation_files -= [f]
-                files_to_update << File.join(base, f) unless changes[:files][f] == [File.size(full_path), File.mtime(full_path).to_i]
-              else
-                files_to_remove << full_path
-              end
-            end
-
-            dirs_to_create = possible_creation_dirs.map{|d| File.join(generate_abs_path(base), d)}
-            if dirs_to_create.any?
-              logger.debug("Creating #{dirs_to_create.length} dirs")
-              @notify_sleep = Time.now.to_i + 60
-              FileUtils.mkdir_p dirs_to_create
-            end
-            created_dirs += dirs_to_create
-            files_to_update += possible_creation_files.map{|f| File.join(base, f)}
-          end
-
-          @notify_sleep = Time.now.to_i + 60 if (files_to_remove + created_dirs + dirs_to_remove + files_to_update).any?
-
-          if files_to_remove.any?
-            logger.debug("DELETING #{files_to_remove.length} files")
-            FileUtils.rm files_to_remove
-          end
-          if dirs_to_remove.any?
-            logger.debug("DELETING #{dirs_to_remove.length} dirs")
-            FileUtils.rm_r dirs_to_remove
-          end
-          if files_to_update.any?
-            logger.debug("CREATING #{files_to_update.length} new entangled file/s")
-            send_to_remote(type: :entangled_files, content: files_to_update.map{|f| Entangler::EntangledFile.new(f) })
-          end
-          @notify_sleep = Time.now.to_f + 0.5 if (files_to_remove + created_dirs + dirs_to_remove + files_to_update).any?
-          @notify_sleep += 60 if files_to_update.any?
-        end
-
-        def process_entangled_files(content)
-          logger.debug("UPDATING #{content.length} entangled file/s from remote")
-          completed_files, updated_files = content.partition(&:done?)
-
-          if completed_files.any?
-            @exported_at = Time.now.to_f
-            @exported_folders = completed_files.map{|ef| "#{File.dirname(generate_abs_path(ef.path))}/" }.uniq
-            completed_files.each(&:export)
-          end
-
-          updated_files = updated_files.find_all{|f| f.state != 1 || f.file_exists? }
-          if updated_files.any?
-            send_to_remote(type: :entangled_files, content: updated_files)
-          end
-          @notify_sleep = Time.now.to_f + 0.5 if completed_files.any?
-        end
-
-        def process_lines(lines)
-          paths = lines.map{|line| line[2..-1] }
-
-          if @exported_at < Time.now.to_f && Time.now.to_f < @exported_at + 2
-            paths -= @exported_folders
-          end
-
-          to_process = paths.map do |path|
-            stripped_path = strip_base_path(path)
-            next unless @opts[:ignore].nil? || @opts[:ignore].none?{|i| stripped_path.match(i) }
-            next unless File.directory?(path)
-
-            [stripped_path, generate_file_list(path)]
-          end.compact.sort_by(&:first)
-
-          return unless to_process.any?
-          logger.debug("PROCESSING #{to_process.count} folder/s:\n#{to_process.map{|c| "#{c[0][1..-1]}/"}.join("\n")}")
-          send_to_remote(type: :new_changes, content: to_process)
-        end
-      end
-    end
-  end
-end

data/lib/notifier/README.md

@@ -1,3 +0,0 @@
-The code in the following subdirectories comes from https://github.com/DmitryKoterov/dklab_realsync, unmodified.
-
-It is being redistrubuted under GPL, which it was originally provided under. For more information on GPL, please see https://www.gnu.org/licenses/gpl-3.0.en.html.

data/lib/notifier/src/darwin/BUILD

@@ -1,7 +0,0 @@
-To compile notify.c, run
-
-clang -framework CoreFoundation -framework CoreServices -o notify notify.c
-
-or (if you don't have clang)
-
-gcc -framework CoreFoundation -framework CoreServices -o notify notify.c

data/lib/notifier/src/darwin/README

@@ -1,12 +0,0 @@
-The notify utility continiously prints changes in a specified directory in the following format:
-
-M changed/path/1
-M changed/path/2
--
-
-
-Where M stands for "Modifed", and "-" is the indicator of end of changeset
-
-Utility only returns specific directories that have changed, it does not print
-the files/directories that have been modified: you need to determine them
-yourself

data/lib/notifier/src/darwin/notify.c

@@ -1,63 +0,0 @@
-#include <CoreServices/CoreServices.h>
-#include <CoreFoundation/CoreFoundation.h>
-#include <sys/stat.h>
-
-static void printChangesFunc(ConstFSEventStreamRef streamRef, void *clientCallBackInfo, size_t numEvents, void *eventPaths, const FSEventStreamEventFlags eventFlags[], const FSEventStreamEventId eventIds[]) {
-	char **paths = eventPaths;
-	int i;
-	for (i = 0; i < numEvents; i++) {
-		printf("M %s\n", paths[i]);
-	}
-	printf("-\n");
-	fflush(stdout);
-}
-
-void initFSEvents(const char *path) {
-
-	/* Define variables and create a CFArray object containing
-	 CFString objects containing paths to watch.
-	 */
-
-	CFStringRef mypath = CFStringCreateWithCString(NULL, path, kCFStringEncodingUTF8);
-	CFArrayRef pathsToWatch = CFArrayCreate(NULL, (const void **)&mypath, 1, NULL);
-	void *callbackInfo = NULL; // could put stream-specific data here.
-	FSEventStreamRef stream;
-	CFAbsoluteTime latency = 0.1; /* Latency in seconds */
-
-	/* Create the stream, passing in a callback */
-	stream = FSEventStreamCreate(NULL,
-	                             &printChangesFunc,
-	                             callbackInfo,
-	                             pathsToWatch,
-	                             kFSEventStreamEventIdSinceNow, /* Or a previous event ID */
-	                             latency,
-	                             kFSEventStreamCreateFlagNone /* Flags explained in reference */
-	                             );
-
-	CFRelease(pathsToWatch);
-	CFRelease(mypath);
-
-	/* Create the stream before calling this. */
-	FSEventStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode);
-	FSEventStreamStart(stream);
-}
-
-int main (int argc, const char * argv[]) {
-
-	if(argc != 2) {
-		printf("Usage: %s <path>\n", argv[0]);
-		return 1;
-	}
-
-	struct stat tmp;
-
-	if(stat(argv[1], &tmp) != 0) {
-		perror("Invalid path");
-		return 2;
-	}
-
-	initFSEvents(argv[1]);
-	CFRunLoopRun();
-
-	return 0;
-}

data/lib/notifier/src/linux/BUILD

@@ -1,4 +0,0 @@
-notify.c was built using dietlibc to be as small as possible (using "bin-i386/diet gcc -static -o notify notify.c").
-If you want to compile it without dietlibc, you can just do
-
-  gcc -o notify notify.c

data/lib/notifier/src/linux/README

@@ -1,18 +0,0 @@
-The notify utility continiously prints changes in a specified directory in the following format:
-
-M changed/path/1
-M changed/path/2
--
-
-Where M stands for "Modifed", and "-" is the indicator of end of changeset
-
-Utility only returns specific directories that have changed, it does not print
-the files/directories that have been modified: you need to determine them
-yourself.
-
-Linux-specific notes:
-
-    1. You might need to adjust '/proc/sys/fs/inotify/max_user_watches' to allow more directories to be watched
-    2. If you have a lot of changes you might also want to increase queue size in /proc/sys/fs/inotify/max_queued_events
-    3. Watched queue can overflow and notify utility can run out of watched directories.
-       When you get exit code 3, you need to restart the daemon and re-process all directories you are interested in.