entangler 0.4.1 → 1.0.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 116d9b520b874e6c30aaac728a0d341547bfdd4f
-  data.tar.gz: e3981339b47c28b20c1510b6e97b32086e174a7e
+  metadata.gz: 967e19ece113d7aa2359cf932a74e53ceb422ea6
+  data.tar.gz: 822f1cbd484e5a775de97f0d6bff6010312a4154
 SHA512:
-  metadata.gz: 42aaecf1a355c18f5f44e09e76fc95ad1c4356b1930bee788386e5577f7d5a1f906cbdc5ea1e3c6e7617aa8735a16cb7dd977d17457e5d4573da232e7e0d7df7
-  data.tar.gz: 35f0c0b7be7c7984bf8d5215f4996b62d28cbf57afa21fc888087b44b768d40e2c594e2a5f21c5284012cf6d1a9b24ebde73ce752f8b5ae3ad28c07a1b9d5d36
+  metadata.gz: a5ba1871dc42e6d6aaf7616b8bc9b793b41d7fb150d1aaa501a453894ecb1fa0ad0c92d2e2385cad855a3c2f9ebf52a07100be2bc659ffc098e8717b0fe1e122
+  data.tar.gz: 46bcd6e8caa9fa1ae0a999dbe3a7aef0a579b073ef8886db943bda8686f340f297131dc503815b4326fc6bdc5604d00d4eab5f2c1a72df2b3fb501a4f7e28069

data/.rubocop.yml

@@ -0,0 +1,9 @@
+AllCops:
+  DisplayCopNames: true
+  ExtraDetails: true
+
+Metrics/LineLength:
+  Max: 120
+
+Style/Documentation:
+  Enabled: false

data/.travis.yml

@@ -1,5 +1,20 @@
 sudo: false
 language: ruby
+os:
+  - linux
+  - osx
 rvm:
-  - 2.3.0
-before_install: gem install bundler -v 1.12.5
+  - 2.3.3
+  - ruby-head
+matrix:
+  include:
+  - os: osx
+    rvm: 2.2.5
+  - os: linux
+    rvm: 2.2.6
+  allow_failures:
+    - rvm: ruby-head
+before_install:
+  - gem install bundler -v 1.13.6
+install:
+  - bundle install --jobs=3 --retry=3

data/README.md

@@ -1,9 +1,8 @@
 # Entangler
 
-Syncing tool used to keep a local and remote (over SSH) folder in sync.
+[![Build Status](https://travis-ci.org/daveallie/entangler.svg?branch=master)](https://travis-ci.org/daveallie/entangler)
 
-## Prerequisites
-  - librsync 2.x
+Syncing tool used to keep a local and remote (over SSH) folder in sync.
 
 ## Installation
 
@@ -19,27 +18,29 @@ $ entangler master /some/base/path user@remote:/some/remote/path
 
 ```
 $ entangler -h
-Entangler v0.4.1
+Entangler v1.0.0.beta1
 
 Usage:
    entangler master <base_dir> <remote_user>@<remote_host>:<remote_base_dir> [options]
    entangler master <base_dir> <other_synced_base_dir> [options]
 
 Options:
-    -i, --ignore '/.git'             Ignore folder when syncing, string is regex if surrounded by '/'
-                                     All folders should be relative to the base sync directory, but should start with a '/'.
+    -i, --ignore '.git'              Ignore path when syncing, string is regex if surrounded by '/'
+                                     All paths should be relative to the base sync directory.
     -p, --port PORT                  Overwrite the SSH port (usually 22)
                                      (doesn't do anything in slave mode)
-    -v, --version                    Show version number
+    -v, --verbose                    Log Debug lines
+        --version                    Show version number
     -h, --help                       Show this message
 ```
 
-### Ignoring folders
+### Ignoring files and folders
 
-If you specify string, instead of regex, it will match any folder path starting with that string, i.e. `-i '/.git'` will ignore the `.git` folder and all its sub-directories.
-If you want to just ignore the the `.git` sub-directories but not the content in the git folder, you can use `-i '/^\/\.git\/[^\/]*/'` which will match any path starting with `/.git/`, but not `/.git` itself.
+If you specify a string, instead of a regex, it will match any path starting with that string, i.e. `-i '.git'` will ignore the `.git` 
+folder and all its sub-directories. If you want to just ignore the the `.git` sub-directories but not the content in the git folder, you'll 
+have to use regex. `-i '/^\.git\/[^\/]+\/.*/'` will match all sub-directories of `.git/`, but not the files in `.git`.
 
-You can specify multiple `-i` or `--ignore` flags to ignore multiple directories.
+You can specify multiple `-i` or `--ignore` flags to ignore multiple paths.
 
 ## Development
 
@@ -51,7 +52,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/daveallie/entangler. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
 
+RuboCop::RakeTask.new(:rubocop)
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: [:rubocop, :spec]

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "entangler"
+require 'bundler/setup'
+require 'entangler'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +10,5 @@ require "entangler"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start

data/entangler.gemspec

@@ -4,24 +4,25 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'entangler/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "entangler"
+  spec.name          = 'entangler'
   spec.version       = Entangler::VERSION
-  spec.authors       = ["Dave Allie"]
-  spec.email         = ["dave@daveallie.com"]
+  spec.authors       = ['Dave Allie']
+  spec.email         = ['dave@daveallie.com']
 
-  spec.summary       = %q{Two way file syncer using platform native notify and rdiff syncing.}
-  spec.description   = %q{Two way file syncer using platform native notify and rdiff syncing.}
-  spec.homepage      = "https://github.com/daveallie/entangler"
-  spec.license       = "MIT"
+  spec.summary       = 'Two way file syncer using platform native notify.'
+  spec.description   = 'Two way file syncer using platform native notify.'
+  spec.homepage      = 'https://github.com/daveallie/entangler'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 1.12"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_dependency "lib_ruby_diff", "~> 0.1"
-  spec.add_dependency "to_regexp", "~> 0.2"
+  spec.add_development_dependency 'bundler', '~> 1.12'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.46'
+  spec.add_dependency 'listen', '~> 3.1'
+  spec.add_dependency 'to_regexp', '~> 0.2'
 end

data/exe/entangler

@@ -5,59 +5,60 @@ require 'to_regexp'
 
 options = {}
 OptionParser.new do |opts|
-  opts.banner = %Q(Entangler v#{Entangler::VERSION}
+  opts.banner = %(Entangler v#{Entangler::VERSION}
 
 Usage:
    entangler master <base_dir> <remote_user>@<remote_host>:<remote_base_dir> [options]
    entangler master <base_dir> <other_synced_base_dir> [options])
 
-  opts.separator ""
-  opts.separator "Options:"
+  opts.separator ''
+  opts.separator 'Options:'
 
-  opts.on("-i", "--ignore '/.git'", "Ignore folder when syncing, string is regex if surrounded by '/'",
-          "All folders should be relative to the base sync directory, but should start with a '/'.") do |ignore|
+  opts.on('-i', "--ignore '.git'", "Ignore path when syncing, string is regex if surrounded by '/'",
+          'All paths should be relative to the base sync directory.') do |ignore|
     options[:ignore] ||= []
     options[:ignore] << ignore
   end
 
-  opts.on("-p", "--port PORT", "Overwrite the SSH port (usually 22)", "(doesn't do anything in slave mode)") do |port|
+  opts.on('-p', '--port PORT', 'Overwrite the SSH port (usually 22)', "(doesn't do anything in slave mode)") do |port|
     options[:port] = port
   end
 
-  opts.on_tail("-v", "--version", "Show version number") do |v|
+  opts.on('-v', '--verbose', 'Log Debug lines') do
+    options[:verbose] = true
+  end
+
+  opts.on_tail('--version', 'Show version number') do
     puts Entangler::VERSION
     exit
   end
 
-  opts.on_tail("-h", "--help", "Show this message") do
+  opts.on_tail('-h', '--help', 'Show this message') do
     puts opts
     exit
   end
 end.parse!
 
 mode = ARGV.shift
-unless mode && ['master', 'slave'].include?(mode)
+unless mode && %w(master slave).include?(mode)
   puts "Mode unknown, please read help:\nentangler -h"
   exit 1
 end
 
 base_dir = ARGV.shift
 unless base_dir
-  puts "Missing base directory"
+  puts 'Missing base directory'
   exit 1
 end
 
 if mode == 'master'
   remote_information = ARGV.shift
 
-  unless remote_information
-    raise 'Missing destination information'
-    exit 1
-  end
+  raise 'Missing destination information' unless remote_information
 
   user = host = path = error = nil
   remote_mode = false
-  if remote_information.match(/[^@]+@[^:]+:.+/)
+  if remote_information =~ /[^@]+@[^:]+:.+/
     remote_mode = true
     user, rest = remote_information.split('@', 2)
     host, path = (rest || '').split(':', 2)
@@ -74,17 +75,27 @@ if mode == 'master'
     exit 1
   end
 
-  opts = {remote_base_dir: path, remote_mode: remote_mode}
-  opts.merge!(remote_user: user, remote_host: host) if remote_mode
+  opts = { remote_base_dir: path, remote_mode: remote_mode }
+  if remote_mode
+    opts[:remote_user] = user
+    opts[:remote_host] = host
+  end
   opts[:remote_port] = options[:port] if options[:port]
 else
-  opts = {mode: 'slave'}
+  opts = { mode: 'slave' }
 end
 
 if options[:ignore]
   opts[:ignore] = options[:ignore].map do |opt|
-    opt.to_regexp(detect: true)
+    if ToRegexp::String.literal? opt
+      source, *rest = opt.as_regexp(detect: true)
+      ::Regexp.new "^#{source}", *rest
+    else
+      opt.to_regexp(detect: true)
+    end
   end
 end
 
+opts[:verbose] = options[:verbose]
+
 Entangler.run(base_dir, opts)

data/lib/entangler/entangled_file.rb

@@ -1,23 +1,15 @@
-require 'lib_ruby_diff'
-require 'tempfile'
+require 'fileutils'
 
 module Entangler
   class EntangledFile
-    # 0: file initialized
-    # 1: sig loaded
-    # 2: delta loaded
-    attr_accessor :state
-    attr_accessor :desired_modtime
-    attr_reader :path
+    attr_accessor :desired_modtime, :action
+    attr_reader :path, :contents
 
-    def initialize(rel_path)
+    def initialize(action, rel_path)
+      @action = action
       @path = rel_path
-      @state = 0
       @desired_modtime = Time.now.to_i
-    end
-
-    def done?
-      @state == 2
+      @contents = nil
     end
 
     def full_path
@@ -25,116 +17,53 @@ module Entangler
     end
 
     def file_exists?
-      File.exists?(full_path)
+      File.exist?(full_path)
     end
 
-    def export
-      raise "Delta file doesn't exist when creaing patched file" unless delta_exists?
-      tempfile = Tempfile.new('final_file')
-      if File.exists?(full_path)
-        LibRubyDiff.patch(full_path, delta_file.path, tempfile.path)
-      else
-        temp_empty_file = Tempfile.new('empty_file')
-        LibRubyDiff.patch(temp_empty_file.path, delta_file.path, tempfile.path)
+    def process
+      if action == :create || action == :update
+        create_parent_directory
+        write_contents
+      elsif action == :delete
+        delete_file
       end
-      tempfile.rewind
-      File.open(full_path, 'w'){|f| f.write(tempfile.read)}
-      tempfile.close
-      tempfile.unlink
-      File.utime(File.atime(full_path), @desired_modtime, full_path)
     end
 
     private
-    def signature_exists?
-      defined?(@signature_tempfile)
-    end
 
-    def signature_file
-      return @signature_tempfile if signature_exists?
-      @signature_tempfile = Tempfile.new('sig_file')
-      if File.exists?(full_path)
-        LibRubyDiff.signature(full_path, @signature_tempfile.path)
+    def create_parent_directory
+      dirname = File.dirname(full_path)
+      if File.exist?(dirname)
+        unless File.directory?(dirname)
+          FileUtils.rm dirname
+          FileUtils.mkdir_p dirname
+        end
       else
-        temp_empty_file = Tempfile.new('empty_file')
-        LibRubyDiff.signature(temp_empty_file.path, @signature_tempfile.path)
+        FileUtils.mkdir_p dirname
       end
-      @signature_tempfile.rewind
-      @signature_tempfile
-    end
-
-    def write_signature(contents)
-      @signature_tempfile = Tempfile.new('sig_file')
-      @signature_tempfile.write(contents)
-      @signature_tempfile.rewind
-    end
-
-    def signature
-      signature_file.read
-    end
-
-    def delta_exists?
-      defined?(@delta_tempfile)
-    end
-
-    def delta_file
-      return @delta_tempfile if delta_exists?
-      raise "Signature file doesn't exist when creaing delta" unless signature_exists?
-
-      @delta_tempfile = Tempfile.new('delta_file')
-      LibRubyDiff.delta(full_path, signature_file.path, @delta_tempfile.path)
-      @delta_tempfile.rewind
-      @delta_tempfile
     end
 
-    def write_delta(contents)
-      @delta_tempfile = Tempfile.new('delta_file')
-      @delta_tempfile.write(contents)
-      @delta_tempfile.rewind
+    def write_contents
+      delete_file if file_exists? && File.directory?(full_path)
+      File.open(full_path, 'w') { |f| f.write(contents) }
+      File.utime(File.atime(full_path), desired_modtime, full_path)
     end
 
-    def delta
-      delta_file.read
-    end
-
-    def close_and_unlink_files
-      if signature_exists?
-        @signature_tempfile.close
-        @signature_tempfile.unlink
-        @signature_tempfile = nil
-      end
-
-      if delta_exists?
-        @delta_tempfile.close
-        @delta_tempfile.unlink
-        @delta_tempfile = nil
-      end
+    def delete_file
+      FileUtils.rm_rf(full_path) if file_exists?
     end
 
     def marshal_dump
-      last_arg = nil
-
-      if @state == 0
-        last_arg = signature_file.read
-        @state = 1
-      elsif @state == 1
+      if file_exists? && (action == :create || action == :update)
         @desired_modtime = File.mtime(full_path).to_i
-        last_arg = delta_file.read
-        @state = 2
+        @contents = File.read(full_path)
       end
 
-      close_and_unlink_files
-
-      [@path, @state, @desired_modtime, last_arg]
+      [action, path, desired_modtime, contents]
     end
 
     def marshal_load(array)
-      @path, @state, @desired_modtime, last_arg = *array
-
-      if @state == 1
-        write_signature(last_arg)
-      elsif @state == 2
-        write_delta(last_arg)
-      end
+      @action, @path, @desired_modtime, @contents = *array
     end
   end
 end

data/lib/entangler/executor/background/base.rb

@@ -1,106 +1,111 @@
+require 'listen'
+require 'entangler/entangled_file'
+
 module Entangler
   module Executor
     module Background
       module Base
         protected
-        def wait_for_threads
-          @consumer_thread.join
-          @remote_io_thread.join
-          @local_io_thread.join
-          Process.wait @notify_daemon_pid
-        end
 
-        def kill_off_threads
-          Process.kill("TERM", @notify_daemon_pid) rescue nil
-          @consumer_thread.terminate
-          @remote_io_thread.terminate
-          @local_io_thread.terminate
+        def start_listener
+          logger.info('starting listener')
+          listener.start
         end
 
-        def start_notify_daemon
-          logger.info('starting notify daemon')
-          r,w = IO.pipe
-          @notify_daemon_pid = spawn(start_notify_daemon_cmd, out: w)
-          w.close
-          @notify_reader = r
+        def stop_listener
+          listener.stop
         end
 
         def start_remote_io
           logger.info('starting remote IO')
           @remote_io_thread = Thread.new do
-            begin
+            with_kill_threads_rescue do
               loop do
                 msg = Marshal.load(@remote_reader)
-                next if msg.nil?
-
-                case msg[:type]
-                when :new_changes
-                  process_new_changes(msg[:content])
-                when :entangled_files
-                  process_entangled_files(msg[:content])
-                end
+                process_remote_changes(msg)
               end
-            rescue => e
-              $stderr.puts e.message
-              $stderr.puts e.backtrace.join("\n")
-              kill_off_threads
             end
           end
         end
 
-        def start_local_io
-          logger.info('starting local IO')
-          @local_action_queue = Queue.new
-          @local_io_thread = Thread.new do
-            begin
-              msg = []
-              loop do
-                ready = IO.select([@notify_reader]).first
-                next unless ready && ready.any?
-                break if ready.first.eof?
-                line = ready.first.gets
-                next if line.nil? || line.empty?
-                line = line.strip
-                next if line == '-'
-                @local_action_queue.push line
-              end
-            rescue => e
-              $stderr.puts e.message
-              $stderr.puts e.backtrace.join("\n")
-              kill_off_threads
+        def wait_for_threads
+          @remote_io_thread.join
+        end
+
+        def kill_off_threads
+          @remote_io_thread.terminate
+        end
+
+        private
+
+        def listener
+          @listener ||= begin
+            l = Listen::Listener.new(base_dir) do |modified, added, removed|
+              process_local_changes(generate_entangled_files(added, :create) +
+                                        generate_entangled_files(modified, :update) +
+                                        generate_entangled_files(removed, :delete))
             end
+            l.ignore!(@opts[:ignore])
+            l
           end
         end
 
-        def start_local_consumer
-          @consumer_thread = Thread.new do
-            loop do
-              msg = [@local_action_queue.pop]
-              loop do
-                sleep 0.2
-                break if @local_action_queue.empty?
-                while !@local_action_queue.empty?
-                  msg << @local_action_queue.pop
-                end
-              end
-              while Time.now.to_f <= @notify_sleep
-                sleep 0.5
-                while !@local_action_queue.empty?
-                  msg << @local_action_queue.pop
-                end
-              end
-              process_lines(msg.uniq)
-              msg = []
-              sleep 0.5
+        def generate_entangled_files(paths, action)
+          paths.map { |path| Entangler::EntangledFile.new(action, strip_base_path(path)) }
+        end
+
+        def remove_recently_changed_files(entangled_files)
+          @recently_received_paths.select! { |_, time| Time.now.to_f < time + 0.5 }
+          paths = @recently_received_paths.map(&:first)
+          entangled_files.reject { |ef| paths.include?(ef.path) }
+        end
+
+        def process_local_changes(changes)
+          with_listener_pause(0) do
+            changes = remove_recently_changed_files(changes)
+            if changes.any?
+              logger.info("PROCESSING #{changes.length} local changes")
+              logger.debug(changes.map(&:path).join("\n"))
+              send_to_remote(changes)
             end
           end
         end
 
-        def start_notify_daemon_cmd
-          uname = `uname`.strip.downcase
-          raise 'Unsupported OS' unless ['darwin', 'linux'].include?(uname)
+        def process_remote_changes(changes)
+          with_listener_pause(1) do
+            return if changes.nil?
+            logger.info("PROCESSING #{changes.length} remote changes")
+            logger.debug(changes.map(&:path).join("\n"))
+            changes.each(&:process)
+            update_recently_received_paths(changes)
+          end
+        end
+
+        def update_recently_received_paths(changes)
+          changes.each do |change|
+            index = @recently_received_paths.index { |path, _| path == change.path }
+            if index.nil?
+              @recently_received_paths << [change.path, Time.now.to_f]
+            else
+              @recently_received_paths[index][1] = Time.now.to_f
+            end
+          end
+        end
+
+        def with_kill_threads_rescue
+          yield
+        rescue => e
+          $stderr.puts e.message
+          $stderr.puts e.backtrace.join("\n")
+          kill_off_threads
+        end
 
-          "#{File.join(File.dirname(File.dirname(File.dirname(File.dirname(__FILE__)))), 'notifier', 'bin', uname, 'notify')} #{self.base_dir}"
+        def with_listener_pause(idx)
+          @listener_pauses[idx] = true
+          listener.pause
+          yield
+          @listener_pauses[idx] = false
+          listener.start if @listener_pauses.none?
         end
       end
     end