entangler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 545e59239dbbe8601083e4d7a827721a0aafbdd4
+  data.tar.gz: f97f1d7eec1f101ad4211acbd872c31ca9876179
+SHA512:
+  metadata.gz: d5f23e59f7bb1319594d4805a82d959e443ea13038c874ccade02b6784de9b2f2065e877d03e0a2051cd00a348e4db18439481a73905d3428bbc3b7facaaf259
+  data.tar.gz: 83fdd384fa0b618539fccc33d1bd8491b4c2edc341bbe8f7a37471405c11820341968d90f434dca165ca4d2c5e2f05062db0742144528cc8c59b08955a07cd86

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/log/
+*.gem

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.12.5

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,49 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at dave@daveallie.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in entangler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Dave Allie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# Entangler
+
+
+
+## Installation
+
+    $ gem install entangler
+
+## Usage
+
+```shell
+entangler master /some/base/path user@remote:/some/remote/path
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/daveallie/entangler. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "entangler"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/entangler.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'entangler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "entangler"
+  spec.version       = Entangler::VERSION
+  spec.authors       = ["Dave Allie"]
+  spec.email         = ["dave@daveallie.com"]
+
+  spec.summary       = %q{Two way file syncer using platform native notify and rdiff syncing.}
+  spec.description   = %q{Two way file syncer using platform native notify and rdiff syncing.}
+  spec.homepage      = "https://github.com/daveallie/entangler"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_dependency "lib_ruby_diff", "~> 0.1"
+end

data/exe/entangler

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+require "entangler"
+
+mode = ARGV[0]
+opts = {}
+
+raise 'Must supply at least 2 args' unless ARGV.length >= 2
+
+if mode == 'master'
+  raise 'Missing remote argument' unless ARGV.length >= 3
+  user, rest = ARGV[2].split('@', 2)
+  host, path = rest.split(':', 2)
+  opts = {remote_base_dir: path, remote_user: user, remote_host: host}
+  opts[:remote_port] = ARGV[3] if ARGV[3]
+elsif mode == 'slave'
+  opts[:mode] = 'slave'
+else
+  raise "Mode: #{mode} unknown, please read documentation"
+end
+
+Entangler.run(ARGV[1], opts)

data/lib/entangler/entangled_file.rb

@@ -0,0 +1,139 @@
+require 'lib_ruby_diff'
+require 'tempfile'
+
+module Entangler
+  class EntangledFile
+    # 0: file initialized
+    # 1: sig loaded
+    # 2: delta loaded
+    attr_accessor :state
+    attr_accessor :desired_modtime
+    attr_reader :path
+
+    def initialize(rel_path)
+      @path = rel_path
+      @state = 0
+      @desired_modtime = Time.now.to_i
+    end
+
+    def done?
+      @state == 2
+    end
+
+    def full_path
+      Entangler.executor.generate_abs_path(@path)
+    end
+
+    def file_exists?
+      File.exists?(full_path)
+    end
+
+    def signature_exists?
+      defined?(@signature_tempfile)
+    end
+
+    def signature_file
+      return @signature_tempfile if signature_exists?
+      @signature_tempfile = Tempfile.new('sig_file')
+      if File.exists?(full_path)
+        LibRubyDiff.signature(full_path, @signature_tempfile.path)
+      else
+        temp_empty_file = Tempfile.new('empty_file')
+        LibRubyDiff.signature(temp_empty_file.path, @signature_tempfile.path)
+      end
+      @signature_tempfile.rewind
+      @signature_tempfile
+    end
+
+    def write_signature(contents)
+      @signature_tempfile = Tempfile.new('sig_file')
+      @signature_tempfile.write(contents)
+      @signature_tempfile.rewind
+    end
+
+    def signature
+      signature_file.read
+    end
+
+    def delta_exists?
+      defined?(@delta_tempfile)
+    end
+
+    def delta_file
+      return @delta_tempfile if delta_exists?
+      raise "Signature file doesn't exist when creaing delta" unless signature_exists?
+
+      @delta_tempfile = Tempfile.new('delta_file')
+      LibRubyDiff.delta(full_path, signature_file.path, @delta_tempfile.path)
+      @delta_tempfile.rewind
+      @delta_tempfile
+    end
+
+    def write_delta(contents)
+      @delta_tempfile = Tempfile.new('delta_file')
+      @delta_tempfile.write(contents)
+      @delta_tempfile.rewind
+    end
+
+    def delta
+      delta_file.read
+    end
+
+    def export
+      raise "Delta file doesn't exist when creaing patched file" unless delta_exists?
+      tempfile = Tempfile.new('final_file')
+      if File.exists?(full_path)
+        LibRubyDiff.patch(full_path, delta_file.path, tempfile.path)
+      else
+        temp_empty_file = Tempfile.new('empty_file')
+        LibRubyDiff.patch(temp_empty_file.path, delta_file.path, tempfile.path)
+      end
+      tempfile.rewind
+      File.open(full_path, 'w'){|f| f.write(tempfile.read)}
+      tempfile.close
+      tempfile.unlink
+      File.utime(File.atime(full_path), @desired_modtime, full_path)
+    end
+
+    def close_and_unlink_files
+      if signature_exists?
+        @signature_tempfile.close
+        @signature_tempfile.unlink
+        @signature_tempfile = nil
+      end
+
+      if delta_exists?
+        @delta_tempfile.close
+        @delta_tempfile.unlink
+        @delta_tempfile = nil
+      end
+    end
+
+    def marshal_dump
+      last_arg = nil
+
+      if @state == 0
+        last_arg = signature_file.read
+        @state = 1
+      elsif @state == 1
+        @desired_modtime = File.mtime(full_path).to_i
+        last_arg = delta_file.read
+        @state = 2
+      end
+
+      close_and_unlink_files
+
+      [@path, @state, @desired_modtime, last_arg]
+    end
+
+    def marshal_load(array)
+      @path, @state, @desired_modtime, last_arg = *array
+
+      if @state == 1
+        write_signature(last_arg)
+      elsif @state == 2
+        write_delta(last_arg)
+      end
+    end
+  end
+end

data/lib/entangler/executor/base.rb

@@ -0,0 +1,242 @@
+require 'logger'
+require 'fileutils'
+require 'thread'
+
+module Entangler
+  module Executor
+    class Base
+      attr_reader :base_dir
+
+      def initialize(base_dir, opts = {})
+        @base_dir = File.realpath(File.expand_path(base_dir))
+        @notify_sleep = 0
+        @opts = opts
+        @opts[:ignore] = [/^\/\.git.*/, /^\/\.entangler.*/, /^\/\.idea.*/, /^\/log.*/, /^\/tmp.*/] unless @opts.has_key?(:ignore)
+        validate_opts
+        logger.info("Starting executor")
+      end
+
+      def validate_opts
+        raise "Base directory doesn't exist" unless Dir.exists?(self.base_dir)
+      end
+
+      def run
+        start_notify_daemon
+        start_local_io
+        start_remote_io
+        start_local_consumer
+        logger.debug("NOTIFY PID: #{@notify_daemon_pid}")
+        Signal.trap("INT") { kill_off_threads }
+        @consumer_thread.join
+        @remote_io_thread.join
+        @local_io_thread.join
+        Process.wait @notify_daemon_pid
+      end
+
+      def kill_off_threads
+        Process.kill("TERM", @notify_daemon_pid) rescue nil
+        @consumer_thread.terminate
+        @remote_io_thread.terminate
+        @local_io_thread.terminate
+      end
+
+      def start_file_transfers(paths, pipe)
+        Marshal.dump(paths.map{|path| EntangledFile.new(path) }, pipe)
+      end
+
+      def start_notify_daemon
+        logger.info('starting notify daemon')
+        r,w = IO.pipe
+        @notify_daemon_pid = spawn(start_notify_daemon_cmd, out: w)
+        w.close
+        @notify_reader = r
+      end
+
+      def start_local_io
+        logger.info('starting local IO')
+        @local_action_queue = Queue.new
+        @local_io_thread = Thread.new do
+          begin
+            msg = []
+            loop do
+              ready = IO.select([@notify_reader]).first
+              next unless ready && ready.any?
+              break if ready.first.eof?
+              line = ready.first.gets
+              next if line.nil? || line.empty?
+              line = line.strip
+              next if line == '-'
+              @local_action_queue.push line
+            end
+          rescue => e
+            $stderr.puts e.message
+            $stderr.puts e.backtrace.join("\n")
+            kill_off_threads
+          end
+        end
+      end
+
+      def start_local_consumer
+        @consumer_thread = Thread.new do
+          loop do
+            msg = [@local_action_queue.pop]
+            while !@local_action_queue.empty?
+              msg << @local_action_queue.pop
+            end
+            while Time.now.to_i <= @notify_sleep
+              sleep 1
+              while !@local_action_queue.empty?
+                msg << @local_action_queue.pop
+              end
+            end
+            process_lines(msg.uniq)
+            msg = []
+            sleep 1
+          end
+        end
+      end
+
+      def send_to_remote(msg = {})
+        Marshal.dump(msg, @remote_writer)
+      end
+
+      def start_remote_io
+        logger.info('starting remote IO')
+        @remote_io_thread = Thread.new do
+          begin
+            loop do
+              msg = Marshal.load(@remote_reader)
+              next if msg.nil?
+
+              case msg[:type]
+              when :new_changes
+                logger.debug("Got #{msg[:content].length} new folder changes from remote")
+
+                created_dirs = []
+                dirs_to_remove = []
+                files_to_remove = []
+                files_to_update = []
+
+                msg[:content].each do |base, changes|
+                  possible_creation_dirs = changes[:dirs].clone
+                  possible_creation_files = changes[:files].keys.clone
+
+                  Dir.entries(generate_abs_path(base)).each do |f|
+                    next if ['.', '..'].include? f
+                    full_path = File.join(generate_abs_path(base), f)
+                    if File.directory?(full_path)
+                      possible_creation_dirs -= [f]
+                      dirs_to_remove << full_path unless changes[:dirs].include?(f)
+                    elsif changes[:files].has_key?(f)
+                      possible_creation_files -= [f]
+                      files_to_update << File.join(base, f) unless changes[:files][f] == [File.size(full_path), File.mtime(full_path).to_i]
+                    else
+                      files_to_remove << full_path
+                    end
+                  end
+
+                  dirs_to_create = possible_creation_dirs.map{|d| File.join(generate_abs_path(base), d)}
+                  if dirs_to_create.any?
+                    logger.debug("Creating #{dirs_to_create.length} dirs")
+                    @notify_sleep = Time.now.to_i + 60
+                    FileUtils.mkdir_p dirs_to_create
+                  end
+                  created_dirs += dirs_to_create
+                  files_to_update += possible_creation_files.map{|f| File.join(base, f)}
+                end
+
+                @notify_sleep = Time.now.to_i + 60 if (files_to_remove + created_dirs + dirs_to_remove + files_to_update).any?
+
+                if files_to_remove.any?
+                  logger.debug("Deleting #{files_to_remove.length} files")
+                  FileUtils.rm files_to_remove
+                end
+                if dirs_to_remove.any?
+                  logger.debug("Deleting #{dirs_to_remove.length} dirs")
+                  FileUtils.rm_r dirs_to_remove
+                end
+                if files_to_update.any?
+                  logger.debug("Creating #{files_to_update.length} new entangled files to sync")
+                  send_to_remote(type: :entangled_files, content: files_to_update.map{|f| Entangler::EntangledFile.new(f) })
+                end
+                @notify_sleep = Time.now.to_i + 1 if (files_to_remove + created_dirs + dirs_to_remove + files_to_update).any?
+                @notify_sleep += 60 if files_to_update.any?
+              when :entangled_files
+                logger.debug("Got #{msg[:content].length} entangled files from remote")
+                completed_files, updated_files = msg[:content].partition(&:done?)
+
+                completed_files.each(&:export)
+
+                updated_files = updated_files.find_all{|f| f.state != 1 || f.file_exists? }
+                if updated_files.any?
+                  send_to_remote(type: :entangled_files, content: updated_files)
+                end
+                @notify_sleep = Time.now.to_i + 1 if completed_files.any?
+              end
+            end
+          rescue => e
+            $stderr.puts e.message
+            $stderr.puts e.backtrace.join("\n")
+            kill_off_threads
+          end
+        end
+      end
+
+      def process_lines(lines)
+        to_process = lines.map do |line|
+          path = line[2..-1]
+          stripped_path = strip_base_path(path)
+          next unless @opts[:ignore].nil? || @opts[:ignore].none?{|i| stripped_path.match(i) }
+          next unless File.directory?(path)
+
+          [stripped_path, generate_file_list(path)]
+        end.compact.sort_by(&:first)
+
+        return unless to_process.any?
+        logger.debug("PROCESSING #{to_process.count} folder/s")
+        send_to_remote(type: :new_changes, content: to_process)
+      end
+
+      def generate_file_list(path)
+        dirs = []
+        files = {}
+
+        Dir.entries(path).each do |f|
+          next if ['.', '..'].include? f
+          f_path = File.join(path, f)
+          if File.directory? f_path
+            dirs << f
+          else
+            files[f] = [File.size(f_path), File.mtime(f_path).to_i]
+          end
+        end
+
+        {dirs: dirs, files: files}
+      end
+
+      def generate_abs_path(rel_path)
+        File.join(self.base_dir, rel_path)
+      end
+
+      def strip_base_path(path, base_dir = self.base_dir)
+        File.expand_path(path).sub(base_dir, '')
+      end
+
+      def start_notify_daemon_cmd
+        uname = `uname`.strip.downcase
+        raise 'Unsupported OS' unless ['darwin', 'linux'].include?(uname)
+
+        "#{File.join(File.dirname(File.dirname(File.dirname(__FILE__))), 'notifier', 'bin', uname, 'notify')} #{self.base_dir}"
+      end
+
+      def logger
+        FileUtils::mkdir_p log_dir
+        @logger ||= Logger.new(File.join(log_dir, 'entangler.log'))
+      end
+
+      def log_dir
+        File.join(base_dir, '.entangler', 'log')
+      end
+    end
+  end
+end

data/lib/entangler/executor/master.rb

@@ -0,0 +1,45 @@
+module Entangler
+  module Executor
+    class Master < Base
+      def validate_opts
+        super
+        raise 'Missing remote base dir' unless @opts.keys.include?(:remote_base_dir)
+        raise 'Missing remote user' unless @opts.keys.include?(:remote_user)
+        raise 'Missing remote host' unless @opts.keys.include?(:remote_host)
+        @opts[:remote_port] ||= '22'
+        res = `ssh -q #{@opts[:remote_user]}@#{@opts[:remote_host]} -p #{@opts[:remote_port]} -C "[[ -d '#{@opts[:remote_base_dir]}' ]] && echo 'ok' || echo 'missing'"`
+        raise 'Cannot connect to remote' if res.empty?
+        raise 'Remote base dir invalid' unless res.strip == 'ok'
+      end
+
+      def run
+        perform_initial_rsync
+        sleep 1
+        start_remote_slave
+        super
+        Process.wait @remote_thread[:pid] rescue nil
+        @remote_writer.close
+        @remote_reader.close
+      end
+
+      def kill_off_threads
+        Process.kill("INT", @remote_thread[:pid])
+        super
+      end
+
+      def perform_initial_rsync
+        logger.info 'Running initial sync'
+        IO.popen("rsync -azv --exclude .git --exclude log --exclude .entangler --exclude tmp -e \"ssh -p #{@opts[:remote_port]}\" --delete #{base_dir}/ #{@opts[:remote_user]}@#{@opts[:remote_host]}:#{@opts[:remote_base_dir]}/").each do |line|
+          logger.debug line.chomp
+        end
+        logger.debug 'Initial sync complete'
+      end
+
+      def start_remote_slave
+        require 'open3'
+        @remote_writer, @remote_reader, remote_err, @remote_thread = Open3.popen3("ssh -q #{@opts[:remote_user]}@#{@opts[:remote_host]} -p #{@opts[:remote_port]} -C \"source ~/.rvm/environments/default && entangler slave #{@opts[:remote_base_dir]}\"")
+        remote_err.close
+      end
+    end
+  end
+end

data/lib/entangler/executor/slave.rb

@@ -0,0 +1,17 @@
+module Entangler
+  module Executor
+    class Slave < Base
+      def initialize(base_dir, opts = {})
+        super(base_dir, opts)
+        STDIN.binmode
+        STDOUT.binmode
+        STDIN.sync = true
+        STDOUT.sync = true
+
+        @remote_reader = STDIN
+        @remote_writer = STDOUT
+        $stderr.reopen(File.join(log_dir, 'entangler.err'), "w")
+      end
+    end
+  end
+end

data/lib/entangler/version.rb

@@ -0,0 +1,3 @@
+module Entangler
+  VERSION = "0.1.0"
+end