nsync 0.0.2

data/lib/nsync/consumer.rb

@@ -0,0 +1,293 @@
+module Nsync
+  # The Nsync::Consumer is used to handle the consumption of data from an Nsync
+  # repo for the entire app. It reads in the differences between the current
+  # version of data in the database and the new data from the producer, finding
+  # and notifying all affected classes and objects.
+  #
+  # Basic Usage:
+  #     
+  #     Nsync::Config.run do |c|
+  #       # The consumer uses a read-only, bare repository (one ending in .git)
+  #       # This will automatically be created if it does not exist
+  #       c.repo_path = "/local/path/to/hold/data.git"
+  #       # The remote repository url from which to pull data
+  #       c.repo_url = "git@examplegithost:username/data.git"
+  #
+  #       # An object that implements the VersionManager interface 
+  #       # (see Nsync::GitVersionManager) for an example
+  #       c.version_manager = MyCustomVersionManager.new
+  #
+  #       # A lock file path to use for this app
+  #       c.lock_file = "/tmp/app_name_nsync.lock"
+  #
+  #       # The class mapping maps from the class names of the producer classes to
+  #       # the class names of their associated consuming classes. A producer can
+  #       # map to one or many consumers, and a consumer can be mapped to one or many
+  #       # producers. Consumer classes should implement the Consumer interface.
+  #       c.map_class "RawDataPostClass", "Post"
+  #       c.map_class "RawDataInfo", "Info"
+  #     end
+  #
+  #     # Create a new consumer object, this will clone the repo if needed
+  #     consumer = Nsync::Consumer.new
+  #
+  #     # update this app to the latest data, pulling if necessary
+  #     consumer.update
+  #
+  #     # rollback the last change
+  #     consumer.rollback
+  class Consumer
+    attr_accessor :repo
+
+    # There was an issue creating or accessing the repository
+    class CouldNotInitializeRepoError < RuntimeError; end
+   
+    # Sets the repository to the repo at config.repo_path
+    #
+    # If config.repo_url is set and the directory at config.repo_path does not
+    # exist yet, a new bare repository will be cloned from config.repo_url
+    def initialize
+      unless get_or_create_repo
+        raise CouldNotInitializeRepoError
+      end
+    end
+
+    # Updates the data to the latest version
+    #
+    # If the repo has a remote origin, the latest changes will be fetched.
+    #
+    # NOTE: It is critical that the version_manager returns correct results
+    # as this method goes from what it says is the latest commit that was loaded in
+    # to HEAD.
+    def update
+      update_repo &&
+      apply_changes(config.version_manager.version,
+                    repo.head.commit.id)
+    end
+
+    # Rolls back data to the previous loaded version
+    #
+    # NOTE: If you rollback and then update, the 'bad' commit will then be reloaded.
+    # This is primarily meant as a way to get back to a known good state quickly, while
+    # the issues are fixed in the producer.
+    def rollback
+      apply_changes(config.version_manager.version,
+                    config.version_manager.previous_version)
+    end
+
+    # @return [Nsync::Config]
+    def config 
+      Nsync.config
+    end
+
+    # Translates and applies the changes between commit id 'a' and commit id 'b' to
+    # the datastore.  This is used internally by rollback and update. Don't use this
+    # unless you absolutely know what you are doing.
+    #
+    # If you must call this directly, understand that 'a' should almost always be the
+    # commit id of the current data that is loaded into the database. 'b' can be any
+    # commit in the graph, forward or backwards.
+    #
+    # @param [String] a current data version commit id
+    # @param [String] b new data version commit id
+    def apply_changes(a, b)
+      config.lock do
+        config.log.info("[NSYNC] Moving Nsync::Consumer from '#{a}' to '#{b}'")
+        clear_queues
+        diffs = nil
+        diffs = repo.diff(a, b)
+
+        changeset = changeset_from_diffs(diffs)
+
+        if config.ordering
+          config.ordering.each do |klass|
+            begin
+              klass = klass.constantize
+              changes = changeset[klass]
+              if changes
+                apply_changes_for_class(klass, changes)
+              end
+            rescue NameError
+              config.log.warn("[NSYNC] Could not find class '#{klass}' from ordering; skipping")
+            end
+          end
+        else
+          changeset.each do |klass, changes|
+            apply_changes_for_class(klass, changes)
+          end
+        end
+        run_after_finished
+        clear_queues
+        config.version_manager.version = b
+      end
+    end
+
+
+    # @private
+    class Change < Struct.new(:id, :diff) 
+      def type
+        if diff.deleted_file
+          :deleted
+        elsif diff.new_file
+          :added
+        else
+          :modified
+        end
+      end
+
+      def data
+        @data ||= JSON.load(diff.b_blob.data)
+      rescue
+        {}
+      end
+    end
+
+    # Adds a callback to the list of callbacks to occur after main processing
+    # of the class specified by 'klass'. Can be used to handle data relations
+    # between objects of the same class.
+    #
+    # Example:
+    #     
+    #     class Post
+    #       def nsync_update(consumer, event_type, filename, data)
+    #         #... normal data update stuff ...
+    #         post = self
+    #         related_post_source_ids = data['related_post_ids']
+    #         consumer.after_class_finished(Post, lambda {
+    #           posts = Post.all(:conditions => 
+    #             {:source_id => related_post_source_ids })
+    #           post.related_posts = posts
+    #         })
+    #       end
+    #     end
+    #
+    # @param [Class] klass
+    # @param [Proc] l
+    def after_class_finished(klass, l)
+      @after_class_finished_queues[klass] ||= []
+      @after_class_finished_queues[klass] << l
+    end
+
+    # Adds a callback to the list of callbacks to occur after main processing
+    # of the class that is currently being processed. This is essentially an
+    # alias for after_class_finished for the current class
+    #
+    # @param [Proc] l
+    def after_current_class_finished(l)
+      after_class_finished(@current_class_for_queue, l)
+    end
+
+
+    # Adds a callback to the list of callbacks to occur after all changes have
+    # been applied.  This queue executes immediately prior to the current
+    # version being updated
+    #
+    # @param [Proc] l
+    def after_finished(l)
+      @after_finished_queue ||= []
+      @after_finished_queue << l
+    end
+
+    protected
+    def get_or_create_repo
+      if config.local? || File.exists?(config.repo_path)
+        return self.repo = Grit::Repo.new(config.repo_path)
+      end
+
+      config.lock do
+        git = Grit::Git.new(config.repo_path)
+        git.clone({:bare => true}, config.repo_url, config.repo_path)
+        self.repo = Grit::Repo.new(config.repo_path)
+        config.version_manager.version = git.rev_list({:reverse => true}, "master").split("\n").first
+        return self.repo
+      end
+    end
+
+    def update_repo
+      return true if config.local?
+      config.lock do
+        repo.remote_fetch('origin')
+        # from http://www.pragmatic-source.com/en/opensource/tips/automatic-synchronization-2-git-repositories
+        repo.git.reset({:soft => true}, 'FETCH_HEAD')
+        true
+      end
+    end
+
+    def apply_changes_for_class(klass, changes)
+      @current_class_for_queue = klass
+      if klass.respond_to?(:nsync_find)
+        changes.each do |change|
+          objects = klass.nsync_find(change.id)
+          if objects.empty? && change.type != :deleted
+            if klass.respond_to?(:nsync_add_data)
+              config.log.info("[NSYNC] Adding data #{diff_path(change.diff)} to #{klass}")
+              klass.nsync_add_data(self, change.type, diff_path(change.diff), change.data)
+            else
+              config.log.warn("[NSYNC] Class '#{klass}' has no method nsync_add_data; skipping")
+            end
+          else
+            objects.each do |obj|
+              if obj.respond_to?(:nsync_update)
+                obj.nsync_update(self, change.type, diff_path(change.diff),
+                                change.data)
+                config.log.info("[NSYNC] Updating from #{diff_path(change.diff)} to #{obj.inspect}")
+              else
+                config.log.info("[NSYNC] Object #{obj.inspect} has no method nsync_update; skipping")
+              end
+            end
+          end
+        end
+      else
+        config.log.warn("[NSYNC] Consumer class '#{klass}' has no method nsync_find; skipping")
+      end
+      @current_class_for_queue = nil
+      run_after_class_finished(klass)
+    end
+
+    def clear_queues
+      @after_class_finished_queues = {}
+      @after_finished_queue = []
+    end
+
+    def run_after_class_finished(klass)
+      queue = @after_class_finished_queues[klass]
+      if queue
+        queue.each do |l|
+          l.call
+        end
+      end
+    end
+
+    def run_after_finished
+      if @after_finished_queue
+        @after_finished_queue.each do |l|
+          l.call
+        end
+      end
+    end
+
+    def changeset_from_diffs(diffs)
+      diffs.inject({}) do |h, diff|
+        next h if diff_path(diff) =~ /\.gitignore$/
+
+        classes, id = consumer_classes_and_id_from_path(diff_path(diff))
+        classes.each do |klass|
+          h[klass] ||= []
+          h[klass] << Change.new(id, diff)
+        end
+        h
+      end
+    end
+
+    def consumer_classes_and_id_from_path(path)
+      producer_class_name = File.dirname(path).camelize
+      id = File.basename(path, ".json")
+      [config.consumer_classes_for(producer_class_name), id]
+    end
+
+    def diff_path(diff)
+      diff.b_path || diff.a_path
+    end
+  end
+end
+

data/lib/nsync/git_version_manager.rb

@@ -0,0 +1,29 @@
+module Nsync
+  class GitVersionManager
+    def initialize(repo_path = nil)
+      @repo_path = repo_path
+    end
+
+    def repo
+      # if the repo is not ready, lets just hope it works
+      @repo ||= Grit::Repo.new(@repo_path || Nsync.config.repo_path) rescue nil
+    end
+
+    def version
+      repo.head.commit.id if repo
+    end
+
+    def version=(val)
+      val
+    end
+
+    def previous_version
+      previous_versions[0]
+    end
+
+    def previous_versions
+      repo.commits("master", 10, 1).map(&:id) if repo
+    end
+  end
+end
+

data/lib/nsync/producer/methods.rb

@@ -0,0 +1,23 @@
+module Nsync
+  class Producer
+    module InstanceMethods
+      def nsync_write
+        nsync_opts = self.class.read_inheritable_attribute(:nsync_opts)
+        if !nsync_opts[:if] || nsync_opts[:if].call(self)
+          Nsync.config.producer_instance.write_file(nsync_filename, to_nsync_hash)
+        elsif Nsync.config.producer_instance.file_exists?(nsync_filename)
+          nsync_destroy
+        end
+      end
+
+      def nsync_destroy
+        Nsync.config.producer_instance.remove_file(nsync_filename)
+      end
+
+      def nsync_filename
+        nsync_opts = self.class.read_inheritable_attribute(:nsync_opts)
+        File.join(self.class.to_s.underscore, "#{send(nsync_opts[:id_key])}.json")
+      end
+    end
+  end
+end

data/lib/nsync/producer.rb

@@ -0,0 +1,129 @@
+module Nsync
+  # The Nsync::Producer is used to tend to the repo and allow files to be
+  # written out and changesets to be commited.  It is a subclass of the
+  # Nsync::Consumer, which allows a Producer to consume itself.  This gives it
+  # the ability to perform rollbacks and undo misstakes in that way
+  #
+  # Basic Usage:
+  #     Nsync::Config.run do |c|
+  #       # The producer uses a standard repository
+  #       # This will automatically be created if it does not exist
+  #       c.repo_path = "/local/path/to/hold/data"
+  #       # The remote repository url will get data pushed to it
+  #       c.repo_push_url = "git@examplegithost:username/data.git"
+  #
+  #       # This must be Nsync::GitVersionManager if you want things like
+  #       # rollback to work.
+  #       c.version_manager = Nsync::GitVersionManager.new
+  #
+  #       # A lock file path to use for this app
+  #       c.lock_file = "/tmp/app_name_nsync.lock"
+  #     end
+  #
+  #     # make some changes that get written out to the repo
+  #
+  #     @producer = Nsync::Producer.new
+  #
+  #     @producer.commit("Some nice changes for you")
+  class Producer < Consumer
+    # Determines whether a file at 'filename' exists in the working tree
+    def file_exists?(filename)
+      File.exists?(File.join(config.repo_path, filename))
+    end
+
+    # Writes a file to the repo at 'filename' with content in json from the
+    # hash
+    #
+    # @param [String] filename path in the working tree to write to
+    # @param [Hash] hash a hash that can be converted to json to be written
+    def write_file(filename, hash)
+      config.cd do
+        dir = File.dirname(filename)
+        unless [".", "/"].include?(dir) || File.directory?(dir)
+          FileUtils.mkdir_p(File.join(config.repo_path, dir))
+        end
+
+        File.open(File.join(config.repo_path, filename), "w") do |f|
+          f.write( (hash.is_a?(Hash))? hash.to_json : hash )
+        end
+        repo.add(File.join(config.repo_path, filename))
+        config.log.info("[NSYNC] Updated file '#{filename}'")
+      end
+      true
+    end
+
+    # Removes a file from the repo at 'filename'
+    def remove_file(filename)
+      FileUtils.rm File.join(config.repo_path, filename)
+      config.log.info("[NSYNC] Removed file '#{filename}'")
+    end
+
+    # Commits and pushes the current changeset
+    def commit(message="Friendly data update")
+      config.lock do
+        config.cd do
+          repo.commit_all(message)
+          config.log.info("[NSYNC] Committed '#{message}' to repo")
+        end
+      end
+      push
+    end
+
+    # Pushes all changes to the repo_push_url
+    def push
+      if config.remote_push?
+        config.cd do
+          repo.git.push({}, config.repo_push_url, "+master")
+          config.log.info("[NSYNC] Pushed changes")
+        end
+      end
+      true
+    end
+
+    # Returns data to its state at HEAD~1 and sets HEAD to that
+    # This new HEAD state is pushed to the repo_push_url.  Hooray git.
+    def rollback
+      commit_to_rollback = config.version_manager.version
+      commit_to_rollback_to = config.version_manager.previous_version
+      config.cd do
+        repo.git.reset({:hard => true}, commit_to_rollback_to)
+        apply_changes(commit_to_rollback, commit_to_rollback_to)
+      end
+      push
+    end
+
+    protected
+
+    def update(*args)
+      super
+    end
+
+    def apply_changes(*args)
+      super
+    end
+    
+    def get_or_create_repo
+      if File.exists?(config.repo_path)
+        return self.repo = Grit::Repo.new(config.repo_path)
+      end
+
+      config.lock do
+        self.repo = Grit::Repo.init(config.repo_path)
+        write_file(".gitignore", "")
+      end
+      commit("Initial Commit")
+    end
+
+    def consumer_classes_and_id_from_path(path)
+      producer_class_name = File.dirname(path).camelize
+      id = File.basename(path, ".json")
+      classes = config.consumer_classes_for(producer_class_name)
+      
+      # refinement to allow the producer to consume itself
+      if classes.empty?
+        classes = [producer_class_name.constantize].compact
+      end
+      [classes, id]
+    end
+  end
+end

data/lib/nsync.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'fileutils'
+
+# just for now
+gem 'activesupport', "~> 2.3.5"
+require 'active_support'
+
+gem "schleyfox-grit", ">= 2.3.0.1"
+require 'grit'
+
+#up the timeout, as these repos can get quite large
+Grit::Git.git_timeout = 60 # 1 minute should do
+Grit::Git.git_max_size = 100.megabytes # tweak this up for very large changesets
+
+gem "schleyfox-lockfile", ">= 1.0.0"
+require 'lockfile'
+
+begin
+  require 'yajl/json_gem'
+rescue LoadError
+  puts "Yajl not installed; falling back to json"
+  require 'json'
+end
+
+require 'nsync/config'
+require 'nsync/consumer'
+require 'nsync/git_version_manager'
+require 'nsync/producer'
+require 'nsync/class_methods'
+require 'nsync/active_record/methods.rb'

data/nsync.gemspec

@@ -0,0 +1,80 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{nsync}
+  s.version = "0.0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Ben Hughes"]
+  s.date = %q{2010-12-06}
+  s.description = %q{Nsync is designed to allow you to have a separate data
+  processing app with its own data processing optimized database and a consumer
+  app with its own database, while keeping the data as in sync as you want it.}
+  s.email = %q{ben@pixelmachine.org}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.md"
+  ]
+  s.files = [
+    "LICENSE",
+     "README.md",
+     "Rakefile",
+     "VERSION",
+     "jeweler_monkey_patch.rb",
+     "lib/nsync.rb",
+     "lib/nsync/active_record/consumer/methods.rb",
+     "lib/nsync/active_record/methods.rb",
+     "lib/nsync/active_record/producer/methods.rb",
+     "lib/nsync/class_methods.rb",
+     "lib/nsync/config.rb",
+     "lib/nsync/consumer.rb",
+     "lib/nsync/git_version_manager.rb",
+     "lib/nsync/producer.rb",
+     "lib/nsync/producer/methods.rb",
+     "nsync.gemspec"
+  ]
+  s.homepage = %q{http://github.com/schleyfox/nsync}
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Keep your data processors and apps in sync}
+  s.test_files = [
+    "test/active_record_test.rb",
+     "test/classes.rb",
+     "test/helper.rb",
+     "test/nsync_config_test.rb",
+     "test/nsync_consumer_test.rb",
+     "test/nsync_producer_test.rb",
+     "test/repo.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_runtime_dependency(%q<activesupport>, ["~> 2.3.5"])
+      s.add_runtime_dependency(%q<schleyfox-grit>, [">= 2.3.0.1"])
+      s.add_runtime_dependency(%q<schleyfox-lockfile>, [">= 1.0.0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<mocha>, [">= 0"])
+    else
+      s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<activesupport>, ["~> 2.3.5"])
+      s.add_dependency(%q<schleyfox-grit>, [">= 2.3.0.1"])
+      s.add_dependency(%q<schleyfox-lockfile>, [">= 1.0.0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<mocha>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<activesupport>, ["~> 2.3.5"])
+    s.add_dependency(%q<schleyfox-grit>, [">= 2.3.0.1"])
+    s.add_dependency(%q<schleyfox-lockfile>, [">= 1.0.0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<mocha>, [">= 0"])
+  end
+end