sync_client 0.0.14

data/README.md

@@ -0,0 +1,80 @@
+# SyncClient
+
+This gem simplifies syncing data between services by using a resque queue and a message queue for guaranteed delivery and eventual consistency of data.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sync_client'
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+Then setup your app as a client (if it isn't already):
+
+```bash
+$ rails g sync_client:install
+```
+
+Edit configuation in `config/initializers/sync_client.rb`
+
+## Usage
+
+Within the model you want to publish attributes to a service include something like the following:
+
+```ruby
+class Team
+  include SyncClient::Publisher
+  publish_changes_of :name, :color, to: :queue, for: [:update, :destroy], if: lambda{|team| !team.name.nil?}
+    # options:
+    #   to: name of queue for publishing (required)
+    #   for: callbacks for publishing (default: :create, :update, :destroy)
+    #   if/unless: condition for publishing
+end
+```
+
+
+A priority is used for publishing to ensure eventual delivery if the message queue does not respond. Supported priority queues include Delayed Job and Resque.
+
+Run the message queue poller:
+
+```bash
+$ bundle exec script/sync_client start
+$ bundle exec script/sync_client status
+$ bundle exec script/sync_client restart
+$ bundle exec script/sync_client stop
+```
+
+Define handlers for the all messages such as the following:
+
+```ruby
+class Game < SyncClient::ServiceResource::Base
+  attr_accessor :id
+  attr_accessor :starts_at
+  attr_accessor :ends_at
+
+  def create
+    Game.create(:game_id => self.id, :starts_at => self.starts_at, :ends_at => self.ends_at)
+  end
+
+  def update
+    game = Game.find(self.id).first
+    game.update_attributes(:starts_at => self.starts_at, :ends_at => self.ends_at)
+  end
+
+  def destroy
+    game = Game.find(self.id)
+    game.each do |ga|
+      ga.destroy
+    end
+  end
+end
+```
+
+

data/Rakefile

@@ -0,0 +1,27 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+begin
+  require 'rdoc/task'
+rescue LoadError
+  require 'rdoc/rdoc'
+  require 'rake/rdoctask'
+  RDoc::Task = Rake::RDocTask
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task :default => :test

data/app/models/jobs/sync_client_jobs/publish.rb

@@ -0,0 +1,12 @@
+module Jobs
+  module SyncClientJobs
+    class Publish
+
+      @queue = :high
+
+      def self.perform(message)
+        SyncClient::PubMessage.new(message).synchronous_publish
+      end
+    end
+  end
+end

data/app/models/sync_client/message.rb

@@ -0,0 +1,15 @@
+module SyncClient
+  class Message
+    attr_accessor :action
+    attr_accessor :object_type
+    attr_accessor :queue
+    attr_accessor :object_attributes
+
+    def initialize(attrs)
+      attrs = {} unless attrs
+      attrs.each do |key, value|
+        send("#{key}=", value) if self.respond_to?("#{key}=")
+      end
+    end
+  end
+end

data/app/models/sync_client/pub_message.rb

@@ -0,0 +1,34 @@
+module SyncClient
+  class PubMessage < Message
+
+    def publish
+      SyncClient.background_task_queue.enqueue(Jobs::SyncClientJobs::Publish, self)
+    end
+
+    def synchronous_publish
+      with_logging do
+        Queuel.with(queue_with_suffix).push package
+      end
+    end
+
+    def package
+      {:action => action, :object_type => object_type_with_service, :object_attributes => object_attributes}
+    end
+
+    def object_type_with_service
+      service = Rails.application.class.parent_name
+      "#{service}::#{object_type}"
+    end
+
+    def queue_with_suffix
+      SyncClient.queue_suffix.blank? ? queue.to_s : "#{queue}_#{SyncClient.queue_suffix}"
+    end
+
+    def with_logging(&block)
+      SyncClient.logger.info("------------------------------------------")
+      SyncClient.logger.info("Publishing Message: #{object_type}##{action}")
+      SyncClient.logger.info("To: #{queue_with_suffix}")
+      yield
+    end
+  end
+end

data/app/models/sync_client/service_resource/base.rb

@@ -0,0 +1,27 @@
+module SyncClient
+  module ServiceResource
+    class Base
+      attr_accessor :error
+
+      # add logic to handle params
+      def initialize(attrs)
+        attrs = {} unless attrs
+        attrs.each do |key, value|
+          send("#{key}=", value) if self.respond_to?("#{key}=")
+        end
+      end
+
+      def create
+        raise NotImplementedError
+      end
+
+      def update
+        raise NotImplementedError
+      end
+
+      def destroy
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/app/models/sync_client/sub_message.rb

@@ -0,0 +1,38 @@
+module SyncClient
+  class SubMessage < Message
+    attr_accessor :success
+
+    def process
+      with_logging do
+        self.success = message_handler.send(action.to_sym) if handler_present?
+      end
+      !!success
+    end
+
+    def handler_present?
+      return true if handler_class and handler_class.actions.include?(action.to_sym)
+      self.success = true #still setting to true to remove message from queue
+      return false
+    end
+
+    def handler_class
+      ::SyncClient.handlers[object_type]
+    end
+
+    def message_handler
+      @handler ||= handler_class.handler.new(object_attributes)
+    end
+
+    def error
+      message_handler.error if handler_class and message_handler.error
+    end
+
+    def with_logging(&block)
+      SyncClient.logger.info("------------------------------------------")
+      SyncClient.logger.info("Recieved Message: #{object_type}##{action}")
+      yield
+      SyncClient.logger.info("Error Occured: #{error}") if error
+      SyncClient.logger.info("Processed Message: #{!!success}")
+    end
+  end
+end

data/lib/generators/sync_client/install_generator.rb

@@ -0,0 +1,24 @@
+require 'rails/generators'
+module SyncClient
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("../../templates", __FILE__)
+      desc "Creates a SyncClient initializer and copy other files to your application."
+      # class_option :bleh
+
+      def copy_initializer
+        template "sync_client.rb", "config/initializers/sync_client.rb"
+      end
+
+      def create_executable_file
+        if ActiveSupport::VERSION::MAJOR >= 4
+          prefix = 'bin'
+        else
+          prefix = 'script'
+        end
+        template "script", "#{prefix}/sync_client"
+        chmod "#{prefix}/sync_client", 0755
+      end
+    end
+  end
+end

data/lib/generators/templates/script

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+require 'daemons'
+require 'sync_client'
+
+Daemons.run_proc('sync_client', :dir_mode => :script, :dir => 'tmp/pids') do
+  require_relative File.join('..', 'config', 'environment.rb')
+  SyncClient::Poller.run
+end

data/lib/generators/templates/sync_client.rb

@@ -0,0 +1,13 @@
+SyncClient.config do |config|
+
+  config.queuel do |c|
+    # c.default_queue 'VenueService'
+    # c.credentials token: 'asdufasdf8a7sd8fa7sdf', project_id: 'project_id'
+    # c.engine :iron_mq
+  end
+  # config.background_task_queue SyncClient::Resque #or SyncClient::DelayedJob or write your own.
+  # config.queue_suffix 'suffix'
+  # config.logger Logger.new
+  # config.add_message_object_handler object_name, handler_class, actions
+  # config.add_message_object_handler 'StatNgin::Game', 'Resource::StatNgin::Game', [:update, :create, :destroy]
+end

data/lib/sync_client.rb

@@ -0,0 +1,46 @@
+require 'rails'
+require 'queuel'
+
+require 'sync_client/publisher'
+require 'sync_client/queue_publisher'
+require 'sync_client/sync_queue'
+require 'sync_client/version'
+require "sync_client/engine"
+require 'sync_client/configurator'
+require 'sync_client/worker'
+require 'sync_client/poller'
+
+module SyncClient
+
+  def self.version_string
+    "SyncClient version #{SyncClient::VERSION}"
+  end
+
+  def self.config
+    @config ||= Configurator.new
+    if block_given?
+      yield @config
+    end
+    @config
+  end
+
+  def self.handlers
+    config.handlers
+  end
+
+  def self.actions
+    Configurator::ACTIONS
+  end
+
+  def self.logger
+    config.sync_logger
+  end
+
+  def self.queue_suffix
+    config.suffix
+  end
+
+  def self.background_task_queue
+    config.task_queue
+  end
+end

data/lib/sync_client/configurations/message_handler.rb

@@ -0,0 +1,15 @@
+module SyncClient
+  module Configurations
+    class MessageHandler
+      attr_reader :name
+      attr_reader :handler
+      attr_reader :actions
+
+      def initialize(name, handler, actions = [])
+        @name = name.to_s.singularize
+        @handler = handler.classify.constantize
+        @actions = actions
+      end
+    end
+  end
+end

data/lib/sync_client/configurator.rb

@@ -0,0 +1,53 @@
+require 'sync_client/configurators/message_handlers'
+require 'sync_client/task_queue/delayed_job'
+require 'sync_client/task_queue/resque'
+require 'sync_client/task_queue/inline_task_queue'
+
+module SyncClient
+  class Configurator
+    private
+    attr_writer :message_handlers
+    attr_writer :suffix
+    attr_writer :sync_logger
+    attr_writer :task_queue
+
+    public
+    attr_reader :message_handlers
+    attr_reader :sync_logger
+    attr_reader :suffix
+    attr_reader :task_queue
+
+    ACTIONS = [:create, :update, :destroy]
+
+    def initialize
+      self.message_handlers = Configurators::MessageHandlers.new
+      self.sync_logger = Logger.new(STDOUT)
+      self.task_queue = SyncClient::InlineTaskQueue
+    end
+
+    def queuel
+      ::Queuel.configure { |c| yield c }
+    end
+
+    def add_message_handler(message, handler, actions)
+      message_handlers.add_message_handler message, handler, actions
+    end
+
+    def handlers
+      message_handlers.message_handlers
+    end
+
+    def queue_suffix(queue_suffix)
+      self.suffix = queue_suffix
+    end
+
+    def logger(logger)
+      self.sync_logger = logger
+    end
+
+    def background_task_queue(queue)
+      self.task_queue = queue
+    end
+
+  end
+end

data/lib/sync_client/configurators/message_handlers.rb

@@ -0,0 +1,15 @@
+require 'sync_client/configurations/message_handler'
+module SyncClient
+  module Configurators
+    class MessageHandlers
+      attr_reader :message_handlers
+      def initialize
+        @message_handlers = {}.with_indifferent_access
+      end
+
+      def add_message_handler(name, handler, actions)
+        message_handlers[name] ||= Configurations::MessageHandler.new(name, handler, actions)
+      end
+    end
+  end
+end

data/lib/sync_client/engine.rb

@@ -0,0 +1,13 @@
+module SyncClient
+  class Engine < ::Rails::Engine
+
+    initializer "sync_client.eager_load", after: :set_autoload_paths do |app|
+      paths["app/models"].each do |load_path|
+        matcher = /\A#{Regexp.escape(load_path)}\/(.*)\.rb\Z/
+        Dir.glob("#{load_path}/**/*.rb").sort.each do |file|
+          require_dependency file.sub(matcher, '\1')
+        end
+      end
+    end
+  end
+end