coney_island 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 271f0c2f183fa09fb618281c98c3d96093d3281c
+  data.tar.gz: 23f311b3570b4417bc9075120a10f3ce649b3549
+SHA512:
+  metadata.gz: 9375cb789ac04d3fc4d290e23438f61536985ab6df1c6274d2a391769e7ce471c5ee17691ad170ab2cd26c706be2963534afc2cd7824d0ff8ab0559ab4e93cf0
+  data.tar.gz: 3e69e4a3273ae65981e8ce1405c6b1173dd0dfb5372db22e73bad8f1f5fd71502326be5463528ba2b954fd4a2a39025a602137919549a3ebb3b757d5d2c149cb

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2014 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= ConeyIsland
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,32 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ConeyIsland'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/bin/coney_island

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.push '.'
+require 'config/environment.rb'
+
+## Copyright 2011 Eric Draut, all rights reserved
+
+ConeyIsland.run_inline
+ConeyIsland.initialize_background
+ConeyIsland.start

data/lib/coney_island.rb

@@ -0,0 +1,301 @@
+module ConeyIsland
+  BG_TIMEOUT_SECONDS = 30
+
+  def self.run_inline
+    @run_inline = true
+  end
+
+  def self.notification_service=(service_name)
+    @notifier = "ConeyIsland::Notifiers::#{service_name}Notifier".constantize
+  end
+
+  def self.amqp_connection
+    @connection
+  end
+
+  def self.amqp_parameters=(params)
+    @amqp_parameters = params
+    @amqp_parameters ||= self.config['amqp_connection']
+  end
+
+  def self.amqp_parameters
+    @amqp_parameters
+  end
+
+  def self.handle_connection
+    @connection ||= AMQP.connect(self.amqp_parameters)
+    @channel  ||= AMQP::Channel.new(@connection)
+    @exchange ||= @channel.topic('coney_island')
+  end
+
+  def self.exchange
+    @exchange
+  end
+
+  def self.channel
+    @channel
+  end
+
+  def self.config
+    if(File.exists?(File.join(Rails.root,"config","coney_island.yml")))
+      @config = Psych.load(File.read(File.join(Rails.root,"config","coney_island.yml")))
+      @config = @config[Rails.env]
+    end
+  end
+
+  #BEGIN web functionality
+
+  def self.submit(*args)
+    if RequestStore.store[:cache_jobs]
+      RequestStore.store[:jobs].push args
+    else
+      self.submit!(args)
+    end
+  end
+
+  def self.submit!(args)
+    if @run_inline
+      self.handle_publish(args)
+    else
+      EventMachine.next_tick do
+        self.handle_publish(args)
+      end
+    end
+  end
+
+  def self.handle_publish(args)
+    self.handle_connection unless @run_inline
+    jobs = (args.first.is_a? Array) ? args : [args]
+    jobs.each do |args|
+      if (args.first.is_a? Class or args.first.is_a? Module) and (args[1].is_a? String or args[1].is_a? Symbol) and args.last.is_a? Hash and 3 == args.length
+        klass = args.shift
+        klass = klass.name unless @run_inline
+        method_name = args.shift
+        job_args = args.shift
+        job_args ||= {}
+        job_args['klass'] = klass
+        job_args['method_name'] = method_name
+        if @run_inline
+          job_args.stringify_keys!
+          method_args = job_args['args']
+          if job_args.has_key? 'instance_id'
+            instance_id = job_args.delete 'instance_id'
+            object = klass.find(instance_id)
+          else
+            object = klass
+          end
+          if method_args && (method_args.length > 0)
+            object.send method_name, *method_args
+          else
+            object.send method_name
+          end
+        else
+          work_queue = job_args.delete :work_queue
+          work_queue ||= 'default'
+          self.exchange.publish((job_args.to_json), routing_key: "carousels.#{work_queue}")
+        end
+      end
+      RequestStore.store[:completed_jobs] ||= 0
+      RequestStore.store[:completed_jobs] += 1
+    end
+  end
+
+  def self.cache_jobs
+    RequestStore.store[:cache_jobs] = true
+    RequestStore.store[:jobs] = []
+  end
+
+  def self.flush_jobs
+    jobs = RequestStore.store[:jobs].dup
+    self.submit!(jobs) if jobs.any?
+    RequestStore.store[:jobs] = []
+  end
+
+  def self.run_with_em(klass, method, *args)
+    EventMachine.run do
+      ConeyIsland.cache_jobs
+      ConeyIsland.submit(klass, method, *args)
+      ConeyIsland.flush_jobs
+      ConeyIsland.publisher_shutdown
+    end
+  end
+
+  def self.publisher_shutdown
+    EventMachine.add_periodic_timer(1) do
+      if RequestStore.store[:jobs] && (RequestStore.store[:jobs].length > RequestStore.store[:completed_jobs])
+        Rails.logger.info("Waiting for #{RequestStore.store[:jobs].length - RequestStore.store[:completed_jobs]} publishes to finish")
+      else
+        Rails.logger.info("Shutting down coney island publisher")
+        EventMachine.stop
+      end
+    end
+  end
+
+  # BEGIN background functionality
+
+  def self.initialize_background
+    ENV['NEW_RELIC_AGENT_ENABLED'] = 'false'
+    ENV['NEWRELIC_ENABLE'] = 'false'
+    @ticket = ARGV[0]
+
+    #TODO: set an env variable or constant that can be checked by pubnub to decide sync or not
+    @log_io = self.config['log'].constantize rescue nil
+    @log_io ||= self.config['log']
+    @log = Logger.new(@log_io)
+
+    @ticket ||= 'default'
+
+    @instance_config = self.config['carousels'][@ticket]
+
+    @prefetch_count = @instance_config['prefetch_count'] if @instance_config
+    @prefetch_count ||= 20
+
+    @worker_count = @instance_config['worker_count'] if @instance_config
+    @worker_count ||= 1
+    @child_count = @worker_count - 1
+    @child_pids = []
+
+    @full_instance_name = @ticket
+    @job_attempts = {}
+
+    @log.level = @config['log_level']
+    @log.info("config: #{self.config}")
+
+  end
+
+  def self.shutdown(signal)
+    shutdown_time = Time.now
+    @child_pids.each do |child_pid|
+      Process.kill(signal, child_pid)
+    end
+    @queue.unsubscribe
+    EventMachine.add_periodic_timer(1) do
+      if @job_attempts.any?
+        @log.info("Waiting for #{@job_attempts.length} requests to finish")
+      else
+        @log.info("Shutting down coney island #{@ticket}")
+        EventMachine.stop
+      end
+    end
+  end
+
+  def self.handle_job(metadata,args,job_id)
+    class_name = args['klass']
+    method_name = args['method_name']
+    klass = class_name.constantize
+    method_args = args['args']
+    timeout = args['timeout']
+    timeout ||= BG_TIMEOUT_SECONDS
+    begin
+      Timeout::timeout(timeout) do
+        if args.has_key? 'instance_id'
+          instance_id = args['instance_id']
+          object = klass.find(instance_id)
+        else
+          object = klass
+        end
+        if method_args and method_args.length > 0
+          object.send method_name, *method_args
+        else
+          object.send method_name
+        end
+      end
+    rescue Timeout::Error => e
+      if @job_attempts.has_key? job_id
+        if @job_attempts[job_id] >= 3
+          @log.error("Request #{job_id} timed out after #{timeout} seconds, bailing out after 3 attempts")
+          self.finalize_job(metadata,job_id)
+          self.poke_the_badger(e, {work_queue: @ticket, job_payload: args, reason: 'Bailed out after 3 attempts'})
+        else
+          @log.error("Request #{job_id} timed out after #{timeout} seconds on attempt number #{@job_attempts[job_id]}, retrying...")
+          @job_attempts[job_id] += 1
+          self.handle_job(metadata,args,job_id)
+        end
+      end
+    rescue Exception => e
+      self.poke_the_badger(e, {work_queue: @ticket, job_payload: args})
+      @log.error("Error executing #{class_name}##{method_name} #{job_id} for id #{args['instance_id']} with args #{args}:")
+      @log.error(e.message)
+      @log.error(e.backtrace.join("\n"))
+      self.finalize_job(metadata,job_id)
+    else
+      self.finalize_job(metadata,job_id)
+    end
+  end
+
+  def self.finalize_job(metadata,job_id)
+    metadata.ack
+    @log.info("finished job #{job_id}")
+    @job_attempts.delete job_id
+  end
+
+  def self.start
+    @child_count.times do
+      child_pid = Process.fork
+      unless child_pid
+        @log.info("started child for ticket #{@ticket} with pid #{Process.pid}")
+        break
+      end
+      @child_pids.push child_pid
+    end
+    defined?(ActiveRecord::Base) and
+      ActiveRecord::Base.establish_connection
+    EventMachine.run do
+
+      Signal.trap('INT') do
+        self.shutdown('INT')
+      end
+      Signal.trap('TERM') do
+        self.shutdown('TERM')
+      end
+
+      self.handle_connection
+      @log.info("Connecting to AMQP broker. Running #{AMQP::VERSION}")
+
+      #send a heartbeat every 15 seconds to avoid aggresive network configurations that close quiet connections
+      heartbeat_exchange = self.channel.fanout('coney_island_heartbeat')
+      EventMachine.add_periodic_timer(15) do
+        heartbeat_exchange.publish({:instance_name => @ticket})
+      end
+
+      self.channel.prefetch @prefetch_count
+      @queue = self.channel.queue(@full_instance_name, auto_delete: false, durable: true)
+      @queue.bind(self.exchange, routing_key: 'carousels.' + @ticket)
+      @queue.subscribe(:ack => true) do |metadata,payload|
+        begin
+          job_id = SecureRandom.uuid
+          @job_attempts[job_id] = 1
+          args = JSON.parse(payload)
+          @log.info ("Starting job #{job_id}: #{args}")
+          if args.has_key? 'delay'
+            EventMachine.add_timer(args['delay'].to_i) do
+              self.handle_job(metadata,args,job_id)
+            end
+          else
+            self.handle_job(metadata,args,job_id)
+          end
+        rescue Timeout::Error => e
+          self.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args, reason: 'timeout in subscribe code before calling job method'})
+        rescue Exception => e
+          self.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args})
+          @log.error("ConeyIsland code error, not application code:\n#{e.inspect}\nARGS: #{args}")
+        end
+      end
+    end
+  end
+
+  def self.poke_the_badger(message, context, attempts = 1)
+    begin
+      Timeout::timeout(3) do
+        @notifier.notify(message, context)
+      end
+    rescue
+      if attempts <= 3
+        attempts += 1
+        self.poke_the_badger(message, context, attempts)
+      end
+    end
+  end
+end
+require 'coney_island/notifiers/honeybadger_notifier'
+

data/lib/coney_island/notifiers/airbrake_notifier.rb

@@ -0,0 +1,9 @@
+module ConeyIsland
+  module Notifiers
+    class AirbrakeNotifier
+      def self.notify(message, extra_params)
+        Airbrake.notify(message, extra_params)
+      end
+    end
+  end
+end

data/lib/coney_island/notifiers/honeybadger_notifier.rb

@@ -0,0 +1,9 @@
+module ConeyIsland
+  module Notifiers
+    class HoneybadgerNotifier
+      def self.notify(message, extra_params)
+        Honeybadger.notify(message, { context: extra_params })
+      end
+    end
+  end
+end

data/lib/coney_island/version.rb

@@ -0,0 +1,3 @@
+module ConeyIsland
+  VERSION = "0.0.3"
+end

data/lib/tasks/coney_island_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :coney_island do
+#   # Task goes here
+# end

data/test/coney_island_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class ConeyIslandTest < ActiveSupport::TestCase
+  test "truth" do
+    assert_kind_of Module, ConeyIsland
+  end
+end

data/test/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.