collective 0.2.0

data/.autotest

@@ -0,0 +1,13 @@
+# (this file is loaded automatically by autotest)
+# -*- encoding: utf-8 -*-
+
+# Bundler support
+# @see https://www.relishapp.com/rspec/rspec-core/docs/autotest
+require "autotest/bundler"
+
+# ... add the following line after all other requires in your ~/.autotest:
+begin
+  require "autotest/fsevent"
+rescue LoadError => x
+  raise x if RUBY_PLATFORM =~ /darwin/
+end

data/.gitignore

@@ -0,0 +1,5 @@
+*.log
+*.pid
+.bundle
+coverage
+Gemfile.lock

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,16 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in collective.gemspec
+gemspec
+
+group :development, :test do
+  gem "guard-rspec"
+  gem "rb-fsevent"      # for guard
+  gem "growl_notify"    # for guard
+  gem "rspec"
+  gem "ruby-debug19", require: false
+end
+
+group :test do
+  gem "simplecov", require: false
+end

data/Guardfile

@@ -0,0 +1,6 @@
+guard "rspec" do
+  watch(%r{^lib/collective\.rb$})       { "spec" }
+  watch(%r{^lib/collective/(.+)\.rb$})  { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^spec/.+_spec\.rb$})
+  watch("spec/helper.rb")         { "spec" }
+end

data/README

@@ -0,0 +1,7 @@
+Collective helps to run pools of workers.
+
+A pool contains several workers.
+
+Workers are processes which run jobs.
+
+Jobs are simple tasks.

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+# the minitest way
+# task :test do
+#   require_relative "./spec/helper"
+#   FileList["spec/**/*_{spec}.rb"].each { |f| load(f) }
+# end
+
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/bin/collective

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+=begin
+
+  Usage:
+  collective [options] [check] | monitor | restart | status | stop
+
+  e.g.
+  collective --dry-run --verbose --env production --name it --chdir /tmp/it check
+  collective /opt/rails/it/current/config/it monitor
+  Options must preceed the command.
+
+  Note: the pid file depends on chdir, env, and name, so you have to specify them for any command to work.
+  They can be specified in a configuration file.
+
+=end
+
+# We don't use Bundler because that is up to the client.
+# The client can use 'bundle exec collective' if desired.
+File.expand_path(File.dirname(__FILE__)+"/../lib").tap { |d| $:.unshift(d) if ! $:.member?(d) }
+require "collective"
+
+my = Collective::Configuration.parse(ARGV)
+
+# TODO use my.options_for_daemon_spawn
+# TODO use my.args_for_daemon_spawn
+
+if ! my.dry_run then
+  case ARGV.first
+  when "stop"
+    Collective::Monitor.new(my).stop_all
+  when "monitor"
+    Collective::Monitor.new(my).monitor
+  else
+    Kernel.abort "Unknown command #{ARGV.first}"
+  end
+end

data/collective.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+File.expand_path("../lib", __FILE__).tap { |p| $:.push(p) unless $:.member?(p) }
+
+require "collective/version"
+
+Gem::Specification.new do |s|
+  s.name        = "collective"
+  s.version     = Collective::VERSION
+  s.authors     = ["Mark Lanett"]
+  s.email       = ["mark.lanett@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Manage a collection of worker processes}
+  
+  s.rubyforge_project = "collective"
+  
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_dependency "redis"
+  s.add_dependency "redis-namespace"
+end

data/demo/demo

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby bin/collective
+# -*- encoding: utf-8 -*-
+
+# This file is evaluated in the context of a Collective::Configuration instance.
+# DSL attributes include: env, set_env, name, set_name (best to set these via command options)
+# DSL methods include: chdir, add_path, set_defaults, add_pool, after_fork.
+
+add_path File.dirname(__FILE__)
+require "job1"
+
+# best to pass these on the command line
+# set_env "development" if ! env
+# set_name "demo"
+# chdir "/tmp/demo"
+
+set_defaults observers: [ :log ] unless env == "test" # No noise when testing please.
+
+set_defaults(
+  pool_min_workers:     1,
+  worker_late:          10,
+  worker_hung:          100,
+  batchsize:            100,
+  worker_max_lifetime:  3600,
+  observers:            [],
+  storage:              :redis
+)
+
+add_pool Job1,
+  worker_late:          60,
+  pool_min_workers:     1,
+  pool_max_workers:     10
+
+after_fork do
+  # reset ActiveRecord and other pools
+  puts "Forked!"
+end

data/demo/demo.rb

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+# This file is evaluated in the context of a Collective::Configuration instance.
+# DSL attributes include: env, set_env, name, set_name (best to set these via command options)
+# DSL methods include: chdir, add_path, set_defaults, add_pool, after_fork.
+
+add_path File.dirname(__FILE__)
+require "job1"
+
+set_env   "development" if ! env
+set_name  "demo"
+chdir     "/tmp/demo"
+
+set_defaults(
+  pool_min_workers:     1,
+  worker_late:          10,
+  worker_hung:          100,
+  batchsize:            100,
+  worker_max_lifetime:  3600
+)
+
+add_pool Job1,
+  worker_late:          60,
+  pool_min_workers:     1,
+  pool_max_workers:     10
+
+after_fork do
+  # reset ActiveRecord and other pools
+  puts "Forked!"
+end

data/demo/demo3

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby bin/collective
+# -*- encoding: utf-8 -*-
+
+# This file is evaluated in the context of a Collective::Configuration instance.
+# DSL attributes include: env, set_env, name, set_name (best to set these via command options)
+# DSL methods include: chdir, add_path, set_defaults, add_pool, after_fork.
+
+add_path File.dirname(__FILE__)
+require "job3"
+
+# best to pass these on the command line
+# set_env "development" if ! env
+# set_name "demo"
+# chdir "/tmp/demo"
+
+set_defaults observers: [ :log ] unless env == "test" # No noise when testing please.
+
+set_defaults(
+  pool_min_workers:     3,
+  worker_late:          10,
+  worker_hung:          100,
+  batchsize:            100,
+  worker_max_lifetime:  3600,
+  observers:            [],
+  storage:              :redis
+)
+
+add_pool Job3,
+  worker_late:          60,
+  pool_min_workers:     10,
+  pool_max_workers:     20
+
+after_fork do
+  # reset ActiveRecord and other pools
+  puts "Forked!"
+end

data/demo/job1.rb

@@ -0,0 +1,31 @@
+require "collective"
+
+=begin
+
+  Mostly does nothing.
+  Otherwise runs in 1-10 seconds.
+
+=end
+
+class Job1
+  
+  def initialize( options = {} )
+  end
+  
+  include Collective::Log
+  
+  def call(context)
+    p = rand
+    
+    case
+    when p < 0.7 # 10%
+      log "No action"
+      false
+    else
+      log "Something"
+      sleep(rand(10))
+      true
+    end
+  end
+  
+end # Job1

data/demo/job2.rb

@@ -0,0 +1,42 @@
+require "collective"
+
+=begin
+
+  Sometimes exits, hangs, or aborts.
+  Otherwise does something.
+
+=end
+
+class Job2
+  
+  def initialize( options = {} )
+  end
+  
+  include Collective::Log
+  
+  def call(context)
+    p = rand
+    
+    case
+    when p < 0.1 # 10%
+      log "Going to exit"
+      exit
+    when p < 0.2 # 10%
+      log "Hang"
+      loop do
+        true
+      end
+    when p < 0.3 # 10%
+      log "Going to abort"
+      abort!
+    when p < 0.9 # 60%
+      log "Nothing to do"
+      false
+    else         # 10%
+      log "Doing something..."
+      sleep(rand(10))
+      true
+    end
+  end
+  
+end # Job2

data/demo/job3.rb

@@ -0,0 +1,44 @@
+require "collective"
+require "collective/checker"
+
+=begin
+
+  Sometimes does something but is slow.
+
+=end
+
+class Job3
+
+  attr :redis
+  attr :storage
+
+  def initialize( options = {} )
+    @redis   = Redis.connect url: "redis://127.0.0.1:6379/0"
+    @storage = Collective::Redis::Storage.new(redis)
+  end
+
+  include Collective::Log
+
+  def call(context)
+    page           = storage.queue_pop( "Next" )
+    activity_count = storage.map_get "Activity", page
+    last_time      = storage.map_get( "Last", page )
+    start          = Time.now
+    checker        = Checker.new activity_count, last_time, start
+    begin
+      return if ! checker.check? # executes ensure block
+
+      log "Processing #{page} with activity #{checker.activity_count}; estimated delay #{checker.estimated_delay} sec"
+
+      sleep(checker.estimated_delay)
+      # checker.checked(rand)
+
+      log "Processed #{page}; updated activity count to #{checker.activity_count}; estimated next time #{checker.next_time}"
+
+      storage.map_set "Last", page, start.to_i
+    ensure
+      storage.queue_add( "Next", page, checker.next_time )
+    end
+  end
+
+end # Job3

data/demo/populate.rb

@@ -0,0 +1,22 @@
+File.expand_path(File.dirname(__FILE__)+"/../lib").tap { |d| $: << d unless $:.member?(d) }
+require "collective"
+require "collective/squiggly"
+require "redis"
+
+redis   = Redis.connect url: "redis://127.0.0.1:6379/0"
+storage = Collective::Redis::Storage.new(redis)
+
+storage.del "Names"
+storage.del "Activity"
+storage.del "Weight"
+storage.del "Next"
+
+(1..1000).each do |i|
+  name     = Squiggly.subject
+  activity = rand(1000)
+
+  storage.map_set "Names", "Page-#{i}", name
+  storage.map_set "Activity", "Page-#{i}", activity
+
+  storage.queue_add( "Next", "Page-#{i}", 0 )
+end

data/lib/collective.rb

@@ -0,0 +1,52 @@
+File.expand_path(File.dirname(__FILE__)).tap { |d| $: << d unless $:.member?(d) }
+
+require "collective/version"
+
+module Collective
+  autoload :Configuration,    "collective/configuration"
+  autoload :Idler,            "collective/idler"
+  autoload :Key,              "collective/key"
+  autoload :LifecycleObserver,"collective/lifecycle_observer"
+  autoload :Log,              "collective/log"
+  autoload :Messager,         "collective/messager"
+  autoload :Monitor,          "collective/monitor"
+  autoload :Policy,           "collective/policy"
+  autoload :Pool,             "collective/pool"
+  autoload :Registry,         "collective/registry"
+  autoload :Worker,           "collective/worker"
+end
+
+module Collective::Mocks
+  autoload :Storage,          "collective/mocks/storage"
+end
+
+module Collective::Utilities
+  autoload :AirbrakeObserver, "collective/utilities/airbrake_observer"
+  autoload :HoptoadObserver,  "collective/utilities/hoptoad_observer"
+  autoload :LogObserver,      "collective/utilities/log_observer"
+  autoload :NullObserver,     "collective/utilities/null_observer"
+  autoload :Observeable,      "collective/utilities/observeable"
+  autoload :ObserverBase,     "collective/utilities/observer_base"
+  autoload :Process,          "collective/utilities/process"
+  autoload :SignalHook,       "collective/utilities/signal_hook"
+  autoload :StorageBase,      "collective/utilities/storage_base"
+end
+
+module Collective::Redis
+  autoload :Storage,          "collective/redis/storage"
+end
+
+module Collective
+  class << self
+
+    # @param classname
+    # @returns class object
+    def resolve_class(classname)
+      classname.split(/::/).inject(Object) { |a,i| a.const_get(i) }
+    end
+
+  end # class
+end
+
+class Collective::ConfigurationError < Exception
+end

data/lib/collective/checker.rb

@@ -0,0 +1,51 @@
+# -*- encoding: utf-8 -*-
+
+class Checker
+
+  DAY = 86400
+
+  # @param activity_count is a daily level of activities
+  # @param last_time is the last time we checked
+  def initialize( activity_count, last_time, check_time = Time.now )
+    @activity_count = activity_count.to_i                     # e.g. 192 actions
+    @last_time      = Time.at last_time.to_i                  # e.g. 1 hour ago
+    @check_time     = check_time                              # e.g. now
+
+    @last_time      = [ @check_time - DAY, @last_time ].max   # reject huge intervals like since epoch
+    @interval       = @check_time - @last_time                # e.g. 1 hour
+
+    # estimations
+    variation       = 2 * ( rand + rand + rand + rand ) / 4   # ~1.0 more or less
+    day_est         = variation * @activity_count             # e.g. ~192
+    @estimate       = ( day_est * @interval / DAY ).to_i      # e.g. 8 (per hour)
+    next_interval   = @estimate > 0 ? @interval / @estimate : @interval * 2
+  end
+
+  def estimate
+    @estimate
+  end
+
+  def checked( actual )
+    @activity_count = ( actual * DAY / @interval ).to_i               # e.g. 12
+    next_interval   = actual > 0 ? @interval / actual : @interval * 2 # e.g. 300 seconds (3600 / 12)
+    @next_time      = @check_time + next_interval                     # e.g. now + 300 seconds
+  end
+
+  # refuse to check too often
+  def check?
+    @last_time < @check_time - 100
+  end
+
+  def activity_count
+    @activity_count
+  end
+
+  def estimated_delay
+    @estimate > 0 ? Math.sqrt(@estimate).to_i : 1
+  end
+
+  def next_time
+    # If not determined yet, calculate a safety time past the Facebook api limit
+    @next_time ? @next_time : @check_time + 1000
+  end
+end