resque-data-warehouse 0.1.0

data/HISTORY.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2011-01-20)
+
+* Initial version.

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Monica McArthur (mechaferret@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+Software), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+Resque Data Warehouse
+=====================
+
+A [Resque][rq] plugin. Requires Resque 1.9.10.
+
+resque-data-warehouse allows you to use Redis to queue up and then Resque to process transactions 
+on transaction-heavy tables that need to be replicated on other tables optimized for 
+reporting. 
+
+Transactions for a given object (classname + ID) are queued up behind a Redis key, 
+and then processed using Resque jobs. If load is low, each transaction will be processed
+almost immediately after it occurs; at higher loads, multiple transactions will queue up
+before the Resque job gets to them, and then only the last transaction will be applied to the
+data warehousing table, thus minimizing database load and dynamically adjusting the delay
+in the copy to match the current load.
+
+This only works with Rails; it has only been tested with Rails 2.3.4 in which case the after_commit
+gem is also required.
+
+Usage / Examples
+----------------
+
+Suppose you have a database class Transactional, that gets a lot of traffic, and you want to have a counterpart
+to this class, TransactionalFact, in a data warehouse. All you need to do are the following:
+
+* Create an ActiveRecord model named TransactionalFact, with all the instance variables you want in it.
+* Add a method to TransactionalFact named execute_transaction, which assumes that all the fields in the fact that 
+match the original Transactional are already set, saves them, and then performs any additional logic (denormalization, etc.)
+to update any remaining fields.
+* Require 'resque-data-warehouse' in Transactional, and add a line "warehoused".
+
+Very simple examples of both classes:
+
+    class Transactional < ActiveRecord::Base
+      require 'resque-data-warehouse'
+      warehoused
+    end
+
+    class TransactionalFact < ActiveRecord::Base
+      def execute_transaction
+        self.save
+        # Any additional logic here
+      end
+    end
+
+Customize & Extend
+==================
+
+No customizations are available right now, but some obvious ones (such as how to derive the name of the warehoused class)
+are likely to occur soon.
+
+Install
+=======
+
+### As a gem
+
+    $ gem install resque-data-warehouse
+
+### In a Rails app, as a plugin
+
+    $ ./script/plugin install git://github.com/mechaferret/resque-data-warehouse
+
+
+Acknowledgements
+================
+
+Thanks to resque and Redis for making this work possible.
+
+[rq]: http://github.com/defunkt/resque

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake/testtask'
+
+task :default => :test
+
+desc 'Run tests.'
+Rake::TestTask.new(:test) do |task|
+  task.test_files = FileList['test/*_test.rb']
+  task.verbose = true
+end
+

data/lib/resque-data-warehouse.rb

@@ -0,0 +1,2 @@
+require 'resque/plugins/data_warehouse'
+self.send(:include, Resque::Plugins::DataWarehouse)

data/lib/resque/plugins/data_warehouse.rb

@@ -0,0 +1,32 @@
+module Resque
+  module Plugins
+    #
+    # data_warehoused
+    #
+    module DataWarehouse
+    	Dir[File.dirname(__FILE__) + '/data_warehouse/*.rb'].each{|g| require g}  
+    	def self.included(base)
+        base.extend ClassMethods
+      end
+      
+      module ClassMethods
+        def warehoused
+          include InstanceMethods
+          after_commit_on_save :record_to_fact
+          after_destroy :destroy_fact
+        end
+      end
+      
+      module InstanceMethods
+        def record_to_fact
+          DataWarehouse::Transaction.new.enqueue(self)
+        end
+
+        def destroy_fact
+          DataWarehouse::Transaction.new.enqueue(self, 'delete')
+        end
+      end
+
+    end
+  end
+end

data/lib/resque/plugins/data_warehouse/fact.rb

@@ -0,0 +1,20 @@
+module Resque
+  module Plugins
+    module DataWarehouse
+
+      module Fact
+        def self.find(type, values)
+          klass = "#{type}Fact".constantize
+          fact = klass.send(:find, values["id"]) rescue nil
+          fact = klass.new if fact.nil?
+          fact.id = values["id"]
+          values.delete("id")
+          values.delete_if{|key,value| !fact.attribute_names.include?(key)}
+          fact.attributes = values
+          fact
+        end
+      end
+
+    end
+  end
+end

data/lib/resque/plugins/data_warehouse/transaction.rb

@@ -0,0 +1,48 @@
+module Resque
+  module Plugins
+    module DataWarehouse
+      class Transaction
+        @queue = :transaction
+
+        def self.perform(transaction_id, transaction_type, transaction_date)
+          puts "Whee-hee, we're gonna work on a transaction for #{transaction_type} ID #{transaction_id} on #{transaction_date}\n"
+          redis = Resque.redis
+          record = TransactionRecord.new(transaction_id, transaction_type)
+          got_lock = false
+          retries = 0
+          while (!got_lock && retries<2)
+            if (got_lock=record.get_lock)
+              num_trans = redis.llen(record.transaction_key)
+              puts "we'll be processing #{num_trans} transactions\n"
+              num_trans_actual = 0
+              while (data = redis.lpop(record.transaction_key)) 
+                num_trans_actual = num_trans_actual+1
+                puts "transaction\n"
+                next_record = TransactionRecord.new(transaction_id, transaction_type).from_json(data)
+                puts "next_record is #{next_record.inspect}\n"
+                record = record.merge(next_record)
+                puts "merged is #{record.inspect}\n"
+              end
+              puts "Read #{num_trans_actual} transactions"
+              puts "Final trans #{record.inspect}\n"
+              record.execute unless num_trans_actual==0
+              puts "done!"
+              record.release_lock
+            else
+              retries = retries+1
+            end
+          end
+        end
+    
+        def enqueue(model, action = 'save')
+          record = TransactionRecord.new(model.id, model.class.to_s, model.updated_at, model.attributes, action)
+          Resque.redis.rpush(record.transaction_key, record.transaction_data.to_json)
+          Resque.enqueue(self.class, model.id, model.class.to_s, model.updated_at)
+        rescue Exception => ex
+          puts "transaction failing due to exception #{ex.inspect} #{ex.backtrace.join("\n")}"
+        end
+        
+      end
+    end
+  end
+end

data/lib/resque/plugins/data_warehouse/transaction_record.rb

@@ -0,0 +1,79 @@
+module Resque
+  module Plugins
+    module DataWarehouse
+      class TransactionRecord
+        attr_accessor :id, :type, :date, :values, :action
+    
+        def initialize(id, type, date = nil, values = nil, action = 'save')
+          self.id     = id
+          self.type   = type
+          self.date   = date
+          self.values = values
+          self.action = action
+          self
+        end
+    
+        def from_json(data)
+          data_array  = JSON.parse(data)
+          self.date   = Time.parse(data_array[0])
+          self.values = data_array[1]
+          self.action = data_array[2]
+          self
+        end
+    
+        def transaction_key
+          "#{self.type}_#{self.id}"
+        end
+    
+        def transaction_data
+          [self.date, self.values, self.action]
+        end
+    
+        def fact
+          @fact ||= Fact.find(self.type, self.values)
+        end
+    
+        def get_lock
+          redis = Resque.redis
+          if redis.setnx("#{self.transaction_key}_lock", 1)
+            puts "got lock"
+            return true
+          else
+            puts "no lock"
+            return false
+          end
+        end
+    
+        def release_lock
+          redis = Resque.redis
+          redis.del("#{self.transaction_key}_lock")
+          puts "released lock"
+        end
+    
+        def execute
+          if self.action=='delete'
+            self.fact.send("destroy")
+          else
+            self.fact.send("execute_transaction")
+          end
+        end
+    
+        def empty?
+          self.id.blank? || self.type.blank? || self.date.blank?
+        end
+    
+        def merge(t2)
+          if t2.empty?
+            return self
+          elsif self.empty?
+            return t2
+          end
+          d = (self.date <= t2.date) ? t2.date : self.date
+          data = (self.date <= t2.date) ? t2.values : self.values
+          action = (self.date <= t2.date) ? t2.action : self.action
+          TransactionRecord.new(self.id, self.type, d, data, action)
+        end
+      end
+    end
+  end
+end

data/test/database.yml

@@ -0,0 +1,6 @@
+mysql:
+  :adapter: mysql
+  :host: localhost
+  :username: root
+  :password: root
+  :database: dw_test

data/test/debug.log

@@ -0,0 +1,16 @@
+# Logfile created on Thu Jan 20 18:20:02 -0800 2011 by logger.rb/22285
+  SQL (0.2ms)   SET SQL_AUTO_IS_NULL=0
+  SQL (0.3ms)   SHOW TABLES
+  SQL (430.6ms)   CREATE TABLE `transactionals` (`id` int(11) DEFAULT NULL auto_increment PRIMARY KEY, `name` varchar(255), `description` varchar(255), `other_id` int(11)) ENGINE=InnoDB
+  SQL (0.4ms)   SHOW TABLES
+  SQL (114.4ms)   CREATE TABLE `transactional_facts` (`id` int(11) DEFAULT NULL auto_increment PRIMARY KEY, `name` varchar(255), `description` varchar(255), `other_id` int(11)) ENGINE=InnoDB
+  SQL (0.4ms)   SHOW TABLES
+  SQL (102.1ms)   CREATE TABLE `schema_migrations` (`version` varchar(255) NOT NULL) ENGINE=InnoDB
+  SQL (172.5ms)   CREATE UNIQUE INDEX `unique_schema_migrations` ON `schema_migrations` (`version`)
+  SQL (0.4ms)   SHOW TABLES
+  SQL (0.1ms)   SELECT version FROM `schema_migrations`
+  SQL (0.4ms)   INSERT INTO `schema_migrations` (version) VALUES ('3')
+  Transactional Columns (38.0ms)   SHOW FIELDS FROM `transactionals`
+  SQL (0.3ms)   BEGIN
+  Transactional Create (0.3ms)   INSERT INTO `transactionals` (`name`, `other_id`, `description`) VALUES(NULL, NULL, NULL)
+  SQL (0.7ms)   COMMIT

data/test/redis-test.conf

@@ -0,0 +1,132 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile ./test/redis-test.pid
+
+# Accept connections on the specified port, default is 6379
+port 9736
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir ./test/
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile stdout
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+
+# slaveof <masterip> <masterport>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+
+# maxmemory <bytes>
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes
+
+# Use object sharing. Can save a lot of memory if you have many common
+# string in your dataset, but performs lookups against the shared objects
+# pool so it uses more CPU and can be a bit slower. Usually it's a good
+# idea.
+#
+# When object sharing is enabled (shareobjects yes) you can use
+# shareobjectspoolsize to control the size of the pool used in order to try
+# object sharing. A bigger pool size will lead to better sharing capabilities.
+# In general you want this value to be at least the double of the number of
+# very common strings you have in your dataset.
+#
+# WARNING: object sharing is experimental, don't enable this feature
+# in production before of Redis 1.0-stable. Still please try this feature in
+# your development environment so that we can test it better.
+#shareobjects no
+#shareobjectspoolsize 1024

data/test/schema.rb

@@ -0,0 +1,16 @@
+ActiveRecord::Schema.define(:version => 3) do
+  create_table :transactionals, :force => true do |t|
+    t.column :name, :string
+    t.column :description, :string
+    t.column :other_id, :integer
+    t.timestamps
+  end
+  
+  create_table :transactional_facts, :force => true do |t|
+    t.column :name, :string
+    t.column :description, :string
+    t.column :other_id, :integer
+    t.timestamps
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,51 @@
+dir = File.dirname(File.expand_path(__FILE__))
+$LOAD_PATH.unshift dir + '/../lib'
+$TESTING = true
+
+require 'rubygems'
+require 'test/unit'
+require 'resque'
+require 'active_record'
+require 'active_record/fixtures'
+require 'active_support'
+require 'active_support/test_case'
+require 'after_commit'
+
+require 'resque-data-warehouse'
+require dir + '/test_models'
+
+config = YAML::load(IO.read(File.dirname(__FILE__) + '/database.yml'))
+ActiveRecord::Base.configurations = {'test' => config[ENV['DB'] || 'mysql']}
+ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations['test'])
+
+load(File.dirname(__FILE__) + "/schema.rb")
+##
+# make sure we can run redis
+if !system("which redis-server")
+  puts '', "** can't find `redis-server` in your path"
+  puts "** try running `sudo rake install`"
+  abort ''
+end
+
+##
+# start our own redis when the tests start,
+# kill it when they end
+at_exit do
+  next if $!
+
+  if defined?(MiniTest)
+    exit_code = MiniTest::Unit.new.run(ARGV)
+  else
+    exit_code = Test::Unit::AutoRunner.run
+  end
+
+  pid = `ps -e -o pid,command | grep [r]edis-test`.split(" ")[0]
+  puts "Killing test redis server..."
+  `rm -f #{dir}/dump.rdb`
+  Process.kill("KILL", pid.to_i)
+  exit exit_code
+end
+
+puts "Starting redis for testing at localhost:9736..."
+`redis-server #{dir}/redis-test.conf`
+Resque.redis = '127.0.0.1:9736'

data/test/test_models.rb

@@ -0,0 +1,10 @@
+class Transactional < ActiveRecord::Base
+  warehoused
+end
+
+class TransactionalFact < ActiveRecord::Base
+  def execute_transaction
+    puts "executing transaction on transactional fact"
+    self.save
+  end
+end

data/test/warehouse_test.rb

@@ -0,0 +1,78 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class LockTest < Test::Unit::TestCase
+  def setup
+    $success = $lock_failed = $lock_expired = 0
+    Resque.redis.flushall
+    @worker = Resque::Worker.new(:transaction)
+  end
+
+  def test_lint
+    assert_nothing_raised do
+      Resque::Plugin.lint(Resque::Plugins::DataWarehouse)
+    end
+  end
+  
+  def test_create
+    t = Transactional.new(:name=>'Test 1', :description=>'First transaction', :other_id=>2)
+    t.save
+    @worker.process
+    tf = TransactionalFact.find(t.id)
+    assert !tf.nil?
+    assert tf.name==t.name
+    assert tf.description==t.description
+    assert tf.other_id==t.other_id
+  end
+
+  def test_update
+    t = Transactional.new(:name=>'Test 1', :description=>'First transaction', :other_id=>2)
+    t.save
+    @worker.process
+    tf = TransactionalFact.find(t.id)
+    assert !tf.nil?
+    assert tf.name==t.name
+    assert tf.description==t.description
+    assert tf.other_id==t.other_id
+    t.description = 'Change me'
+    t.save
+    @worker.process
+    tf = TransactionalFact.find(t.id)
+    assert !tf.nil?
+    assert tf.name==t.name
+    assert tf.description==t.description
+    assert tf.other_id==t.other_id
+  end
+
+  def test_delete
+    t = Transactional.new(:name=>'Test 1', :description=>'First transaction', :other_id=>2)
+    t.save
+    @worker.process
+    tf = TransactionalFact.find(t.id)
+    assert !tf.nil?
+    assert tf.name==t.name
+    assert tf.description==t.description
+    assert tf.other_id==t.other_id
+    t.destroy
+    @worker.process
+    assert_raise(ActiveRecord::RecordNotFound) do
+      tf = TransactionalFact.find(t.id)
+    end
+  end
+
+  def test_multiple_transactions
+    t = Transactional.new(:name=>'Test 1', :description=>'First transaction', :other_id=>2)
+    t.save
+    assert_raise(ActiveRecord::RecordNotFound) do
+      tf = TransactionalFact.find(t.id)
+    end
+    t.description = 'Update me'
+    t.save
+    @worker.process
+    tf = TransactionalFact.find(t.id)
+    assert !tf.nil?
+    assert tf.name==t.name
+    assert tf.description==t.description
+    assert tf.other_id==t.other_id
+  end
+
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification 
+name: resque-data-warehouse
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Monica McArthur
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-21 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 39
+        segments: 
+        - 1
+        - 9
+        - 10
+        version: 1.9.10
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rails
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 3
+        - 4
+        version: 2.3.4
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: after_commit
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 0
+        - 8
+        version: 1.0.8
+  type: :runtime
+  version_requirements: *id003
+description: "  A Resque plugin. Allows you to use Redis to queue up and then Resque to process transactions \n  on transaction-heavy tables that need to be replicated on other tables optimized for \n  reporting.\n"
+email: mechaferret@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- Rakefile
+- LICENSE
+- HISTORY.md
+- lib/resque/plugins/data_warehouse/fact.rb
+- lib/resque/plugins/data_warehouse/transaction.rb
+- lib/resque/plugins/data_warehouse/transaction_record.rb
+- lib/resque/plugins/data_warehouse.rb
+- lib/resque-data-warehouse.rb
+- test/database.yml
+- test/debug.log
+- test/redis-test.conf
+- test/schema.rb
+- test/test_helper.rb
+- test/test_models.rb
+- test/warehouse_test.rb
+has_rdoc: true
+homepage: http://github.com/mechaferret/resque-data-warehouse
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A Resque plugin for using Resque and Redis to store and process transactions between transactional and data warehouse tables.
+test_files: []
+