beetle 0.1 → 0.2.0

data/etc/redis-master.conf

@@ -1,189 +0,0 @@
-# Redis configuration file example
-
-# By default Redis does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
-daemonize no
-
-# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
-# You can specify a custom pid file location here.
-pidfile ./tmp/redis-master.pid
-
-# Accept connections on the specified port, default is 6379
-port 6379
-
-# If you want you can bind a single interface, if the bind option is not
-# specified all the interfaces will listen for connections.
-#
-# bind 127.0.0.1
-
-# Close the connection after a client is idle for N seconds (0 to disable)
-timeout 300
-
-# Set server verbosity to 'debug'
-# it can be one of:
-# debug (a lot of information, useful for development/testing)
-# notice (moderately verbose, what you want in production probably)
-# warning (only very important / critical messages are logged)
-loglevel debug
-
-# Specify the log file name. Also 'stdout' can be used to force
-# the demon to log on the standard output. Note that if you use standard
-# output for logging but daemonize, logs will be sent to /dev/null
-logfile stdout
-
-# Set the number of databases. The default database is DB 0, you can select
-# a different one on a per-connection basis using SELECT <dbid> where
-# dbid is a number between 0 and 'databases'-1
-databases 16
-
-################################ SNAPSHOTTING  #################################
-#
-# Save the DB on disk:
-#
-#   save <seconds> <changes>
-#
-#   Will save the DB if both the given number of seconds and the given
-#   number of write operations against the DB occurred.
-#
-#   In the example below the behaviour will be to save:
-#   after 900 sec (15 min) if at least 1 key changed
-#   after 300 sec (5 min) if at least 10 keys changed
-#   after 60 sec if at least 10000 keys changed
-save 900 1
-save 300 10
-save 60 10000
-
-# Compress string objects using LZF when dump .rdb databases?
-# For default that's set to 'yes' as it's almost always a win.
-# If you want to save some CPU in the saving child set it to 'no' but
-# the dataset will likely be bigger if you have compressible values or keys.
-rdbcompression yes
-
-# The filename where to dump the DB
-dbfilename dump.rdb
-
-# For default save/load DB in/from the working directory
-# Note that you must specify a directory not a file name.
-dir ./tmp/master/
-
-################################# REPLICATION #################################
-
-# Master-Slave replication. Use slaveof to make a Redis instance a copy of
-# another Redis server. Note that the configuration is local to the slave
-# so for example it is possible to configure the slave to save the DB with a
-# different interval, or to listen to another port, and so on.
-#
-# slaveof <masterip> <masterport>
-
-# If the master is password protected (using the "requirepass" configuration
-# directive below) it is possible to tell the slave to authenticate before
-# starting the replication synchronization process, otherwise the master will
-# refuse the slave request.
-#
-# masterauth <master-password>
-
-################################## SECURITY ###################################
-
-# Require clients to issue AUTH <PASSWORD> before processing any other
-# commands.  This might be useful in environments in which you do not trust
-# others with access to the host running redis-server.
-#
-# This should stay commented out for backward compatibility and because most
-# people do not need auth (e.g. they run their own servers).
-#
-# requirepass foobared
-
-################################### LIMITS ####################################
-
-# Set the max number of connected clients at the same time. By default there
-# is no limit, and it's up to the number of file descriptors the Redis process
-# is able to open. The special value '0' means no limts.
-# Once the limit is reached Redis will close all the new connections sending
-# an error 'max number of clients reached'.
-#
-# maxclients 128
-
-# Don't use more memory than the specified amount of bytes.
-# When the memory limit is reached Redis will try to remove keys with an
-# EXPIRE set. It will try to start freeing keys that are going to expire
-# in little time and preserve keys with a longer time to live.
-# Redis will also try to remove objects from free lists if possible.
-#
-# If all this fails, Redis will start to reply with errors to commands
-# that will use more memory, like SET, LPUSH, and so on, and will continue
-# to reply to most read-only commands like GET.
-#
-# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
-# 'state' server or cache, not as a real DB. When Redis is used as a real
-# database the memory usage will grow over the weeks, it will be obvious if
-# it is going to use too much memory in the long run, and you'll have the time
-# to upgrade. With maxmemory after the limit is reached you'll start to get
-# errors for write operations, and this may even lead to DB inconsistency.
-#
-# maxmemory <bytes>
-
-############################## APPEND ONLY MODE ###############################
-
-# By default Redis asynchronously dumps the dataset on disk. If you can live
-# with the idea that the latest records will be lost if something like a crash
-# happens this is the preferred way to run Redis. If instead you care a lot
-# about your data and don't want to that a single record can get lost you should
-# enable the append only mode: when this mode is enabled Redis will append
-# every write operation received in the file appendonly.log. This file will
-# be read on startup in order to rebuild the full dataset in memory.
-#
-# Note that you can have both the async dumps and the append only file if you
-# like (you have to comment the "save" statements above to disable the dumps).
-# Still if append only mode is enabled Redis will load the data from the
-# log file at startup ignoring the dump.rdb file.
-#
-# The name of the append only file is "appendonly.log"
-#
-# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append
-# log file in background when it gets too big.
-
-appendonly yes
-
-# The fsync() call tells the Operating System to actually write data on disk
-# instead to wait for more data in the output buffer. Some OS will really flush 
-# data on disk, some other OS will just try to do it ASAP.
-#
-# Redis supports three different modes:
-#
-# no: don't fsync, just let the OS flush the data when it wants. Faster.
-# always: fsync after every write to the append only log . Slow, Safest.
-# everysec: fsync only if one second passed since the last fsync. Compromise.
-#
-# The default is "always" that's the safer of the options. It's up to you to
-# understand if you can relax this to "everysec" that will fsync every second
-# or to "no" that will let the operating system flush the output buffer when
-# it want, for better performances (but if you can live with the idea of
-# some data loss consider the default persistence mode that's snapshotting).
-
-# appendfsync always
-appendfsync everysec
-# appendfsync no
-
-############################### ADVANCED CONFIG ###############################
-
-# Glue small output buffers together in order to send small replies in a
-# single TCP packet. Uses a bit more CPU but most of the times it is a win
-# in terms of number of queries per second. Use 'yes' if unsure.
-glueoutputbuf yes
-
-# Use object sharing. Can save a lot of memory if you have many common
-# string in your dataset, but performs lookups against the shared objects
-# pool so it uses more CPU and can be a bit slower. Usually it's a good
-# idea.
-#
-# When object sharing is enabled (shareobjects yes) you can use
-# shareobjectspoolsize to control the size of the pool used in order to try
-# object sharing. A bigger pool size will lead to better sharing capabilities.
-# In general you want this value to be at least the double of the number of
-# very common strings you have in your dataset.
-#
-# WARNING: object sharing is experimental, don't enable this feature
-# in production before of Redis 1.0-stable. Still please try this feature in
-# your development environment so that we can test it better.
-shareobjects no
-shareobjectspoolsize 1024

data/etc/redis-slave.conf

@@ -1,189 +0,0 @@
-# Redis configuration file example
-
-# By default Redis does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
-daemonize no
-
-# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
-# You can specify a custom pid file location here.
-pidfile ./tmp/redis-slave.pid
-
-# Accept connections on the specified port, default is 6379
-port 6380
-
-# If you want you can bind a single interface, if the bind option is not
-# specified all the interfaces will listen for connections.
-#
-# bind 127.0.0.1
-
-# Close the connection after a client is idle for N seconds (0 to disable)
-timeout 300
-
-# Set server verbosity to 'debug'
-# it can be one of:
-# debug (a lot of information, useful for development/testing)
-# notice (moderately verbose, what you want in production probably)
-# warning (only very important / critical messages are logged)
-loglevel debug
-
-# Specify the log file name. Also 'stdout' can be used to force
-# the demon to log on the standard output. Note that if you use standard
-# output for logging but daemonize, logs will be sent to /dev/null
-logfile stdout
-
-# Set the number of databases. The default database is DB 0, you can select
-# a different one on a per-connection basis using SELECT <dbid> where
-# dbid is a number between 0 and 'databases'-1
-databases 16
-
-################################ SNAPSHOTTING  #################################
-#
-# Save the DB on disk:
-#
-#   save <seconds> <changes>
-#
-#   Will save the DB if both the given number of seconds and the given
-#   number of write operations against the DB occurred.
-#
-#   In the example below the behaviour will be to save:
-#   after 900 sec (15 min) if at least 1 key changed
-#   after 300 sec (5 min) if at least 10 keys changed
-#   after 60 sec if at least 10000 keys changed
-save 900 1
-save 300 10
-save 60 10000
-
-# Compress string objects using LZF when dump .rdb databases?
-# For default that's set to 'yes' as it's almost always a win.
-# If you want to save some CPU in the saving child set it to 'no' but
-# the dataset will likely be bigger if you have compressible values or keys.
-rdbcompression yes
-
-# The filename where to dump the DB
-dbfilename dump.rdb
-
-# For default save/load DB in/from the working directory
-# Note that you must specify a directory not a file name.
-dir ./tmp/slave/
-
-################################# REPLICATION #################################
-
-# Master-Slave replication. Use slaveof to make a Redis instance a copy of
-# another Redis server. Note that the configuration is local to the slave
-# so for example it is possible to configure the slave to save the DB with a
-# different interval, or to listen to another port, and so on.
-#
-slaveof 127.0.0.1 6379
-
-# If the master is password protected (using the "requirepass" configuration
-# directive below) it is possible to tell the slave to authenticate before
-# starting the replication synchronization process, otherwise the master will
-# refuse the slave request.
-#
-# masterauth <master-password>
-
-################################## SECURITY ###################################
-
-# Require clients to issue AUTH <PASSWORD> before processing any other
-# commands.  This might be useful in environments in which you do not trust
-# others with access to the host running redis-server.
-#
-# This should stay commented out for backward compatibility and because most
-# people do not need auth (e.g. they run their own servers).
-#
-# requirepass foobared
-
-################################### LIMITS ####################################
-
-# Set the max number of connected clients at the same time. By default there
-# is no limit, and it's up to the number of file descriptors the Redis process
-# is able to open. The special value '0' means no limts.
-# Once the limit is reached Redis will close all the new connections sending
-# an error 'max number of clients reached'.
-#
-# maxclients 128
-
-# Don't use more memory than the specified amount of bytes.
-# When the memory limit is reached Redis will try to remove keys with an
-# EXPIRE set. It will try to start freeing keys that are going to expire
-# in little time and preserve keys with a longer time to live.
-# Redis will also try to remove objects from free lists if possible.
-#
-# If all this fails, Redis will start to reply with errors to commands
-# that will use more memory, like SET, LPUSH, and so on, and will continue
-# to reply to most read-only commands like GET.
-#
-# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
-# 'state' server or cache, not as a real DB. When Redis is used as a real
-# database the memory usage will grow over the weeks, it will be obvious if
-# it is going to use too much memory in the long run, and you'll have the time
-# to upgrade. With maxmemory after the limit is reached you'll start to get
-# errors for write operations, and this may even lead to DB inconsistency.
-#
-# maxmemory <bytes>
-
-############################## APPEND ONLY MODE ###############################
-
-# By default Redis asynchronously dumps the dataset on disk. If you can live
-# with the idea that the latest records will be lost if something like a crash
-# happens this is the preferred way to run Redis. If instead you care a lot
-# about your data and don't want to that a single record can get lost you should
-# enable the append only mode: when this mode is enabled Redis will append
-# every write operation received in the file appendonly.log. This file will
-# be read on startup in order to rebuild the full dataset in memory.
-#
-# Note that you can have both the async dumps and the append only file if you
-# like (you have to comment the "save" statements above to disable the dumps).
-# Still if append only mode is enabled Redis will load the data from the
-# log file at startup ignoring the dump.rdb file.
-#
-# The name of the append only file is "appendonly.log"
-#
-# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append
-# log file in background when it gets too big.
-
-appendonly yes
-
-# The fsync() call tells the Operating System to actually write data on disk
-# instead to wait for more data in the output buffer. Some OS will really flush 
-# data on disk, some other OS will just try to do it ASAP.
-#
-# Redis supports three different modes:
-#
-# no: don't fsync, just let the OS flush the data when it wants. Faster.
-# always: fsync after every write to the append only log . Slow, Safest.
-# everysec: fsync only if one second passed since the last fsync. Compromise.
-#
-# The default is "always" that's the safer of the options. It's up to you to
-# understand if you can relax this to "everysec" that will fsync every second
-# or to "no" that will let the operating system flush the output buffer when
-# it want, for better performances (but if you can live with the idea of
-# some data loss consider the default persistence mode that's snapshotting).
-
-# appendfsync always
-appendfsync everysec
-# appendfsync no
-
-############################### ADVANCED CONFIG ###############################
-
-# Glue small output buffers together in order to send small replies in a
-# single TCP packet. Uses a bit more CPU but most of the times it is a win
-# in terms of number of queries per second. Use 'yes' if unsure.
-glueoutputbuf yes
-
-# Use object sharing. Can save a lot of memory if you have many common
-# string in your dataset, but performs lookups against the shared objects
-# pool so it uses more CPU and can be a bit slower. Usually it's a good
-# idea.
-#
-# When object sharing is enabled (shareobjects yes) you can use
-# shareobjectspoolsize to control the size of the pool used in order to try
-# object sharing. A bigger pool size will lead to better sharing capabilities.
-# In general you want this value to be at least the double of the number of
-# very common strings you have in your dataset.
-#
-# WARNING: object sharing is experimental, don't enable this feature
-# in production before of Redis 1.0-stable. Still please try this feature in
-# your development environment so that we can test it better.
-shareobjects no
-shareobjectspoolsize 1024

data/examples/redis_failover.rb

@@ -1,65 +0,0 @@
-# Testing redis failover functionality
-require "rubygems"
-require File.expand_path(File.dirname(__FILE__)+"/../lib/beetle")
-
-Beetle.config.logger.level = Logger::INFO
-Beetle.config.redis_hosts = "localhost:6379, localhost:6380"
-Beetle.config.servers = "localhost:5672, localhost:5673"
-
-# instantiate a client
-client = Beetle::Client.new
-
-# register a durable queue named 'test'
-# this implicitly registers a durable topic exchange called 'test'
-client.register_queue(:test)
-client.purge(:test)
-client.register_message(:test, :redundant => true)
-
-# publish some test messages
-# at this point, the exchange will be created on the server and the queue will be bound to the exchange
-N = 10
-n = 0
-N.times do |i|
-  n += client.publish(:test, "Hello#{i+1}")
-end
-puts "published #{n} test messages"
-puts
-
-# check whether we were able to publish all messages
-if n != 2*N
-  puts "could not publish all messages"
-  exit 1
-end
-
-# register a handler for the test message, listing on queue "test"
-k = 0
-client.register_handler(:test) do |m|
-  k += 1
-  puts "Received test message from server #{m.server}"
-  puts "Message content: #{m.data}"
-  puts
-  sleep 1
-end
-
-# hack to switch redis programmatically
-class Beetle::DeduplicationStore
-  def switch_redis
-    slave = redis_instances.find{|r| r.server != redis.server}
-    redis.shutdown rescue nil
-    logger.info "Beetle: shut down master #{redis.server}"
-    slave.slaveof("no one")
-    logger.info "Beetle: enabled master mode on #{slave.server}"
-  end
-end
-
-# start listening
-# this starts the event machine loop using EM.run
-# the block passed to listen will be yielded as the last step of the setup process
-client.listen do
-  trap("INT") { client.stop_listening }
-  EM.add_timer(5) { client.deduplication_store.switch_redis }
-  EM.add_timer(11) { client.stop_listening }
-end
-
-puts "Received #{k} test messages"
-raise "Your setup is borked" if N != k

data/script/start_rabbit

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# export RABBITMQ_MNESIA_BASE=/var/lib/rabbitmq/mnesia2
-# Defaults to /var/lib/rabbitmq/mnesia. Set this to the directory where Mnesia
-# database files should be placed.
-
-# export RABBITMQ_LOG_BASE
-# Defaults to /var/log/rabbitmq. Log files generated by the server will be placed
-# in this directory.
-
-export RABBITMQ_NODENAME=$1
-# Defaults to rabbit. This can be useful if you want to run more than one node
-# per machine - RABBITMQ_NODENAME should be unique per erlang-node-and-machine
-# combination. See clustering on a single machine guide at <http://www.rab-
-# bitmq.com/clustering.html#single-machine> for details.
-
-# RABBITMQ_NODE_IP_ADDRESS
-# Defaults to 0.0.0.0. This can be changed if you only want to bind to one net-
-# work interface.
-
-export RABBITMQ_NODE_PORT=$2
-# Defaults to 5672.
-
-# RABBITMQ_CLUSTER_CONFIG_FILE
-# Defaults to /etc/rabbitmq/rabbitmq_cluster.config. If this file is present it
-# is used by the server to auto-configure a RabbitMQ cluster.  See the clustering
-# guide at <http://www.rabbitmq.com/clustering.html> for details.
-
-rabbitmq-server

data/snafu.rb

@@ -1,55 +0,0 @@
-# The simplest case
-client.register_message(:something_happened)                  # => key: something_happened
-
-# with options
-client.register_message(:
-
-####################
-# Message Grouping #
-####################
-
-client.register_message(:delete_something, :group => :jobs)   # => key: jobs.delete_something
-client.register_message(:create_something, :group => :jobs)   # => key: jobs.create_something
-
-# You can register a handler for a message group
-client.register_handler(JobsHandler, :group => :jobs)         # bind queue with: jobs.*
-
-# And still register on single messages
-client.register_handler(DeletedJobHandler, :delete_something) # bind queue with: *.delete_something
-
-######################
-# Handler Definition #
-######################
-
-# With a Handler class that implements .process(message)
-client.register_handler(MyProcessor, :something_happened)                    # => queue: my_processor
-
-# With a String / Symbol and a block
-client.register_handler("Other Processor", :delete_something, :something_happened) lambda { |message| foobar(message) } # => queue: other_processor, bound with: *.delete_something and *.something_happened
-
-# With extra parameters
-client.register_handler(VeryImportant, :delete_something, :immediate => true) # queue: very_important, :immediate => true
-
-###################################
-# Wiring, Subscribing, Publishing #
-###################################
-client.wire! # => all the binding magic happens
-
-client.subscribe
-
-client.publish(:delete_something, 'payload')
-
-__END__
-
-Whats happening when wire! is called? (pseudocode)
-1. all the messages are registered
-     messages = [{:name => :delete_something, :group => :jobs, :bound => false}, {:name => :something_happened, :bound => false}]
-2. all the queues for the handlers are created and bound...
-     my_processor_queue = queue(:my_processor).bind(exchange, :key => '*.something_happened')
-     jobs_handler_queue = queue(:jobs_handler).bind(exchange, :key => 'jobs.*')
-     handlers_with_queues = [[jobs_handler_queue, JobsHandler], [my_processor_queue, block_or_class]]
-3. every handler definition binds a queue for the handler to a list of messages and marks the message as bound.
-4. If in the end a message isn't bound to a queue at least once, an exception is raised
-
-Exceptions will be thrown if:
-* after all m

data/test/beetle.yml

@@ -1,81 +0,0 @@
-# list all standard exchanges used by the main xing app, along with their options for declaration
-# used by producers and consumers
-exchanges:
-  test:
-    type: "topic"
-    durable: true
-  deadletter:
-    type: "topic"
-    durable: true
-  redundant:
-    type: "topic"
-    durable: true
-
-# list all standard queues along with their binding declaration
-# this section is only used by consumers
-queues:
-  test:                  # binding options
-    exchange: "test"     # Bandersnatch default is the name of the queue
-    passive: false       # amqp default is false
-    durable: true        # amqp default is false
-    exclusive: false     # amqp default is false
-    auto_delete: false   # amqp default is false
-    nowait: true         # amqp default is true
-    key: "#"             # listen to every message
-  deadletter:
-    exchange: "deadletter"
-    durable: true
-    key: "#"
-  redundant:
-    exchange: "redundant"
-    durable: true
-    key: "#"
-  additional_queue:
-    exchange: "redundant"
-    durable: true
-    key: "#"
-
-# list all messages we can publish
-messages:
-  test:
-    queue: "test"
-    # Spefify the queue for listeners (default is message name)
-    key: "test"
-    # Specifies the routing key pattern for message subscription.
-    ttl: <%= 1.hour %>
-    # Specifies the time interval after which messages are silently dropped (seconds)
-    mandatory: true
-    # default is false
-    # Tells the server how to react if the message
-    # cannot be routed to a queue. If set to _true_, the server will return an unroutable message
-    # with a Return method. If this flag is zero, the server silently drops the message.
-    immediate: false
-    # default is false
-    # Tells the server how to react if the message
-    # cannot be routed to a queue consumer immediately. If set to _true_, the server will return an
-    # undeliverable message with a Return method. If set to _false_, the server will queue the message,
-    # but with no guarantee that it will ever be consumed.
-    persistent: true
-    # default is false
-    # Tells the server whether to persist the message
-    # If set to _true_, the message will be persisted to disk and not lost if the server restarts.
-    # If set to _false_, the message will not be persisted across server restart. Setting to _true_
-    # incurs a performance penalty as there is an extra cost associated with disk access.
-  deadletter:
-    key: "deadletter"
-    persistent: true
-  redundant:
-    key: "redundant"
-    persistent: true
-    redundant: true
-
-development: &development
-    hostname: localhost:5672, localhost:5673
-    # hostname: localhost:5672
-    msg_id_store:
-      host: localhost
-      db: 4
-
-test:
-    <<: *development
-    hostname: localhost:5672

data/tmp/master/.gitignore

@@ -1,2 +0,0 @@
-*.rdb
-appendonly.aof

data/tmp/slave/.gitignore

@@ -1,3 +0,0 @@
-*.rdb
-appendonly.aof
-