flamingo 0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Hayes Davis
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,73 @@
+Flamingo
+========
+Flamingo is a resque-based system for handling the Twitter Streaming API.
+
+This is *early alpha* code. There will be a lot of change and things like tests 
+coming in the future. That said, it does work so give it a try if you have the 
+need.
+
+Dependencies
+------------
+* redis
+* resque
+* sinatra
+* twitter-stream
+* yajl-ruby
+
+By default, the `resque` gem installs the latest 2.x `redis` gem, so if
+you are using Redis 1.x, you may want to swap it out.
+
+    $ gem list | grep redis
+    redis (2.0.3)
+    $ gem remove redis --version=2.0.3 -V
+
+    $ gem install redis --version=1.0.7
+    $ gem list | grep redis
+    redis (1.0.7)
+
+Getting Started
+---------------
+1. Install the gem
+        sudo gem install flamingo
+
+2. Create a config file (see `examples/flamingo.yml`) with at least a username and password
+
+        username: USERNAME
+        password: PASSWORD
+        stream: filter
+        logging:
+          dest: /YOUR/LOG/PATH.LOG
+          level: LOGLEVEL
+
+    `LOGLEVEL` is one of the following:
+    `DEBUG` < `INFO` < `WARN` < `ERROR` < `FATAL` < `UNKNOWN`
+
+3. Start the Redis server
+
+        $ redis-server
+
+4. Configure tracking using `flamingo` client (installed during `gem install`)
+
+        $ flamingo
+        >> s = Stream.get(:filter)
+        >> s.params[:track] = %w(FOO BAR BAZ)
+        >> Subscription.new('YOUR_QUEUE').save
+
+5. Start the Flamingo Daemon (`flamingod` installed during `gem install`)
+
+        $ flamingod -c your/config/file.yml
+
+        
+6. Consume events with a resque worker
+
+        class HandleFlamingoEvent
+          
+          # type: One of "tweet" or "delete"
+          # event: a hash of the json data from twitter
+          def self.perform(type,event)
+            # Do stuff with the data
+          end
+          
+        end
+        
+        $ QUEUE=YOUR_QUEUE rake resque:work

data/Rakefile

@@ -0,0 +1,3 @@
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__),'lib')
+require 'flamingo'
+require 'resque/tasks'

data/bin/flamingo

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'flamingo'
+
+include Flamingo
+
+puts "Flamingo client #{Flamingo::VERSION}"
+
+begin
+  Flamingo.configure!(ARGV[0])
+rescue => e
+  $stderr.puts "Could not start: #{e.message}"
+  exit(-1)
+end
+
+ARGV.clear # Remove args so IRB doesn't try to load them
+
+require 'irb'
+IRB.start

data/bin/flamingo-web

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'flamingo'
+
+Flamingo.configure!(ARGV[0])
+host,port = Flamingo.config.web.host('0.0.0.0:4711').split(":")
+Flamingo::Web::Server.run! :host=>host, :port=>port.to_i

data/bin/flamingod

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'flamingo'
+require 'optparse'
+
+puts "Starting flamingod #{Flamingo::VERSION}"
+
+begin
+  options = {:daemonize => nil, :config => nil}
+
+  opts = OptionParser.new do |opts|
+    opts.banner = <<-EOF
+  Usage:
+    flamingod [<options>]
+  EOF
+
+    opts.on("-cCONFIG", "--config-file=CONFIG", "Configuration File") do |x|
+      options[:config_file] = x
+    end
+
+    opts.on("-d", "--daemonize", "Run in Background (Daemonize)") do |x|
+      options[:daemonize] = true
+    end
+
+    opts.on("-b", "--background", "Run in Background (Daemonize)") do |x|
+      options[:daemonize] = true
+    end
+
+    opts.on("-f", "--foreground", "Run in Foreground (default)") do |x|
+      options[:daemonize] = nil
+    end
+
+    opts.on("-pPID_FILE", "--pid-file=PID_FILE", "PID File (only when daemonized)") do |x|
+      options[:pid_file] = x
+    end
+
+  end
+
+  opts.parse!
+
+  Flamingo.configure!(options[:config_file])
+  if options[:pid_file]
+    Flamingo.config.pid_file = File.expand_path(options[:pid_file])
+  end
+
+rescue => e
+  $stderr.puts "Could not start: #{e.message}"
+  exit(-1)
+end
+
+flamingod = Flamingo::Daemon::Flamingod.new 
+if options[:daemonize]
+  begin
+    pid = flamingod.run_as_daemon()
+    puts "flamingod process #{pid} started" 
+  rescue => e
+    $stderr.puts "Could not start: #{e.message}"
+    exit(-2)
+  end
+else
+  flamingod.run()
+end

data/examples/Rakefile

@@ -0,0 +1,15 @@
+# Simple example that reads from a subscription queue and writes the events 
+# to STDOUT
+# Usage (from this directory):
+#   $ QUEUE=YOUR_QUEUE rake resque:work
+
+require 'rubygems'
+require 'resque/tasks'
+
+class HandleFlamingoEvent
+  
+  def self.perform(type,event)
+    puts type, event
+  end
+  
+end

data/examples/flamingo.yml

@@ -0,0 +1,10 @@
+username: SCREEN_NAME
+password: PASSWORD
+stream: filter
+logging:
+  dest: /your/log/file/here.log
+  level: INFO
+redis:
+  host: 0.0.0.0:6379
+web:
+  host: 0.0.0.0:4711

data/lib/flamingo.rb

@@ -0,0 +1,151 @@
+require 'rubygems'
+require 'redis/namespace'
+require 'twitter/json_stream'
+require 'resque'
+require 'logger'
+require 'yaml'
+require 'erb'
+require 'cgi'
+require 'active_support'
+require 'sinatra/base'
+
+require 'flamingo/version'
+require 'flamingo/config'
+require 'flamingo/dispatch_event'
+require 'flamingo/dispatch_error'
+require 'flamingo/stream_params'
+require 'flamingo/stream'
+require 'flamingo/subscription'
+require 'flamingo/wader'
+require 'flamingo/daemon/pid_file'
+require 'flamingo/daemon/child_process'
+require 'flamingo/daemon/dispatcher_process'
+require 'flamingo/daemon/web_server_process'
+require 'flamingo/daemon/wader_process'
+require 'flamingo/daemon/flamingod'
+require 'flamingo/logging/formatter'
+require 'flamingo/web/server'
+
+module Flamingo
+  
+  class << self
+    
+    def configure!(config_file=nil)
+      config_file = find_config_file(config_file)
+      @config = Flamingo::Config.load(config_file)
+      validate_config!
+      logger.info "Loaded config file from #{config_file}"
+    end
+    
+    def config
+      @config
+    end
+    
+    # PHD: Lovingly borrowed from Resque
+
+    # Accepts:
+    #   1. A 'hostname:port' string
+    #   2. A 'hostname:port:db' string (to select the Redis db)
+    #   3. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`,
+    #      or `Redis::Namespace`.
+    def redis=(server)
+      case server
+      when String
+        host, port, db = server.split(':')
+        redis = Redis.new(:host => host, :port => port,
+          :thread_safe => true, :db => db)
+        @redis = Redis::Namespace.new(:flamingo, :redis => redis)
+      when Redis, Redis::Client, Redis::DistRedis
+        @redis = Redis::Namespace.new(:flamingo, :redis => server)
+      when Redis::Namespace
+        @redis = server
+      else
+        raise "Invalid redis configuration: #{server.inspect}"
+      end
+    end
+  
+    # Returns the current Redis connection. If none has been created, will
+    # create a new one.
+    def redis
+      return @redis if @redis
+      self.redis = config.redis.host('localhost:6379')
+      self.redis
+    end
+
+    def new_logger
+      # determine log file location (default is root_dir/log/flamingo.log)
+      if valid_logging_dest?(config.logging.dest(nil))
+        log_dest = config.logging.dest
+      else
+        log_dest = File.join(root_dir,'log','flamingo.log')
+      end
+
+      # determine logging level (default is Logger::INFO)
+      begin
+        log_level = Logger.const_get(config.logging.level.upcase)
+      rescue
+        log_level = Logger::INFO
+      end
+
+      # create logger facility
+      logger = Logger.new(log_dest)
+      logger.level = log_level
+      logger.formatter = Flamingo::Logging::Formatter.new
+      logger
+    end
+
+    def logger
+      @logger ||= new_logger
+    end
+    
+    private
+      def root_dir
+        File.expand_path(File.dirname(__FILE__)+'/..')
+      end
+    
+      def new_logger
+        dest = config.logging.dest(nil)
+        if valid_logging_dest?(dest)
+          log_file = dest
+        else
+          log_file = File.join(root_dir,'log','flamingo.log')
+        end
+  
+        # determine logging level (default is Logger::INFO)
+        begin
+          log_level = Logger.const_get(config.logging.level('INFO').upcase)
+        rescue
+          log_level = Logger::INFO
+        end
+  
+        # create logger facility
+        logger = Logger.new(log_file)
+        logger.level = log_level
+        logger.formatter = Flamingo::Logging::Formatter.new
+        logger
+      end
+      
+      def valid_logging_dest?(dest)
+        return false unless dest
+        File.writable?(File.dirname(dest))
+      end
+
+      def validate_config!
+        unless config.username(nil) && config.password(nil)
+          raise "The config file must be YAML formatted and contain a username and password. See examples/flamingo.yml."
+        end
+      end
+      
+      def find_config_file(config_file=nil)
+        locations = [config_file,"./flamingo.yml","~/flamingo.yml"].compact.uniq
+        found = locations.find do |file|
+          file && File.exist?(file)
+        end
+        unless found
+          raise "No config file found in any of #{locations.join(",")}"
+        end
+        File.expand_path(found)
+      end
+
+  end 
+end

data/lib/flamingo/config.rb

@@ -0,0 +1,52 @@
+module Flamingo
+  class Config
+
+    def self.load(file)
+      new(YAML.load(IO.read(file)))
+    end
+  
+    def initialize(hash={})
+      @data = hash
+    end
+    
+    def method_missing(name,*args,&block)
+      if name.to_s =~ /(.+)=$/
+        @data[$1] = *args
+      else
+        value = @data[name.to_s]
+        if value.is_a?(Hash)
+          self.class.new(value)
+        elsif value.nil? || empty_config?(value)
+          if !args.empty?
+            # Return a default if the value isn't set and there's an argument
+            args.length == 1 ? args[0] : args
+          elsif block_given?
+            # Run the block to get the default value
+            yield
+          else
+            # Return back a config object
+            value = self.class.new
+            @data[name.to_s] = value
+            value
+          end
+        else
+          value
+        end
+      end
+    end
+    
+    def respond_to?(name)
+      true
+    end
+    
+    def empty?
+      @data.empty?
+    end
+    
+    private
+      def empty_config?(value)
+        value.is_a?(self.class) && value.empty?
+      end
+
+  end 
+end

data/lib/flamingo/daemon/child_process.rb

@@ -0,0 +1,16 @@
+module Flamingo
+  module Daemon
+    class ChildProcess
+      attr_accessor :pid
+  
+      def kill(sig)
+        Process.kill(sig,pid)
+      end
+      alias_method :signal, :kill
+      
+      def start
+        self.pid = fork { run }
+      end      
+    end
+  end
+end

data/lib/flamingo/daemon/dispatcher_process.rb

@@ -0,0 +1,15 @@
+module Flamingo
+  module Daemon
+    class DispatcherProcess < ChildProcess
+      def run
+        worker = Resque::Worker.new(:flamingo)
+        def worker.procline(value)
+          # Hack to get around resque insisting on setting the proces name
+          $0 = "flamingod-dispatcher"
+        end
+        Flamingo.logger.info "Starting dispatcher on pid=#{Process.pid} under pid=#{Process.ppid}"
+        worker.work(1) # Wait 1s between jobs
+      end
+    end    
+  end
+end

data/lib/flamingo/daemon/flamingod.rb

@@ -0,0 +1,113 @@
+module Flamingo
+  module Daemon
+    class Flamingod
+
+      def exit_signaled?
+        @exit_signaled
+      end
+
+      def exit_signaled=(val)
+        Flamingo.logger.info "Exit signal set to #{val}"
+        @exit_signaled = val
+      end
+
+      def start_new_wader
+        Flamingo.logger.info "Flamingod starting new wader"
+        wader = WaderProcess.new
+        wader.start
+        wader
+      end
+
+      def start_new_dispatcher
+        Flamingo.logger.info "Flamingod starting new dispatcher"
+        dispatcher = DispatcherProcess.new
+        dispatcher.start
+        dispatcher
+      end
+
+      def start_new_web_server
+        Flamingo.logger.info "Flamingod starting new web server"
+        ws = WebServerProcess.new
+        ws.start
+        ws
+      end
+
+      def trap_signals
+        trap("KILL") { terminate! }
+        trap("TERM") { terminate! }
+        trap("INT")  { terminate! }
+        trap("USR1") { restart_wader }
+      end
+
+      def restart_wader
+        Flamingo.logger.info "Flamingod restarting wader pid=#{@wader.pid} with SIGINT"
+        @wader.kill("INT")
+      end
+
+      def signal_children(sig)
+        pids = (children.map {|c| c.pid}).join(",")
+        Flamingo.logger.info "Flamingod sending SIG#{sig} to pids=#{pids}"
+        children.each {|child| child.signal(sig) }
+      end
+
+      def terminate!
+        Flamingo.logger.info "Flamingod terminating"
+        self.exit_signaled = true
+        signal_children("INT")
+      end
+
+      def children
+        [@wader,@web_server] + @dispatchers
+      end
+
+      def start_children
+        Flamingo.logger.info "Flamingod starting children"
+        @wader = start_new_wader
+        @dispatchers = [start_new_dispatcher]
+        @web_server = start_new_web_server
+      end
+
+      def wait_on_children()
+        until exit_signaled?
+          child_pid = Process.wait(-1)
+          unless exit_signaled?
+            if @wader.pid == child_pid
+              @wader = start_new_wader
+            elsif @web_server.pid == child_pid
+              @web_server = start_new_web_server
+            elsif (to_delete = @dispatchers.find{|d| d.pid == child_pid})
+              @dispatchers.delete(to_delete)
+              @dispatchers << start_new_dispatcher
+            else
+              Flamingo.logger.info "Received exit from unknown child #{child_pid}"
+            end
+          end
+        end
+      end
+
+      def run_as_daemon
+        pid_file = PidFile.new
+        if pid_file.running?
+          raise "flamingod process #{pid_file.read} appears to be running"
+        end
+        pid = fork do
+          pid_file.write(Process.pid)
+          [$stdout,$stdin,$stderr].each do |io|
+            io.reopen '/dev/null' rescue nil
+          end
+          run
+          pid_file.delete
+        end
+        Process.detach(pid)
+        pid
+      end
+
+      def run
+        $0 = 'flamingod'
+        trap_signals
+        start_children
+        wait_on_children
+      end
+    end
+  end
+end

data/lib/flamingo/daemon/pid_file.rb

@@ -0,0 +1,48 @@
+module Flamingo
+  module Daemon
+    class PidFile
+      
+      def read
+        File.read(file).strip rescue nil
+      end
+      
+      def exists?
+        File.exist?(file) rescue false
+      end
+      
+      def running?
+        #PHD The code below borrowed from the daemons gem
+        return false unless exists?
+        # Check if process is in existence
+        # The simplest way to do this is to send signal '0'
+        # (which is a single system call) that doesn't actually
+        # send a signal
+        begin
+          Process.kill(0, pid)
+          return true
+        rescue Errno::ESRCH
+          return false
+        rescue ::Exception   # for example on EPERM (process exists but does not belong to us)
+          return true
+        end
+      end
+      
+      def delete
+        File.delete(file) if file
+      end
+      
+      def write(pid)
+        File.open(file,"w") {|f| f.write("#{pid}\n") }
+        true
+      rescue 
+        false
+      end
+      
+      private
+        def file
+          Flamingo.config.pid_file(nil)
+        end
+      
+    end
+  end
+end

data/lib/flamingo/daemon/wader_process.rb

@@ -0,0 +1,28 @@
+module Flamingo
+  module Daemon
+    class WaderProcess < ChildProcess
+      def register_signal_handlers
+        trap("INT")  { stop }
+      end
+      
+      def run
+        register_signal_handlers
+        $0 = 'flamingod-wader'
+        config = Flamingo.config
+        
+        screen_name = config.username
+        password    = config.password
+        stream      = Stream.get(config.stream)
+
+        @wader = Flamingo::Wader.new(screen_name,password,stream)
+        Flamingo.logger.info "Starting wader on pid=#{Process.pid} under pid=#{Process.ppid}"
+        @wader.run
+        Flamingo.logger.info "Wader pid=#{Process.pid} stopped"
+      end
+      
+      def stop
+        @wader.stop
+      end
+    end
+  end
+end

data/lib/flamingo/daemon/web_server_process.rb

@@ -0,0 +1,12 @@
+module Flamingo
+  module Daemon
+    class WebServerProcess < ChildProcess
+      def run
+        $0 = 'flamingod-web'
+        host, port = Flamingo.config.web.host('0.0.0.0:4711').split(":")
+        Flamingo::Web::Server.run! :host=>host, :port=>port.to_i, 
+          :daemon_pid=>Process.ppid
+      end
+    end
+  end  
+end

data/lib/flamingo/dispatch_error.rb

@@ -0,0 +1,11 @@
+module Flamingo
+  class DispatchError
+    
+    @queue = :flamingo
+    
+    def self.perform(type,message,data)
+      Flamingo.logger.info("#{type}, #{message}, #{data.inspect}\n")
+    end
+    
+  end
+end

data/lib/flamingo/dispatch_event.rb

@@ -0,0 +1,40 @@
+module Flamingo
+  class DispatchEvent
+    
+    @queue = :flamingo
+    @parser = Yajl::Parser.new(:symbolize_keys => true)
+    
+    class << self
+    
+      def perform(event_json)
+        #TODO Track stats including: tweets per second and last tweet time
+        #TODO Provide some first-level check for repeated status ids
+        #TODO Consider subscribers for receiving particular terms - do the heavy 
+        #     lifting of parsing tweets and delivering them to particular subscribers
+        #TODO Consider window of tweets (approx 3 seconds) and sort before 
+        #     dispatching to improve in-order delivery (helps with "k-sorted") 
+        type, event = typed_event(parse(event_json))
+#        Flamingo.logger.info Flamingo.router.destinations(type,event).inspect
+        Subscription.all.each do |sub|
+          Resque::Job.create(sub.name, "HandleFlamingoEvent", type, event)
+          Flamingo.logger.debug "Put job on subscription queue #{sub.name} for #{event_json}"
+        end
+      end
+      
+      def parse(json)
+        @parser.parse(json)
+      end
+      
+      def typed_event(event)
+        if event[:delete]
+          [:delete,event[:delete]]
+        elsif event[:link]
+          [:link,event[:link]]
+        else
+          [:tweet,event]
+        end
+      end
+      
+    end
+  end
+end

data/lib/flamingo/logging/formatter.rb

@@ -0,0 +1,12 @@
+module Flamingo
+  module Logging
+    class Formatter < Logger::Formatter
+      def call(severity, time, progname, msg)
+        entry = "\n[#{time.utc.strftime("%Y-%m-%d %H:%M:%S")}, #{severity}"
+        entry << ", #{progname}" if progname
+        entry << "] - #{msg2str(msg)}"
+        entry
+      end
+    end
+  end
+end

data/lib/flamingo/stream.rb

@@ -0,0 +1,62 @@
+module Flamingo
+  
+  class Stream
+    
+    VERSION = 1
+    
+    RESOURCES = HashWithIndifferentAccess.new(
+      :filter   => "statuses/filter",
+      :firehose => "statuses/firehose",
+      :retweet  => "statuses/retweet",
+      :sample   => "statuses/sample"
+    )
+
+    class << self
+      def get(name)
+        new(name,StreamParams.new(name))
+      end
+    end
+    
+    attr_accessor :name, :params
+    
+    def initialize(name,params)
+      self.name = name
+      self.params = params
+    end
+    
+    def connect(options)
+      conn_opts = {:ssl => true, :user_agent => "Flamingo/0.1" }.
+        merge(options).merge(:path=>path)
+      Twitter::JSONStream.connect(conn_opts)
+    end
+    
+    def path
+      "/#{VERSION}/#{resource}.json?#{query}"
+    end
+    
+    def resource
+      RESOURCES[name.to_sym]
+    end
+    
+    def to_json
+      ActiveSupport::JSON.encode(
+        :name=>name,:resource=>resource,:params=>params.all
+      )
+    end
+    
+    private
+      def query
+        params.map{|key,value| "#{key}=#{param_value(value)}" }.join("&")
+      end
+    
+      def param_value(val)
+        case val
+          when String then CGI.escape(val)
+          when Array then val.map{|v| CGI.escape(v) }.join(",")
+          else nil
+        end
+      end    
+    
+  end
+  
+end

data/lib/flamingo/stream_params.rb

@@ -0,0 +1,74 @@
+module Flamingo
+  
+  class StreamParams
+    
+    include Enumerable
+    
+    attr_accessor :stream_name
+    
+    def initialize(stream_name)
+      self.stream_name = stream_name
+    end
+    
+    def set(key,*values)
+      delete(key)
+      add(key,*values)
+    end
+    
+    def []=(key,values)
+      values = [values] unless values.is_a?(Array)
+      set(key,*values)
+    end
+    
+    def add(key,*values)
+      values.each do |value|
+        Flamingo.redis.sadd redis_key(key), value
+      end
+    end
+    
+    def remove(key,*values)
+      values.each do |value|
+        Flamingo.redis.srem redis_key(key), value
+      end
+    end
+    
+    def delete(key)
+      Flamingo.redis.del redis_key(key)
+    end
+    
+    def get(key)
+      Flamingo.redis.smembers redis_key(key)
+    end
+    alias_method :[], :get
+
+    def keys
+      Flamingo.redis.keys(redis_key_pattern).map do |key|
+        key.split("?")[1].to_sym
+      end
+    end
+
+    def all
+      keys.inject({}) do |h,key|
+        h[key] = get(key)
+        h
+      end
+    end
+    
+    def each
+      keys.each do |key|
+        yield(key,get(key))
+      end
+    end
+    
+    private
+      def redis_key_pattern
+        "streams/#{stream_name}?*"
+      end
+      
+      def redis_key(key)
+        "streams/#{stream_name}?#{key}"
+      end
+    
+  end
+  
+end

data/lib/flamingo/subscription.rb

@@ -0,0 +1,39 @@
+module Flamingo
+  
+  class Subscription
+    
+    class << self
+      
+      def all
+        Flamingo.redis.smembers("subscriptions").map do |name|
+          new(name)
+        end
+      end
+
+      def find(name)
+        if Flamingo.redis.sismember("subscriptions",name)
+          Subscription.new(name)
+        end
+      end
+      
+    end
+    
+    attr_accessor :name
+        
+    def initialize(name)
+      self.name = name  
+    end
+    
+    def save
+      Flamingo.logger.info("Adding #{name} to subscriptions")
+      Flamingo.redis.sadd("subscriptions",name)
+    end
+    
+    def delete
+      Flamingo.logger.info("Removing #{name} from subscriptions")
+      Flamingo.redis.srem("subscriptions",name)
+    end
+    
+  end
+  
+end

data/lib/flamingo/version.rb

@@ -0,0 +1,3 @@
+module Flamingo
+  Version = VERSION = '0.1'
+end

data/lib/flamingo/wader.rb

@@ -0,0 +1,57 @@
+module Flamingo
+  class Wader
+    
+    attr_accessor :screen_name, :password, :stream, :connection
+    
+    def initialize(screen_name,password,stream)
+      self.screen_name = screen_name
+      self.password = password
+      self.stream = stream
+    end
+    
+    def run
+      EventMachine::run do
+        self.connection = stream.connect(:auth=>"#{screen_name}:#{password}")
+        Flamingo.logger.info("Listening on stream: #{stream.path}")
+  
+        connection.each_item do |event_json|
+          dispatch_event(event_json)
+        end
+  
+        connection.on_error do |message|
+          dispatch_error(:generic,message)
+        end
+  
+        connection.on_reconnect do |timeout, retries|
+          dispatch_error(:reconnection,
+            "Will reconnect after #{timeout}. Retry \##{retries}",
+            {:timeout=>timeout,:retries=>retries}
+          )
+        end
+  
+        connection.on_max_reconnects do |timeout, retries|
+          dispatch_error(:fatal,
+            "Failed to reconnect after #{retries} retries",
+            {:timeout=>timeout,:retries=>retries}
+          )
+        end
+      end  
+    end
+   
+    def stop
+      connection.stop
+      EM.stop
+    end
+    
+    private
+      def dispatch_event(event_json)
+        Flamingo.logger.debug "Wader dispatched event"
+        Resque.enqueue(Flamingo::DispatchEvent,event_json)
+      end
+      
+      def dispatch_error(type,message,data={})
+        Flamingo.logger.error "Received error: #{message}"
+        Resque.enqueue(Flamingo::DispatchError,type,message,data)
+      end
+  end
+end

data/lib/flamingo/web/server.rb

@@ -0,0 +1,129 @@
+module Flamingo
+  module Web
+    class Server < Sinatra::Base
+      
+      set :root, File.expand_path(File.dirname(__FILE__))
+      set :static, true
+      set :logging, true
+  
+      get '/' do
+        content_type 'text/plain'
+        api = self.methods.select do |method|
+          (method =~ /^(GET|POST) /) && !(method =~ /png$/)
+        end
+        api.sort.join("\n")
+      end
+      
+      get '/streams/:name.json' do
+        stream = Stream.get(params[:name])
+        to_json(
+          :name=>stream.name,
+          :resource=>stream.resource,
+          :params=>stream.params.all
+        )
+      end
+      
+      # Usage:
+      # streams/filter?track=a,b,c&follow=d,e,f
+      put '/streams/:name.json' do
+        key = params[:key]
+        stream = Stream.get(params[:name])
+        params.keys.each do |key|
+          unless key.to_sym == :name
+            Flamingo.logger.info "Setting #{key} to #{params[key]}"
+            stream.params[key] = params[key].split(",")
+          end
+        end
+        change_predicates  
+        to_json(
+          :name=>stream.name,
+          :resource=>stream.resource,
+          :params=>stream.params.all
+        )      
+      end
+      
+      get '/streams/:name/:key.json' do
+        stream = Stream.get(params[:name])
+        to_json(stream.params[params[:key]])
+      end
+      
+      # One of:
+      #   Add values to the existing key
+      #     ?values=A,B,C
+      #   Add and remove in a single request
+      #     ?add=A,B&remove=C
+      post '/streams/:name/:key.json' do
+        key = params[:key]
+        stream = Stream.get(params[:name])
+        new_terms = params[:add] || params[:values]
+        stream.params.add(key,*new_terms.split(","))
+        remove_terms = params[:remove]
+        if remove_terms
+          stream.params.remove(key,*remove_terms.split(","))
+        end
+        change_predicates
+        to_json(stream.params[key])
+      end
+      
+      put '/streams/:name/:key.json' do
+        key = params[:key]
+        stream = Stream.get(params[:name])
+        new_terms = params[:values]
+        stream.params[key] = new_terms.split(",")
+        change_predicates
+        to_json(stream.params[key])
+      end
+      
+      delete '/streams/:name/:key.json' do
+        key = params[:key]
+        stream = Stream.get(params[:name])
+        if params[:values].blank?
+          stream.params.delete(key)
+        else
+          stream.params.remove(key,*params[:values].split(","))
+        end
+        change_predicates
+        to_json(stream.params[key])
+      end
+      
+      #Subscriptions
+      get '/subscriptions.json' do 
+        subs = Subscription.all.map do |sub|
+          {:name=>sub.name}
+        end
+        to_json(subs)
+      end
+  
+      post '/subscriptions.json' do
+        sub = Subscription.new(params[:name])
+        sub.save
+        to_json(:name=>sub.name)
+      end
+      
+      get '/subscriptions/:name.json' do
+        sub = Subscription.find(params[:name])
+        not_found(to_json(:error=>"Subscription does not exist")) unless sub
+        to_json(:name=>sub.name)
+      end
+      
+      delete '/subscriptions/:name.json' do
+        sub = Subscription.find(params[:name])
+        not_found(to_json(:error=>"Subscription does not exist")) unless sub
+        sub.delete
+        to_json(:name=>sub.name)
+      end
+      
+      private
+        def change_predicates
+          if options.respond_to?(:daemon_pid)
+            Process.kill("USR1",options.daemon_pid)
+            Flamingo.logger.info "Rotating wader in daemon"
+          end
+        end
+    
+        def to_json(value)
+          ActiveSupport::JSON.encode(value)
+        end
+    end
+  end
+end

metadata

@@ -0,0 +1,204 @@
+--- !ruby/object:Gem::Specification 
+name: flamingo
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Hayes Davis
+- Jerry Chen
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-07-19 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 25
+        segments: 
+        - 1
+        - 0
+        - 7
+        version: 1.0.7
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: redis-namespace
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        - 7
+        - 0
+        version: 0.7.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 61
+        segments: 
+        - 1
+        - 9
+        - 7
+        version: 1.9.7
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 63
+        segments: 
+        - 0
+        - 9
+        - 2
+        version: 0.9.2
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: twitter-stream
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 19
+        segments: 
+        - 0
+        - 1
+        - 4
+        version: 0.1.4
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: yajl-ruby
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 6
+        - 7
+        version: 0.6.7
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 1
+        - 0
+        version: 2.1.0
+  type: :runtime
+  version_requirements: *id007
+description: "    Flamingo makes it easy to wade through the Twitter Streaming API by \n    handling all connectivity and resource management for you. You just tell \n    it what to track and consume the information in a resque queue. \n\n    Flamingo isn't a traditional ruby gem. You don't require it into your code.\n    Instead, it's designed to run as a daemon like redis or mysql. It provides \n    a REST interface to change the parameters sent to the Twitter Streaming \n    resource. All events from the streaming API are placed on a resque job \n    queue where your application can process them.\n\n"
+email: hayes@appozite.com
+executables: 
+- flamingo
+- flamingod
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.md
+files: 
+- README.md
+- Rakefile
+- lib/flamingo/config.rb
+- lib/flamingo/daemon/child_process.rb
+- lib/flamingo/daemon/dispatcher_process.rb
+- lib/flamingo/daemon/flamingod.rb
+- lib/flamingo/daemon/pid_file.rb
+- lib/flamingo/daemon/wader_process.rb
+- lib/flamingo/daemon/web_server_process.rb
+- lib/flamingo/dispatch_error.rb
+- lib/flamingo/dispatch_event.rb
+- lib/flamingo/logging/formatter.rb
+- lib/flamingo/stream.rb
+- lib/flamingo/stream_params.rb
+- lib/flamingo/subscription.rb
+- lib/flamingo/version.rb
+- lib/flamingo/wader.rb
+- lib/flamingo/web/server.rb
+- lib/flamingo.rb
+- bin/flamingo
+- bin/flamingo-web
+- bin/flamingod
+- examples/flamingo.yml
+- examples/Rakefile
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/hayesdavis/flamingo
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Flamingo is an elegant way to wade into the Twitter Streaming API.
+test_files: []
+