messed 0.0.1

data/lib/messed/interface/adapter/twitter_consumer.rb

@@ -0,0 +1,35 @@
+class Messed
+  class Interface
+    class Adapter
+      class TwitterConsumer < Adapter
+        
+        attr_accessor :started_at, :packets_processed, :errors, :last_error, :last_ok, :last_status
+        
+        def init
+          @started_at = Time.new
+          @packets_processed = 0
+          @errors = 0
+          @last_error = nil
+          @last_ok = nil
+          @last_status = nil
+        end
+
+        def type
+          :twitter
+        end
+        
+        def status
+          {
+            :started_at => started_at,
+            :packets_processed => packets_processed,
+            :errors => errors,
+            :last_error => last_error,
+            :last_ok => last_ok,
+            :last_status => last_status
+          }
+        end
+        
+      end
+    end
+  end
+end

data/lib/messed/interface/adapter/twitter_search.rb

@@ -0,0 +1,111 @@
+require 'set'
+require 'rack'
+
+class Messed
+  class Interface
+    class Adapter
+      class TwitterSearch < TwitterConsumer
+
+        attr_reader :packets_processed
+
+        def build_query
+          Rack::Utils.build_query(interface.configuration.options[:fetch][:query])
+        end
+        
+        def start
+          @ids ||= default_ids
+          # do work.
+          begin
+            query = build_query
+            logger.debug "Twitter Search -> #{query.inspect}"
+            http = EventMachine::HttpRequest.new("http://#{interface.configuration.options[:fetch][:host]}/#{interface.configuration.options[:fetch][:path]}").
+              get(:query => query, :timeout => 30)
+            http.callback {
+              self.last_status = http.response_header.status
+              case http.response_header.status
+                when 200
+                  self.last_ok = Time.new
+                  data = JSON.parse(http.response)
+                  interface.configuration.options[:fetch][:query][:since_id] = data['max_id'] if interface.configuration.options[:fetch][:query]
+                  @test_set = Set.new(@ids)
+                  data['results'].each do |result|
+                    result_to_message(result)
+                  end
+                  trim_ids
+              end
+              EM.add_timer(interface.configuration.options[:interval]) do
+                start
+              end
+            }
+            http.errback {
+              self.errors += 1
+              self.last_error = Time.new
+              EM.add_timer(interface.configuration.options[:interval]) do
+                start
+              end
+            }
+          rescue RuntimeError
+            EM.add_timer(interface.configuration.options[:interval]) do
+              start
+            end
+          end
+        end
+
+        def id_retention_count
+          interface.configuration.options[:id_retention_size] || 500
+        end
+
+        def trim_ids
+          while @ids.size > id_retention_count
+            @ids.shift
+          end
+        end
+
+        def store_ids?
+          id_retention_file
+        end
+
+        def write_ids
+          File.open(id_retention_file, 'w') { |f| f << @ids.to_json }
+        end
+        
+        def read_ids
+          @ids = JSON.parse(File.read(id_retention_file))
+        end
+        
+        def default_ids
+          store_ids? ? read_ids : []
+        rescue 
+          logger.error "Twitter Search: Unable to load ids from #{id_retention_file.inspect}"
+          []
+        end
+        
+        def id_retention_file
+          interface.configuration.options[:id_retention_tmp_file]
+        end
+
+        def result_to_message(result)
+          unless @test_set.include?(result['id'])
+            message = Message::Twitter.new do |m|
+              m.body = result['text']
+              m.from = result['from_user']
+              m.to = result['to_user_id']
+              m.created_at = Time.rfc2822(result['created_at'])
+              m.profile_image_url = result['profile_image_url']
+              m.id = result['id']
+              m.geo = result['geo']
+              m.from_user_id = result['from_user_id']
+              m.iso_language_code = result['iso_language_code']
+              m.source = result['source']
+            end
+            @ids << message.id
+            @packets_processed += 1
+            interface.application.incoming << message
+            logger.debug "Twitter Search: Adding message #{message.id}: #{message.body} to incoming queue"
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/messed/interface/adapter/twitter_sender.rb

@@ -0,0 +1,44 @@
+require 'twitter'
+
+class Messed
+  class Interface
+    class Adapter
+      class TwitterSender < Adapter
+        
+        def message_class
+          Messed::Message::Twitter
+        end
+        
+        def start
+          jack = EM::Beanstalk.new
+          jack.watch(interface.application.outgoing.tube) do
+            jack.use(interface.application.outgoing.tube) do
+              jack.each_job do |job|
+                process_outgoing(job, message_class.from_json(job.body))
+              end
+            end
+          end
+        end
+        
+        def process_outgoing(job, message)
+          if message.private?
+            req = EventMachine::HttpRequest.new("http://twitter.com/direct_messages/new.json")
+            data = {:body => {:user => message.to_user_id, :text => message.body}, :timeout => 30, :head => {'authorization' => [interface.configuration.options[:username], interface.configuration.options[:password]]}}
+            data[:body][:in_reply_to] = message.in_reply_to if message.in_reply_to
+            http = req.post(data)
+            http.callback {
+              job.delete
+            }
+          else
+            http = EventMachine::HttpRequest.new("http://twitter.com/statuses/update.json").
+              post(:body => {:status => message.body}, :timeout => 30)
+            http.callback {
+              job.delete
+            }
+          end
+        end
+        
+      end
+    end
+  end
+end

data/lib/messed/interface/adapter/twitter_streaming.rb

@@ -0,0 +1,50 @@
+require 'twitter/json_stream'
+
+class Messed
+  class Interface
+    class Adapter
+      class TwitterStreaming < TwitterConsumer
+        
+        include Logger::LoggingModule
+
+        def start
+          stream = Twitter::JSONStream.connect(
+            :path    => "/1/statuses/#{interface.configuration.options['streaming']['type']}.json?track=%23classicmoviequotes",
+            :auth    => "#{interface.configuration.options['username']}:#{interface.configuration.options['password']}"
+          )
+          
+          stream.each_item do |item|
+            self.last_ok = Time.new
+            result = JSON.parse(item)
+            message = Message::Twitter.new do |m|
+              m.body = result['text']
+              m.from = result['from_user']
+              m.to = result['to_user_id']
+              m.created_at = Time.rfc2822(result['created_at'])
+              m.profile_image_url = result['profile_image_url']
+              m.id = result['id']
+              m.geo = result['geo']
+              m.from_user_id = result['from_user_id']
+              m.iso_language_code = result['iso_language_code']
+              m.source = result['source']
+            end
+            self.packets_processed += 1
+            interface.application.incoming << message
+          end
+
+          stream.on_error do |message|
+            logger.error "message #{message.inspect}"
+            self.errors += 1
+            self.last_error = message
+          end
+
+          stream.on_max_reconnects do |timeout, retries|
+            logger.error "message -> #{timeout} #{retries}"
+            # Something is wrong on your side. Send yourself an email.
+          end
+        end
+        
+      end
+    end
+  end
+end

data/lib/messed/interface/adapter.rb

@@ -0,0 +1,68 @@
+require 'json'
+
+class Messed
+  class Interface
+    class Adapter
+      
+      include Logger::LoggingModule
+      
+      autoload :TwitterSearch,    File.join('messed', 'interface', 'adapter', 'twitter_search')
+      autoload :TwitterSender,    File.join('messed', 'interface', 'adapter', 'twitter_sender')
+      autoload :TwitterStreaming, File.join('messed', 'interface', 'adapter', 'twitter_streaming')
+
+      # abstract class for twitter consumption
+      autoload :TwitterConsumer,  File.join(File.dirname(__FILE__), 'adapter', 'twitter_consumer')
+      
+      Registry = {}
+      def self.register_for_name(name, class_name)
+        Registry[name] = class_name
+      end
+      
+      register_for_name :twitter_search,    'Messed::Interface::Adapter::TwitterSearch'
+      register_for_name :twitter_sender,    'Messed::Interface::Adapter::TwitterSender'
+      register_for_name :twitter_streaming, 'Messed::Interface::Adapter::TwitterStreaming'
+      
+      def self.for_name(name)
+        class_name = Registry[name]
+        class_name ? 
+          class_name.constantize :
+          raise("No adapter for #{name}")
+      end
+      
+      attr_reader :interface
+      
+      def initialize(interface)
+        @interface = interface
+        init
+      end
+      
+      def save_state(state)
+        File.open(state_file, 'w') {|f| f << state.to_json }
+      end
+
+      def load_state
+        JSON.parse(File.read(state_file))
+      end
+      
+      def state_file
+        "/tmp/#{name}.state"
+      end
+      
+      # any post initialization goes here
+      def init
+      end
+      
+      def type
+        raise "method type must be defined"
+      end
+
+      def start(detach)
+        raise "method start must be defined"
+      end
+
+      def send(message)
+      end
+
+    end
+  end
+end

data/lib/messed/interface/runner.rb

@@ -0,0 +1,20 @@
+require 'thin'
+
+class Messed
+  class Interface
+    class Runner
+      
+      attr_reader :runnable, :options
+      
+      def initialize(runnable, options)
+        @runnable = runnable
+        @options = options
+      end
+      
+      def start
+        runnable.start(options[:detach])
+      end
+      
+    end
+  end
+end

data/lib/messed/interface.rb

@@ -0,0 +1,74 @@
+class Messed
+  class Interface
+    
+    include Logger::LoggingModule
+    
+    autoload :Runner,      File.join('messed', 'interface', 'runner')
+    autoload :Adapter,     File.join('messed', 'interface', 'adapter')
+    
+    def self.interface_from_configuration(booter, name, configuration)
+      interface = Interface.new
+      interface.send(:name=,          name)
+      interface.send(:configuration=, configuration)
+      interface.send(:adapter=,       Adapter.for_name(configuration.adapter).new(interface))
+      interface.send(:booter=,        booter)
+      interface
+    end
+
+    def to_s
+      "#{Array(configuration['mode']) * ', '} ->> #{name}"
+    end
+    
+    def status
+      {
+        'adapter' => adapter.status,
+        'configuration' => configuration,
+        'name' => name
+      }
+    end
+    
+    attr_reader :booter, :configuration, :adapter, :name, :started_at
+
+    def initialize
+      @started_at = Time.new
+    end
+    
+    def application
+      booter.application
+    end
+    
+    def status_host
+      configuration.status_address || '0.0.0.0'
+    end
+    
+    def status_port
+      configuration.status_port || 11190
+    end
+    
+    def stop
+      Process.kill("INT", booter.read_pid_file(configuration.pid_file))
+      exit(0)
+    end
+    
+    def start
+      begin
+        EM.start_server(status_host, status_port, EMRunner::StatusHandler) do |c|
+          c.interface = self
+        end
+        logger.info "Status handler for #{self.class} started on #{status_host}:#{status_port}"
+      rescue RuntimeError => e
+        if e.message =~ /no acceptor/
+          logger.error "Unable to start status handler"
+        else
+          raise e
+        end
+          
+      end
+      booter.write_pid_file(configuration.pid_file)
+      adapter.start
+    end
+    
+    protected
+    attr_writer :booter, :configuration, :adapter, :name
+  end
+end

data/lib/messed/logger.rb

@@ -0,0 +1,50 @@
+require 'logger'
+
+class Messed
+  class Logger
+    module LoggingModule
+      def self.included(cls)
+        cls.class_eval(<<-HERE_DOC, __FILE__, __LINE__)
+      
+        def self.logger
+          Messed::Logger.instance.logger
+        end
+      
+        def self.logger=(logger)
+          Messed::Logger.instance.logger = logger
+        end
+      
+        HERE_DOC
+      end
+      
+      def logger
+        Messed::Logger.instance.logger
+      end
+
+      def logger=(logger)
+        Messed::Logger.instance.logger = logger
+      end
+
+    end
+    
+    attr_accessor :logger
+    
+    def initialize
+      setup_logger
+    end
+    private :initialize
+    
+    def self.instance
+      @@instance
+    end
+    
+    def setup_logger(logger = ::Logger.new(STDOUT), log_level = :debug)
+      @logger = logger
+      @logger.level = ::Logger.const_get(log_level.to_s.upcase.to_sym) if log_level
+    end
+    
+    @@instance = self.new
+
+  end
+end
+    

data/lib/messed/matcher.rb

@@ -0,0 +1,50 @@
+class Messed
+  class Matcher
+    
+    attr_accessor :destination
+
+    def stop_processing?
+      true
+    end
+    
+    class Conditional < Matcher
+
+      attr_reader :matches
+
+      def initialize(body_matcher, other_matchers = nil)
+        @body_matcher = body_matcher
+        @other_matchers = other_matchers
+      end
+
+      def match?(message)
+        matches = true
+        self.matches = nil
+        if @body_matcher
+          matches &&= @body_matcher === message.body
+          self.matches = Regexp.last_match if matches && @body_matcher.is_a?(Regexp)
+        end
+
+        if @other_matchers
+          keys = @other_matchers.keys
+          unless matches
+            key = keys.pop
+            matches &&= @other_matchers[key] === message.send(key)
+          end
+        end
+        matches
+      end
+
+      private
+      attr_writer :matches
+    end
+    
+    class Always < Matcher
+      
+      def match?(message)
+        true
+      end
+      
+    end
+    
+  end
+end

data/lib/messed/message/twitter.rb

@@ -0,0 +1,23 @@
+class Messed
+  class Message
+    class Twitter < Message
+
+      attr_accessor :created_at, :profile_image_url, :id, :geo, :from_user_id, :iso_language_code, :source, :private, :to_user_id
+      hash_accessor :profile_image_url, :id, :geo, :from_user_id, :iso_language_code, :source, :to_user_id, :private
+      hash_convert  :created_at => Hashify::Convert::Time
+      
+      def initialize(body = nil)
+        super(body)
+        self.private = false
+      end
+
+      def unique_id
+        from_user_id
+      end
+
+      alias_method :private?, :private
+      
+    end
+  end
+end
+      

data/lib/messed/message.rb

@@ -0,0 +1,35 @@
+class Messed
+
+  class Message
+
+    autoload :Twitter,     File.join('messed', 'message', 'twitter')
+
+    include Hashify
+    include Hashify::Json
+
+    attr_accessor :body
+    attr_accessor :from
+    attr_accessor :to
+    attr_accessor :enqueued_at
+    attr_accessor :in_reply_to
+    
+    hash_accessor :body, :from, :to
+    hash_convert  :enqueued_at => Hashify::Convert::Time
+    hash_convert  :in_reply_to => [proc{|x| x && x.to_hash}, proc{|x, parent| x && parent.class.from_hash(x)}]
+    
+    def self.class_for_type(type)
+      case type
+      when :twitter
+        Twitter
+      else
+        raise type
+      end
+    end
+
+    def initialize(body = nil)
+      self.body = body
+      yield self if block_given?
+    end
+    
+  end
+end

data/lib/messed/queue/beanstalk.rb

@@ -0,0 +1,56 @@
+class Messed
+  class Queue
+    class Beanstalk < Queue
+
+      include Logger::LoggingModule
+
+      attr_accessor :application
+      attr_reader   :tube
+      
+      def initialize(tube, host = '127.0.0.1', port = 11300)
+        @tube, @host, @port = tube, host, port
+        @beanstalk = ::Beanstalk::Pool.new(Array("#{host}:#{port}"))
+        @beanstalk.use(tube)
+        @beanstalk.watch(tube)
+      end
+      
+      def status
+        @beanstalk.stats_tube(tube)
+      end
+      
+      def take(block = true)
+        job = beanstalk.reserve
+        begin 
+          message = application.message_class.from_json(job.body)
+        rescue JSON::ParserError
+          logger.error "malformed message #{job.body}"
+          job.delete
+        else
+          yield message
+          job.delete
+        end
+      end
+      
+      def <<(message)
+        beanstalk.put message.to_json
+      end
+      
+      def jobs_available?
+        not jobs_available.zero?
+      end
+      
+      def drain!
+        while jobs_available?
+          beanstalk.reserve.delete
+        end
+      end
+      
+      def jobs_available
+        beanstalk.stats_tube(tube)['current-jobs-ready']
+      end
+      protected
+      attr_reader :beanstalk
+      
+    end
+  end
+end

data/lib/messed/queue.rb

@@ -0,0 +1,17 @@
+require 'beanstalk-client'
+
+class Messed
+  class Queue
+    
+    autoload :Beanstalk, File.join('messed', 'queue', 'beanstalk')
+    
+    def <<(message)
+      raise
+    end
+    
+    def take
+      raise
+    end
+    
+  end
+end

data/lib/messed/session/memcache.rb

@@ -0,0 +1,40 @@
+require 'memcached'
+require 'json'
+require 'digest/md5'
+
+class Messed
+  class Session
+    class Memcache < Session
+      
+      def initialize(config = "127.0.0.1:11211")
+        @connection = Memcached.new(config)
+      end
+      
+      def reset!(id)
+        connection.delete(id)
+      end
+      
+      def with(id)
+        data = data(id)
+        begin
+          yield data
+        ensure
+          connection.set(key(id), data.to_json)
+        end
+      end
+      
+      protected
+      attr_reader :connection
+      
+      def data(id)
+        JSON.parse(connection.get(key(id))).strhash
+      rescue Memcached::NotFound
+        StrHash.new
+      end
+      
+      def key(id)
+        Digest::MD5.hexdigest(id.to_s)
+      end
+    end
+  end
+end

data/lib/messed/session.rb

@@ -0,0 +1,7 @@
+class Messed
+  class Session
+
+    autoload :Memcache,     File.join('messed', 'session', 'memcache')
+    
+  end
+end