gandalf 0.0.2 → 0.0.3

data/lib/gandalf.rb

@@ -4,13 +4,17 @@ require 'feedzirra'
 require 'activesupport'
 require 'rufus/scheduler'
 require 'dm-core'
-#has a bug
+# Has a bug
 #require 'dm-aggregates'
 
 require 'digest/md5'
 require 'json'
 
+require 'redis_ext/redis'
 require 'redis_ext/redis_queue'
+require 'redis_ext/pipeline'
+require 'redis_ext/dist_redis'
+require 'redis_ext/hash_ring'
 
 require 'gandalf/scheduler'
 require 'gandalf/worker'

data/lib/gandalf/models.rb

@@ -18,7 +18,6 @@ module Gandalf
   class Post
     include DataMapper::Resource
 
-    storage_names[:default] = 'posts'
     property :id, Serial, :field => 'psid'
     property :channel_id, Integer, :length => 11, :index => true
     property :link, Text, :length => 255
@@ -35,6 +34,8 @@ module Gandalf
       self.title = self.title[0,255]
       if self.description
         self.description.gsub!(/\<[^\>]+\>|\n|&nbsp;/,' ')
+        self.description.gsub!(/&gt;/,'<')
+        self.description.gsub!(/&lt;/,'>')
         self.description.gsub!(/\s{2,}/,' ')
       end
     end

data/lib/gandalf/scheduler.rb

@@ -17,9 +17,13 @@ module Gandalf
     attr_accessor :redis
 
     # Sets workers' queues with Redis connection object.
-    def setup(options = {:seed_class => Seed})
+    def setup(options = {})
       @redis = Redis.new(:host => self.redis_host, :db => self.redis_db_id)
-      @Seed = options[:seed_class]
+      if options[:seed_class]
+        @Seed = options[:seed_class]
+      else
+        @Seed = Seed
+      end 
       workers.each { |worker| worker.setup(:redis => @redis) }
     end
 

data/lib/gandalf/worker.rb

@@ -12,15 +12,19 @@ module Gandalf
 
     belongs_to :scheduler
 
-    def setup(options = {:post_class => Post})
+    def setup(options = {})
       @queue = RedisQueue.new(:key => self.id, :redis => options[:redis]) unless @queue
-      @Post = (options[:post_class])
+      if options[:post_class]
+        @Post = options[:post_class]
+      else
+        @Post = Post
+      end
     end
 
     def run
       @crawl_scheduler = Rufus::Scheduler.start_new unless @crawl_scheduler
       @crawl_scheduler.every interval do
-        crawl new_jobs(max_jobs)
+        crawl new_jobs(max_jobs) if jobs_to_do > 0
       end
     end
 

data/lib/redis_ext/dist_redis.rb

@@ -0,0 +1,124 @@
+require 'redis'
+require 'hash_ring'
+class DistRedis
+  attr_reader :ring
+  def initialize(opts={})
+    hosts = []
+
+    db = opts[:db] || nil
+    timeout = opts[:timeout] || nil
+
+    raise Error, "No hosts given" unless opts[:hosts]
+
+    opts[:hosts].each do |h|
+      host, port = h.split(':')
+      hosts << Redis.new(:host => host, :port => port, :db => db, :timeout => timeout)
+    end
+
+    @ring = HashRing.new hosts
+  end
+
+  def node_for_key(key)
+    key = $1 if key =~ /\{(.*)?\}/
+    @ring.get_node(key)
+  end
+
+  def add_server(server)
+    server, port = server.split(':')
+    @ring.add_node Redis.new(:host => server, :port => port)
+  end
+
+  def method_missing(sym, *args, &blk)
+    if redis = node_for_key(args.first.to_s)
+      redis.send sym, *args, &blk
+    else
+      super
+    end
+  end
+
+  def keys(glob)
+    @ring.nodes.map do |red|
+      red.keys(glob)
+    end
+  end
+
+  def save
+    on_each_node :save
+  end
+
+  def bgsave
+    on_each_node :bgsave
+  end
+
+  def quit
+    on_each_node :quit
+  end
+
+  def flush_all
+    on_each_node :flush_all
+  end
+  alias_method :flushall, :flush_all
+
+  def flush_db
+    on_each_node :flush_db
+  end
+  alias_method :flushdb, :flush_db
+
+  def delete_cloud!
+    @ring.nodes.each do |red|
+      red.keys("*").each do |key|
+        red.delete key
+      end
+    end
+  end
+
+  def on_each_node(command, *args)
+    @ring.nodes.each do |red|
+      red.send(command, *args)
+    end
+  end
+
+end
+
+
+if __FILE__ == $0
+
+r = DistRedis.new 'localhost:6379', 'localhost:6380', 'localhost:6381', 'localhost:6382'
+  r['urmom'] = 'urmom'
+  r['urdad'] = 'urdad'
+  r['urmom1'] = 'urmom1'
+  r['urdad1'] = 'urdad1'
+  r['urmom2'] = 'urmom2'
+  r['urdad2'] = 'urdad2'
+  r['urmom3'] = 'urmom3'
+  r['urdad3'] = 'urdad3'
+  p r['urmom']
+  p r['urdad']
+  p r['urmom1']
+  p r['urdad1']
+  p r['urmom2']
+  p r['urdad2']
+  p r['urmom3']
+  p r['urdad3']
+
+  r.push_tail 'listor', 'foo1'
+  r.push_tail 'listor', 'foo2'
+  r.push_tail 'listor', 'foo3'
+  r.push_tail 'listor', 'foo4'
+  r.push_tail 'listor', 'foo5'
+
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+  p r.pop_tail('listor')
+
+  puts "key distribution:"
+
+  r.ring.nodes.each do |red|
+    p [red.port, red.keys("*")]
+  end
+  r.delete_cloud!
+  p r.keys('*')
+
+end

data/lib/redis_ext/hash_ring.rb

@@ -0,0 +1,128 @@
+require 'zlib'
+
+class HashRing
+
+  POINTS_PER_SERVER = 160 # this is the default in libmemcached
+
+  attr_reader :ring, :sorted_keys, :replicas, :nodes
+
+  # nodes is a list of objects that have a proper to_s representation.
+  # replicas indicates how many virtual points should be used pr. node,
+  # replicas are required to improve the distribution.
+  def initialize(nodes=[], replicas=POINTS_PER_SERVER)
+    @replicas = replicas
+    @ring = {}
+    @nodes = []
+    @sorted_keys = []
+    nodes.each do |node|
+      add_node(node)
+    end
+  end
+  
+  # Adds a `node` to the hash ring (including a number of replicas).
+  def add_node(node)
+    @nodes << node
+    @replicas.times do |i|
+      key = Zlib.crc32("#{node}:#{i}")
+      @ring[key] = node
+      @sorted_keys << key
+    end
+    @sorted_keys.sort!
+  end
+  
+  def remove_node(node)
+    @nodes.reject!{|n| n.to_s == node.to_s}
+    @replicas.times do |i|
+      key = Zlib.crc32("#{node}:#{i}")
+      @ring.delete(key)
+      @sorted_keys.reject! {|k| k == key}
+    end
+  end
+  
+  # get the node in the hash ring for this key
+  def get_node(key)
+    get_node_pos(key)[0]
+  end
+  
+  def get_node_pos(key)
+    return [nil,nil] if @ring.size == 0
+    crc = Zlib.crc32(key)
+    idx = HashRing.binary_search(@sorted_keys, crc)
+    return [@ring[@sorted_keys[idx]], idx]
+  end
+  
+  def iter_nodes(key)
+    return [nil,nil] if @ring.size == 0
+    node, pos = get_node_pos(key)
+    @sorted_keys[pos..-1].each do |k|
+      yield @ring[k]
+    end  
+  end
+  
+  class << self
+
+    # gem install RubyInline to use this code
+    # Native extension to perform the binary search within the hashring.
+    # There's a pure ruby version below so this is purely optional
+    # for performance.  In testing 20k gets and sets, the native
+    # binary search shaved about 12% off the runtime (9sec -> 8sec).
+    begin
+      require 'inline'
+      inline do |builder|
+        builder.c <<-EOM
+        int binary_search(VALUE ary, unsigned int r) {
+            int upper = RARRAY_LEN(ary) - 1;
+            int lower = 0;
+            int idx = 0;
+
+            while (lower <= upper) {
+                idx = (lower + upper) / 2;
+
+                VALUE continuumValue = RARRAY_PTR(ary)[idx];
+                unsigned int l = NUM2UINT(continuumValue);
+                if (l == r) {
+                    return idx;
+                }
+                else if (l > r) {
+                    upper = idx - 1;
+                }
+                else {
+                    lower = idx + 1;
+                }
+            }
+            return upper;
+        }
+        EOM
+      end
+    rescue Exception => e
+      # Find the closest index in HashRing with value <= the given value
+      def binary_search(ary, value, &block)
+        upper = ary.size - 1
+        lower = 0
+        idx = 0
+
+        while(lower <= upper) do
+          idx = (lower + upper) / 2
+          comp = ary[idx] <=> value
+
+          if comp == 0
+            return idx
+          elsif comp > 0
+            upper = idx - 1
+          else
+            lower = idx + 1
+          end
+        end
+        return upper
+      end
+
+    end
+  end
+
+end
+
+# ring = HashRing.new ['server1', 'server2', 'server3']
+# p ring
+# #
+# p ring.get_node "kjhjkjlkjlkkh"
+# 

data/lib/redis_ext/pipeline.rb

@@ -0,0 +1,22 @@
+require "redis"
+
+class Redis
+  class Pipeline < Redis
+    BUFFER_SIZE = 50_000
+    
+    def initialize(redis)
+      @redis = redis
+      @commands = []
+    end
+   
+    def call_command(command)
+      @commands << command
+    end
+
+    def execute
+      @redis.call_command(@commands)
+      @commands.clear
+    end
+    
+  end
+end

data/lib/redis_ext/redis.rb

@@ -0,0 +1,316 @@
+require 'socket'
+require File.join(File.dirname(__FILE__),'pipeline')
+
+begin
+  if RUBY_VERSION >= '1.9'
+    require 'timeout'
+    RedisTimer = Timeout
+  else
+    require 'system_timer'
+    RedisTimer = SystemTimer
+  end
+rescue LoadError
+  RedisTimer = nil
+end
+
+class Redis
+  OK      = "OK".freeze
+  MINUS    = "-".freeze
+  PLUS     = "+".freeze
+  COLON    = ":".freeze
+  DOLLAR   = "$".freeze
+  ASTERISK = "*".freeze
+
+  BULK_COMMANDS = {
+    "set"       => true,
+    "setnx"     => true,
+    "rpush"     => true,
+    "lpush"     => true,
+    "lset"      => true,
+    "lrem"      => true,
+    "sadd"      => true,
+    "srem"      => true,
+    "sismember" => true,
+    "echo"      => true,
+    "getset"    => true,
+    "smove"     => true
+  }
+
+  BOOLEAN_PROCESSOR = lambda{|r| r == 1 }
+
+  REPLY_PROCESSOR = {
+    "exists"    => BOOLEAN_PROCESSOR,
+    "sismember" => BOOLEAN_PROCESSOR,
+    "sadd"      => BOOLEAN_PROCESSOR,
+    "srem"      => BOOLEAN_PROCESSOR,
+    "smove"     => BOOLEAN_PROCESSOR,
+    "move"      => BOOLEAN_PROCESSOR,
+    "setnx"     => BOOLEAN_PROCESSOR,
+    "del"       => BOOLEAN_PROCESSOR,
+    "renamenx"  => BOOLEAN_PROCESSOR,
+    "expire"    => BOOLEAN_PROCESSOR,
+    "keys"      => lambda{|r| r.split(" ")},
+    "info"      => lambda{|r|
+      info = {}
+      r.each_line {|kv|
+        k,v = kv.split(":",2).map{|x| x.chomp}
+        info[k.to_sym] = v
+      }
+      info
+    }
+  }
+
+  ALIASES = {
+    "flush_db"             => "flushdb",
+    "flush_all"            => "flushall",
+    "last_save"            => "lastsave",
+    "key?"                 => "exists",
+    "delete"               => "del",
+    "randkey"              => "randomkey",
+    "list_length"          => "llen",
+    "push_tail"            => "rpush",
+    "push_head"            => "lpush",
+    "pop_tail"             => "rpop",
+    "pop_head"             => "lpop",
+    "list_set"             => "lset",
+    "list_range"           => "lrange",
+    "list_trim"            => "ltrim",
+    "list_index"           => "lindex",
+    "list_rm"              => "lrem",
+    "set_add"              => "sadd",
+    "set_delete"           => "srem",
+    "set_count"            => "scard",
+    "set_member?"          => "sismember",
+    "set_members"          => "smembers",
+    "set_intersect"        => "sinter",
+    "set_intersect_store"  => "sinterstore",
+    "set_inter_store"      => "sinterstore",
+    "set_union"            => "sunion",
+    "set_union_store"      => "sunionstore",
+    "set_diff"             => "sdiff",
+    "set_diff_store"       => "sdiffstore",
+    "set_move"             => "smove",
+    "set_unless_exists"    => "setnx",
+    "rename_unless_exists" => "renamenx",
+    "type?"                => "type"
+  }
+
+  DISABLED_COMMANDS = {
+    "monitor" => true,
+    "sync"    => true
+  }
+
+  def initialize(options = {})
+    @host    =  options[:host]    || '127.0.0.1'
+    @port    = (options[:port]    || 6379).to_i
+    @db      = (options[:db]      || 0).to_i
+    @timeout = (options[:timeout] || 5).to_i
+    @password = options[:password]
+    @logger  =  options[:logger]
+
+    @logger.info { self.to_s } if @logger
+    connect_to_server
+  end
+
+  def to_s
+    "Redis Client connected to #{server} against DB #{@db}"
+  end
+
+  def server
+    "#{@host}:#{@port}"
+  end
+
+  def connect_to_server
+    @sock = connect_to(@host, @port, @timeout == 0 ? nil : @timeout)
+    call_command(["auth",@password]) if @password
+    call_command(["select",@db]) unless @db == 0
+  end
+
+  def connect_to(host, port, timeout=nil)
+    # We support connect() timeout only if system_timer is availabe
+    # or if we are running against Ruby >= 1.9
+    # Timeout reading from the socket instead will be supported anyway.
+    if @timeout != 0 and RedisTimer
+      begin
+        sock = TCPSocket.new(host, port)
+      rescue Timeout::Error
+        @sock = nil
+        raise Timeout::Error, "Timeout connecting to the server"
+      end
+    else
+      sock = TCPSocket.new(host, port)
+    end
+    sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
+
+    # If the timeout is set we set the low level socket options in order
+    # to make sure a blocking read will return after the specified number
+    # of seconds. This hack is from memcached ruby client.
+    if timeout
+      secs   = Integer(timeout)
+      usecs  = Integer((timeout - secs) * 1_000_000)
+      optval = [secs, usecs].pack("l_2")
+      sock.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
+      sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
+    end
+    sock
+  end
+
+  def method_missing(*argv)
+    call_command(argv)
+  end
+
+  def call_command(argv)
+    @logger.debug { argv.inspect } if @logger
+
+    # this wrapper to raw_call_command handle reconnection on socket
+    # error. We try to reconnect just one time, otherwise let the error
+    # araise.
+    connect_to_server if !@sock
+
+    begin
+      raw_call_command(argv.dup)
+    rescue Errno::ECONNRESET, Errno::EPIPE
+      @sock.close
+      @sock = nil
+      connect_to_server
+      raw_call_command(argv.dup)
+    end
+  end
+
+  def raw_call_command(argvp)
+    pipeline = argvp[0].is_a?(Array)
+
+    unless pipeline
+      argvv = [argvp]
+    else
+      argvv = argvp
+    end
+
+    command = ''
+
+    argvv.each do |argv|
+      bulk = nil
+      argv[0] = argv[0].to_s.downcase
+      argv[0] = ALIASES[argv[0]] if ALIASES[argv[0]]
+      raise "#{argv[0]} command is disabled" if DISABLED_COMMANDS[argv[0]]
+      if BULK_COMMANDS[argv[0]] and argv.length > 1
+        bulk = argv[-1].to_s
+        argv[-1] = bulk.respond_to?(:bytesize) ? bulk.bytesize : bulk.size
+      end
+      command << "#{argv.join(' ')}\r\n"
+      command << "#{bulk}\r\n" if bulk
+    end
+
+    @sock.write(command)
+
+    results = argvv.map do |argv|
+      processor = REPLY_PROCESSOR[argv[0]]
+      processor ? processor.call(read_reply) : read_reply
+    end
+
+    return pipeline ? results : results[0]
+  end
+
+  def select(*args)
+    raise "SELECT not allowed, use the :db option when creating the object"
+  end
+
+  def [](key)
+    self.get(key)
+  end
+
+  def []=(key,value)
+    set(key,value)
+  end
+
+  def set(key, value, expiry=nil)
+    s = call_command([:set, key, value]) == OK
+    expire(key, expiry) if s && expiry
+    s
+  end
+
+  def sort(key, options = {})
+    cmd = ["SORT"]
+    cmd << key
+    cmd << "BY #{options[:by]}" if options[:by]
+    cmd << "GET #{[options[:get]].flatten * ' GET '}" if options[:get]
+    cmd << "#{options[:order]}" if options[:order]
+    cmd << "LIMIT #{options[:limit].join(' ')}" if options[:limit]
+    call_command(cmd)
+  end
+
+  def incr(key, increment = nil)
+    call_command(increment ? ["incrby",key,increment] : ["incr",key])
+  end
+
+  def decr(key,decrement = nil)
+    call_command(decrement ? ["decrby",key,decrement] : ["decr",key])
+  end
+
+  # Similar to memcache.rb's #get_multi, returns a hash mapping
+  # keys to values.
+  def mapped_mget(*keys)
+    mget(*keys).inject({}) do |hash, value|
+      key = keys.shift
+      value.nil? ? hash : hash.merge(key => value)
+    end
+  end
+
+  # Ruby defines a now deprecated type method so we need to override it here
+  # since it will never hit method_missing
+  def type(key)
+    call_command(['type', key])
+  end
+
+  def quit
+    call_command(['quit'])
+  rescue Errno::ECONNRESET
+  end
+
+  def pipelined(&block)
+    pipeline = Pipeline.new self
+    yield pipeline
+    pipeline.execute
+  end
+
+  def read_reply
+    # We read the first byte using read() mainly because gets() is
+    # immune to raw socket timeouts.
+    begin
+      rtype = @sock.read(1)
+    rescue Errno::EAGAIN
+      # We want to make sure it reconnects on the next command after the
+      # timeout. Otherwise the server may reply in the meantime leaving
+      # the protocol in a desync status.
+      @sock = nil
+      raise Errno::EAGAIN, "Timeout reading from the socket"
+    end
+
+    raise Errno::ECONNRESET,"Connection lost" if !rtype
+    line = @sock.gets
+    case rtype
+    when MINUS
+      raise MINUS + line.strip
+    when PLUS
+      line.strip
+    when COLON
+      line.to_i
+    when DOLLAR
+      bulklen = line.to_i
+      return nil if bulklen == -1
+      data = @sock.read(bulklen)
+      @sock.read(2) # CRLF
+      data
+    when ASTERISK
+      objects = line.to_i
+      return nil if bulklen == -1
+      res = []
+      objects.times {
+        res << read_reply
+      }
+      res
+    else
+      raise "Protocol error, got '#{rtype}' as initial reply byte"
+    end
+  end
+end

data/lib/redis_ext/redis_queue.rb

@@ -1,4 +1,3 @@
-require 'redis'
 require 'json'
 
 class RedisQueue
@@ -20,8 +19,8 @@ class RedisQueue
       new_hash = {}
       value.each{|k, v| new_hash[k.to_sym] = v}
       return new_hash 
-    rescue Exception => ex
-      puts ex
+    # Returns nil on any kind of exception
+    rescue Exception
       return nil
     end
   end
@@ -30,9 +29,9 @@ class RedisQueue
     @redis.llen(@key)
   end
 
-  def pop_first(length)
-    list = [] 
-    length.times do
+  def pop_first(count)
+    list = []
+    count.times do
       element = self.pop
       break unless element
       list << element

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: gandalf
 version: !ruby/object:Gem::Version 
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors: 
 - Kijun Seo
@@ -62,6 +62,10 @@ extra_rdoc_files: []
 
 files: 
 - lib/redis_ext/redis_queue.rb
+- lib/redis_ext/redis.rb
+- lib/redis_ext/dist_redis.rb
+- lib/redis_ext/pipeline.rb
+- lib/redis_ext/hash_ring.rb
 - lib/gandalf.rb
 - lib/gandalf/models.rb
 - lib/gandalf/scheduler.rb