syncache 1.0.0 → 1.2

data/ChangeLog.mtn

@@ -0,0 +1,183 @@
+-----------------------------------------------------------------
+Revision: 8bc8efa1310743cf39e14bdf69d4429cadebcb61
+Ancestor: 6f9a39297f610f7095473ac34f68674963561d9c
+Author: angdraug@debian.org
+Date: 25/12/09 19:56:53
+Branch: syncache-head
+
+Modified files:
+        bin/syncache-drb man/syncache-drb.1
+
+ChangeLog: 
+
+man page updated for new options
+
+-----------------------------------------------------------------
+Revision: 6f9a39297f610f7095473ac34f68674963561d9c
+Ancestor: 57f2c6cd4fbe8a04750aebeff72b094dc4d24476
+Author: angdraug@debian.org
+Date: 25/12/09 19:46:42
+Branch: syncache-head
+
+Modified files:
+        bin/syncache-drb
+
+ChangeLog: 
+
+New options: --user, --error-log
+
+ * Redirect STDERR to an error log when specified.
+ * Allow to specify any to run as user instead of just nobody.
+ * Create the pidfile under /var/run/syncache-drb to solve the
+   undeletable pidfile problem.
+
+-----------------------------------------------------------------
+Revision: 57f2c6cd4fbe8a04750aebeff72b094dc4d24476
+Ancestor: 0b3852cfbf276589043a4bb87c37ab024df694cc
+Author: angdraug@debian.org
+Date: 26/10/09 20:08:45
+Branch: syncache-head
+
+Modified files:
+        bin/syncache-drb
+
+ChangeLog: 
+
+untaint URI supplied on command line; report exception backtraces to syslog
+
+-----------------------------------------------------------------
+Revision: 0b3852cfbf276589043a4bb87c37ab024df694cc
+Ancestor: 69d9bb2d250a211e52baab031e5f840032fdee74
+Author: angdraug@debian.org
+Date: 07/08/09 14:08:50
+Branch: syncache-head
+
+Modified files:
+        lib/syncache.rb lib/syncache_sync_patch.rb
+
+ChangeLog: 
+
+expand syncache_sync_patch to fix the same bug in Ruby 1.9.0, move RUBY_VERSION condition into the patch file
+
+-----------------------------------------------------------------
+Revision: 69d9bb2d250a211e52baab031e5f840032fdee74
+Ancestor: de7c67553a07c221dc2064cc19a64c32286c72ae
+Author: angdraug@debian.org
+Date: 28/07/09 17:45:55
+Branch: syncache-head
+
+Added files:
+        test/ts_syncache.rb
+Added directories:
+        test
+
+ChangeLog: 
+
+unit test re-added from Samizdat::Cache
+
+-----------------------------------------------------------------
+Revision: de7c67553a07c221dc2064cc19a64c32286c72ae
+Ancestor: f95d143aed0ae773800a96cafb4e1b9eb87cc407
+Author: angdraug@debian.org
+Date: 28/07/09 17:29:16
+Branch: syncache-head
+
+Modified files:
+        bin/syncache-drb
+
+ChangeLog: 
+
+fixes in handling of syncache-drb options
+
+-----------------------------------------------------------------
+Revision: f95d143aed0ae773800a96cafb4e1b9eb87cc407
+Ancestor: ed2cc574a6324a95a69f81c4a3ed3e04bd26ade4
+Author: angdraug@debian.org
+Date: 27/07/09 17:50:10
+Branch: syncache-head
+
+Modified files:
+        README.rdoc
+
+ChangeLog: 
+
+updated README.rdoc to reflect renamed DRb server script
+
+-----------------------------------------------------------------
+Revision: ed2cc574a6324a95a69f81c4a3ed3e04bd26ade4
+Ancestor: e9c480978fdb812474c40a785d7b4cae8aa77c58
+Author: angdraug@debian.org
+Date: 27/07/09 17:32:17
+Branch: syncache-head
+
+Renamed entries:
+        man/syncache-drb-server.1 to man/syncache-drb.1
+Modified files:
+        man/syncache-drb.1
+
+ChangeLog: 
+
+renamed the samizdat-drb-server.1 manpage along with the script
+
+-----------------------------------------------------------------
+Revision: e9c480978fdb812474c40a785d7b4cae8aa77c58
+Ancestor: e6c0480854ee7af7f7f9956d9c6489452ec99a65
+Author: angdraug@debian.org
+Date: 27/07/09 17:16:05
+Branch: syncache-head
+
+Renamed entries:
+        bin/syncache-drb-server to bin/syncache-drb
+Modified files:
+        bin/syncache-drb
+
+ChangeLog: 
+
+renamed syncache-drb-server to syncache-drb to make it fit within 15 character limit
+
+-----------------------------------------------------------------
+Revision: e6c0480854ee7af7f7f9956d9c6489452ec99a65
+Ancestor: 6818ae15675fa101beb33c7fc21635e404a09fbe
+Author: angdraug@debian.org
+Date: 27/07/09 16:37:53
+Branch: syncache-head
+
+Modified files:
+        lib/syncache.rb
+
+ChangeLog: 
+
+conditionally load monkey patch for the sync.rb bug only for Ruby versions < 1.8.7p173
+
+-----------------------------------------------------------------
+Revision: 6818ae15675fa101beb33c7fc21635e404a09fbe
+Ancestor: 56a8b045280179815127c312f703c55bc63d7081
+Author: angdraug@debian.org
+Date: 27/07/09 15:57:09
+Branch: syncache-head
+
+Modified files:
+        bin/syncache-drb-server
+
+ChangeLog: 
+
+fixed Samizdat legacy: cache -> @cache
+
+-----------------------------------------------------------------
+Revision: 56a8b045280179815127c312f703c55bc63d7081
+Ancestor: 
+Author: angdraug@debian.org
+Date: 27/07/09 15:40:15
+Branch: syncache-head
+
+Added files:
+        COPYING README.rdoc bin/syncache-drb-server lib/syncache.rb
+        lib/syncache_sync_patch.rb man/syncache-drb-server.1
+        setup.rb
+Added directories:
+        . bin lib man
+
+ChangeLog: 
+
+initial checkin: SynCache is a spin-off of cache.rb from Samizdat project
+

data/README.rdoc

@@ -42,8 +42,8 @@ overriding its #replacement_index method.
 Cache entries are automatically invalidated when their +ttl+ (time to
 live) is exceeded. Entries can be explicitly invalidated by #flush
 method. The method can use <tt>===</tt> operator to compare cache keys
-against flush base (so that base can be e.g. a Regexp), and invalidates
-all entries when invoked without the +base+ parameter.
+against flush base (so that base can be e.g. a Regexp). When invoked
+without the +base+ parameter, it invalidates all entries.
 
 The +flush_delay+ initialization option allows to limit cache's flush
 rate. When this option is set, SynCache will make sure that at least
@@ -70,22 +70,29 @@ To access a remote cache, you will need to use DRb library:
   # allow remote cache to access local objects from fetch_or_add blocks
   DRb.start_service('druby://localhost:0')
 
+For the above to work properly, all your fetch_or_add blocks should be
+thread-safe, because DRb will run them in their own threads. Also, if a
+Ruby process crashes is the middle of such block, the key will remain
+locked in the remote cache until its +ttl+ runs out.
+
+To work around these limitations, you can wrap access to a remote cache
+using a SynCache::RemoteCache object:
+
+  require 'syncache'
+
+  # connect to the remote cache
+  @cache = SynCache::RemoteCache.new('druby://localhost:9000')
+
+SynCache::RemoteCache implements its own version of fetch_or_add that
+runs the supplied block locally in the current thread and would give up
+and take over a locked key if the client that originally locked it takes
+too long to free it up.
+
 
 == Copying
 
-  Copyright (c) 2002-2009  Dmitry Borodaenko <angdraug@debian.org>
+  Copyright (c) 2002-2011  Dmitry Borodaenko <angdraug@debian.org>
 
   This program is free software.
   You can distribute/modify this program under the terms of the GNU
   General Public License version 3 or later.
-  
-
-== Note on Patches/Pull Requests
- 
-* Fork the project.
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a
-  future version unintentionally.
-* Commit, do not mess with rakefile, version, or history.
-  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.

data/bin/syncache-drb

@@ -0,0 +1,204 @@
+#!/usr/bin/env ruby
+#
+# SynCache dRuby object cache server
+# (originally written for Samizdat project)
+#
+#   Copyright (c) 2002-2011  Dmitry Borodaenko <angdraug@debian.org>
+#
+#   This program is free software.
+#   You can distribute/modify this program under the terms of
+#   the GNU General Public License version 3 or later.
+#
+# vim: et sw=2 sts=2 ts=8 tw=0
+
+require 'getoptlong'
+require 'etc'
+require 'syslog'
+require 'drb'
+require 'syncache'
+
+class SynCacheDRb
+  PNAME = 'syncache-drb'
+
+  def usage
+    puts %{
+Usage: #{PNAME} [ options ] [ URI ]
+
+Options:
+
+  URI
+      URI with druby: schema that DRb server binds to, default is
+      druby://localhost:9000
+
+  --help
+      display this message and quit
+
+  --ttl SECONDS
+      time-to-live value for cache entries, default is 24 hours
+
+  --size ENTRIES
+      maximum number of objects in cache, default is 10000
+
+  --flush-delay SECONDS
+      rate-limit flush operations: if less than that number of seconds
+      has passed since last flush, next flush will be delayed; default
+      is no rate limit
+
+  --user USER
+      Run as USER if started as root. Default is nobody.
+
+  --error-log ERROR_LOG_PATH
+      File to write errors to. Default is /dev/null. When run as root,
+      the file is chowned to USER:adm.
+
+  --debug
+      Enable debug mode. If an error log is specified with --error-log,
+      all messages will be sent there instead of syslog.
+
+  --pidfile PATH
+      Path to the pidfile. By default, pidfile is created under
+      /var/run/#{PNAME}/ when run as root, or under $TMPDIR
+      otherwise. Location should be writeable by USER.
+
+}
+    exit
+  end
+
+  def set_options
+    opts = GetoptLong.new(
+      [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+      [ '--ttl', '-t', GetoptLong::REQUIRED_ARGUMENT ],
+      [ '--size', '-s', GetoptLong::REQUIRED_ARGUMENT ],
+      [ '--flush-delay', '-f', GetoptLong::REQUIRED_ARGUMENT ],
+      [ '--user', '-u', GetoptLong::REQUIRED_ARGUMENT ],
+      [ '--error-log', '-e', GetoptLong::REQUIRED_ARGUMENT ],
+      [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+      [ '--pidfile', '-p', GetoptLong::REQUIRED_ARGUMENT ]
+    )
+
+    @uri = 'druby://localhost:9000'
+    @ttl = 24*60*60
+    @size = 10000
+    @flush_delay = nil
+    @user = 'nobody'
+    @error_log = '/dev/null'
+    @debug = false
+    @pidfile = (0 == Process.uid) ?
+      "/var/run/#{PNAME}/#{PNAME}.pid" :
+      (ENV.has_key?('TMPDIR') ? ENV['TMPDIR'] : '/tmp') + "/#{PNAME}.pid"
+
+    opts.each do |opt, arg|
+      case opt
+      when '--help'
+        usage
+      when '--ttl'
+        @ttl = arg.to_i
+      when '--size'
+        @size = arg.to_i
+      when '--flush-delay'
+        @flush_delay = arg.to_i
+      when '--user'
+        @user = arg.dup.untaint
+      when '--error-log'
+        @error_log = arg.dup.untaint
+      when '--debug'
+        @debug = true
+      when '--pidfile'
+        @pidfile = arg.dup.untaint
+      end
+    end
+
+    @uri = ARGV[0].dup.untaint if ARGV[0]
+    @user = Etc.getpwnam(@user)
+  end
+
+  def log(message, level = :info)
+    if @log.respond_to? level
+      @log.send(level, message)
+    else
+      STDERR << 'syncache: ' + message + "\n"
+      STDERR.flush
+    end
+  end
+
+  def daemonize
+    $0 = PNAME
+    ENV.clear
+    exit if fork
+    Process.setsid
+    exit if fork
+    Dir.chdir '/'
+    File.umask 0022
+
+    STDIN.reopen '/dev/null'
+    STDOUT.reopen '/dev/null', 'a'
+    STDERR.reopen @error_log, 'a'
+
+    if 0 == Process.uid
+      if @error_log != '/dev/null'
+        File.chown(@user.uid, Etc.getgrnam('adm').gid, @error_log)
+        File.chmod(0640, @error_log)
+      end
+
+      # drop root priviledge
+      Process::Sys.setgid(@user.gid)
+      Process::Sys.setuid(@user.uid)
+    end
+
+    File.open(@pidfile, 'w') {|f| f.puts Process.pid }
+  end
+
+  def open_syslog
+    @log = Syslog.open PNAME
+  end
+
+  def ready
+    @cache = SynCache::Cache.new(@ttl, @size, @flush_delay)   # initialize cache
+    @cache.debug = true if @debug
+  end
+
+  def trap_signals
+    shutdown = lambda do |sig|
+      Thread.new do
+        sleep 0.1
+        DRb.stop_service
+      end
+    end
+    Signal.trap('INT', shutdown)
+    Signal.trap('TERM', shutdown)
+    Signal.trap('HUP') do |sig|   # kill -HUP to flush cache
+      @cache.flush
+      log 'cache flushed'
+    end
+  end
+
+  def start_service
+    $SAFE = 1
+    DRb.start_service(@uri, @cache)
+    log 'started'
+  end
+
+  def wait_for_shutdown
+    DRb.thread.join
+    File.delete @pidfile
+    log 'shut down'
+  end
+
+  def run
+    set_options
+    daemonize
+    open_syslog unless @debug and @error_log
+    begin
+      ready
+      trap_signals
+      start_service
+      wait_for_shutdown
+    rescue => error
+      log "#{error.class.to_s}: #{error.to_s}", :err
+      error.backtrace.each {|line| log('  ' + line, :err)  }
+      raise
+    end
+  end
+end
+
+SynCacheDRb.new.run

data/lib/syncache/remote.rb

@@ -0,0 +1,89 @@
+# SynCache: thread-safe time-limited cache with flexible replacement policy
+# (originally written for Samizdat project)
+#
+#   Copyright (c) 2002-2011  Dmitry Borodaenko <angdraug@debian.org>
+#
+#   This program is free software.
+#   You can distribute/modify this program under the terms of
+#   the GNU General Public License version 3 or later.
+#
+# vim: et sw=2 sts=2 ts=8 tw=0
+
+require 'drb'
+require 'delegate'
+
+module SynCache
+
+REMOTE_TIMEOUT = 60 * 5   # 5 minutes
+REMOTE_FIRST_DELAY = 0.1    # 100 ms
+
+class Placeholder
+  def initialize
+    @id = rand(9223372036854775808)
+    @timestamp = Time.now
+  end
+
+  attr_reader :id, :timestamp
+
+  def ===(other)
+    other.kind_of?(Placeholder) and other.id == @id
+  end
+end
+
+# Connects to a remote SynCache instance over DRb at the provided URI and
+# replaces the remote fetch_or_add method with a slightly less bullet-proof
+# version that invokes the supplied block locally (instead of sending it over
+# DRb to the cache and then back to a different local thread via a local DRb
+# service).
+#
+# If another RemoteCache client is already working on the same key, this client
+# will wait, using randomly increasing intervals. When a configured timeout
+# runs out (counting from the time the other client has put a placeholder in
+# the cache), the client will give up, discard the other client's placeholder
+# and start working on the key itself.
+#
+# Mixing access to the same cache entries from direct and RemoteCache clients
+# is not recommended.
+#
+class RemoteCache
+  def initialize(uri, timeout=REMOTE_TIMEOUT, first_delay=REMOTE_FIRST_DELAY)
+    @timeout = timeout
+    @first_delay = first_delay
+    @cache = DRbObject.new_with_uri(uri)
+  end
+
+  def method_missing(method, *args)
+    @cache.send(method, *args)
+  end
+
+  def fetch_or_add(key)
+    placeholder = Placeholder.new
+    value = @cache.fetch_or_add(key) { placeholder }
+
+    case value
+    when placeholder
+      # our placeholder
+      value = @cache[key] = yield
+
+    when Placeholder
+      # someone else's placeholder
+      delay = @first_delay
+      while value.kind_of?(Placeholder) and Time.now < value.timestamp + @timeout
+        sleep(delay)
+        delay *= 1 + rand
+        value = @cache[key]
+      end
+
+      if value.kind_of?(Placeholder)
+        # ran out of time: give up and do it ourselves
+        @cache.delete(key)
+        @cache[key] = Placeholder.new
+        value = @cache[key] = yield
+      end
+    end
+
+    value
+  end
+end
+
+end   # module SynCache