boned 0.2.6 → 0.3.0

data/config/redis-server.conf

@@ -1,68 +1,51 @@
-# Redis configuration file for Boned
+# BS REDIS 2.0 CONFIG (dev) -- 2010-11-17
 
-# Changes:
-# * port
-# * daemonize
-# * dbfilename
+# NOTE: NOTE: auto-generated by delano on tucker at 2010-11-29 11:49:17 -0500
 
-# By default Redis does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+dir /tmp
+
+pidfile boned-redis.pid
+logfile boned-redis.log
+dbfilename boned-redis.rdb
+
+port 8045
+bind 127.0.0.1
 daemonize yes
 
-# The filename where to dump the DB
-dbfilename /tmp/boned-redis.rdb
+timeout 300
 
-# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
-# You can specify a custom pid file location here.
-pidfile /var/run/redis.pid
+#loglevel debug
+#loglevel verbose
+loglevel warning
 
-# Accept connections on the specified port, default is 6379
-port 8045
+databases 16
 
-# If you want you can bind a single interface, if the bind option is not
-# specified all the interfaces will listen for connections.
-#
-# bind 127.0.0.1
+save 900 100
+save 300 5000
 
-# Close the connection after a client is idle for N seconds (0 to disable)
-timeout 300
 
-# Save the DB on disk:
-#
-#   save <seconds> <changes>
-#
-#   Will save the DB if both the given number of seconds and the given
-#   number of write operations against the DB occurred.
-#
-#   In the example below the behaviour will be to save:
-#   after 900 sec (15 min) if at least 1 key changed
-#   after 300 sec (5 min) if at least 10 keys changed
-#   after 60 sec if at least 10000 keys changed
-save 600 1
-save 300 10
-save 60 10000
+rdbcompression yes
 
+# requirepass foobared
+# maxclients 0
 
-# For default save/load DB in/from the working directory
-# Note that you must specify a directory not a file name.
-dir ./
+appendonly no
+appendfilename redis.aof
 
-# Set server verbosity to 'debug'
-# it can be one of:
-# debug (a lot of information, useful for development/testing)
-# notice (moderately verbose, what you want in production probably)
-# warning (only very important / critical messages are logged)
-loglevel debug
+# TODO: Consider having separate configs when the usecase for Redis
+# changes. For example, one for production, another for batch processing. 
+#
+# Nothing is changed from here on out:
 
-# Specify the log file name. Also 'stdout' can be used to force
-# the demon to log on the standard output. Note that if you use standard
-# output for logging but daemonize, logs will be sent to /dev/null
-logfile stdout
+################################## INCLUDES ###################################
 
-# Set the number of databases. The default database is DB 0, you can select
-# a different one on a per-connection basis using SELECT <dbid> where
-# dbid is a number between 0 and 'databases'-1
-databases 16
+# Include one or more other config files here.  This is useful if you
+# have a standard template that goes to all redis server but also need
+# to customize a few per-server settings.  Include files can include
+# other files, so use this wisely.
+#
+# include /path/to/local.conf
+# include /path/to/other.conf
 
 ################################# REPLICATION #################################
 
@@ -70,67 +53,16 @@ databases 16
 # another Redis server. Note that the configuration is local to the slave
 # so for example it is possible to configure the slave to save the DB with a
 # different interval, or to listen to another port, and so on.
-
-# slaveof <masterip> <masterport>
-
-################################## SECURITY ###################################
-
-# Require clients to issue AUTH <PASSWORD> before processing any other
-# commands.  This might be useful in environments in which you do not trust
-# others with access to the host running redis-server.
 #
-# This should stay commented out for backward compatibility and because most
-# people do not need auth (e.g. they run their own servers).
-
-# requirepass 
-
-################################### LIMITS ####################################
-
-# Set the max number of connected clients at the same time. By default there
-# is no limit, and it's up to the number of file descriptors the Redis process
-# is able to open. The special value '0' means no limts.
-# Once the limit is reached Redis will close all the new connections sending
-# an error 'max number of clients reached'.
-
-# maxclients 128
-
-# Don't use more memory than the specified amount of bytes.
-# When the memory limit is reached Redis will try to remove keys with an
-# EXPIRE set. It will try to start freeing keys that are going to expire
-# in little time and preserve keys with a longer time to live.
-# Redis will also try to remove objects from free lists if possible.
-#
-# If all this fails, Redis will start to reply with errors to commands
-# that will use more memory, like SET, LPUSH, and so on, and will continue
-# to reply to most read-only commands like GET.
-#
-# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
-# 'state' server or cache, not as a real DB. When Redis is used as a real
-# database the memory usage will grow over the weeks, it will be obvious if
-# it is going to use too much memory in the long run, and you'll have the time
-# to upgrade. With maxmemory after the limit is reached you'll start to get
-# errors for write operations, and this may even lead to DB inconsistency.
-
-# maxmemory <bytes>
-
-############################## APPEND ONLY MODE ###############################
+# slaveof <masterip> <masterport>
 
-# By default Redis asynchronously dumps the dataset on disk. If you can live
-# with the idea that the latest records will be lost if something like a crash
-# happens this is the preferred way to run Redis. If instead you care a lot
-# about your data and don't want to that a single record can get lost you should
-# enable the append only mode: when this mode is enabled Redis will append
-# every write operation received in the file appendonly.log. This file will
-# be read on startup in order to rebuild the full dataset in memory.
-#
-# Note that you can have both the async dumps and the append only file if you
-# like (you have to comment the "save" statements above to disable the dumps).
-# Still if append only mode is enabled Redis will load the data from the
-# log file at startup ignoring the dump.rdb file.
+# If the master is password protected (using the "requirepass" configuration
+# directive below) it is possible to tell the slave to authenticate before
+# starting the replication synchronization process, otherwise the master will
+# refuse the slave request.
 #
-# The name of the append only file is "appendonly.log"
+# masterauth <master-password>
 
-appendonly no
 
 # The fsync() call tells the Operating System to actually write data on disk
 # instead to wait for more data in the output buffer. Some OS will really flush 
@@ -142,16 +74,92 @@ appendonly no
 # always: fsync after every write to the append only log . Slow, Safest.
 # everysec: fsync only if one second passed since the last fsync. Compromise.
 #
-# The default is "always" that's the safer of the options. It's up to you to
-# understand if you can relax this to "everysec" that will fsync every second
-# or to "no" that will let the operating system flush the output buffer when
-# it want, for better performances (but if you can live with the idea of
-# some data loss consider the default persistence mode that's snapshotting).
+# The default is "everysec" that's usually the right compromise between
+# speed and data safety. It's up to you to understand if you can relax this to
+# "no" that will will let the operating system flush the output buffer when
+# it wants, for better performances (but if you can live with the idea of
+# some data loss consider the default persistence mode that's snapshotting),
+# or on the contrary, use "always" that's very slow but a bit safer than
+# everysec.
+#
+# If unsure, use "everysec".
 
-appendfsync always
-# appendfsync everysec
+# appendfsync always
+appendfsync everysec
 # appendfsync no
 
+################################ VIRTUAL MEMORY ###############################
+
+# Virtual Memory allows Redis to work with datasets bigger than the actual
+# amount of RAM needed to hold the whole dataset in memory.
+# In order to do so very used keys are taken in memory while the other keys
+# are swapped into a swap file, similarly to what operating systems do
+# with memory pages.
+#
+# To enable VM just set 'vm-enabled' to yes, and set the following three
+# VM parameters accordingly to your needs.
+
+vm-enabled no
+# vm-enabled yes
+
+# This is the path of the Redis swap file. As you can guess, swap files
+# can't be shared by different Redis instances, so make sure to use a swap
+# file for every redis process you are running. Redis will complain if the
+# swap file is already in use.
+#
+# The best kind of storage for the Redis swap file (that's accessed at random) 
+# is a Solid State Disk (SSD).
+#
+# *** WARNING *** if you are using a shared hosting the default of putting
+# the swap file under /tmp is not secure. Create a dir with access granted
+# only to Redis user and configure Redis to create the swap file there.
+vm-swap-file /tmp/redis.swap
+
+# vm-max-memory configures the VM to use at max the specified amount of
+# RAM. Everything that deos not fit will be swapped on disk *if* possible, that
+# is, if there is still enough contiguous space in the swap file.
+#
+# With vm-max-memory 0 the system will swap everything it can. Not a good
+# default, just specify the max amount of RAM you can in bytes, but it's
+# better to leave some margin. For instance specify an amount of RAM
+# that's more or less between 60 and 80% of your free RAM.
+vm-max-memory 0
+
+# Redis swap files is split into pages. An object can be saved using multiple
+# contiguous pages, but pages can't be shared between different objects.
+# So if your page is too big, small objects swapped out on disk will waste
+# a lot of space. If you page is too small, there is less space in the swap
+# file (assuming you configured the same number of total swap file pages).
+#
+# If you use a lot of small objects, use a page size of 64 or 32 bytes.
+# If you use a lot of big objects, use a bigger page size.
+# If unsure, use the default :)
+vm-page-size 32
+
+# Number of total memory pages in the swap file.
+# Given that the page table (a bitmap of free/used pages) is taken in memory,
+# every 8 pages on disk will consume 1 byte of RAM.
+#
+# The total swap size is vm-page-size * vm-pages
+#
+# With the default of 32-bytes memory pages and 134217728 pages Redis will
+# use a 4 GB swap file, that will use 16 MB of RAM for the page table.
+#
+# It's better to use the smallest acceptable value for your application,
+# but the default is large in order to work in most conditions.
+vm-pages 134217728
+
+# Max number of VM I/O threads running at the same time.
+# This threads are used to read/write data from/to swap file, since they
+# also encode and decode objects from disk to memory or the reverse, a bigger
+# number of threads can help with big objects even if they can't help with
+# I/O itself as the physical device may not be able to couple with many
+# reads/writes operations at the same time.
+#
+# The special value of 0 turn off threaded I/O and enables the blocking
+# Virtual Memory implementation.
+vm-max-threads 4
+
 ############################### ADVANCED CONFIG ###############################
 
 # Glue small output buffers together in order to send small replies in a
@@ -159,19 +167,29 @@ appendfsync always
 # in terms of number of queries per second. Use 'yes' if unsure.
 glueoutputbuf yes
 
-# Use object sharing. Can save a lot of memory if you have many common
-# string in your dataset, but performs lookups against the shared objects
-# pool so it uses more CPU and can be a bit slower. Usually it's a good
-# idea.
-#
-# When object sharing is enabled (shareobjects yes) you can use
-# shareobjectspoolsize to control the size of the pool used in order to try
-# object sharing. A bigger pool size will lead to better sharing capabilities.
-# In general you want this value to be at least the double of the number of
-# very common strings you have in your dataset.
-#
-# WARNING: object sharing is experimental, don't enable this feature
-# in production before of Redis 1.0-stable. Still please try this feature in
-# your development environment so that we can test it better.
-shareobjects no
-shareobjectspoolsize 1024
+# Hashes are encoded in a special way (much more memory efficient) when they
+# have at max a given numer of elements, and the biggest element does not
+# exceed a given threshold. You can configure this limits with the following
+# configuration directives.
+hash-max-zipmap-entries 64
+hash-max-zipmap-value 512
+
+# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
+# order to help rehashing the main Redis hash table (the one mapping top-level
+# keys to values). The hash table implementation redis uses (see dict.c)
+# performs a lazy rehashing: the more operation you run into an hash table
+# that is rhashing, the more rehashing "steps" are performed, so if the
+# server is idle the rehashing is never complete and some more memory is used
+# by the hash table.
+# 
+# The default is to use this millisecond 10 times every second in order to
+# active rehashing the main dictionaries, freeing memory when possible.
+#
+# If unsure:
+# use "activerehashing no" if you have hard latency requirements and it is
+# not a good thing in your environment that Redis can reply form time to time
+# to queries with 2 milliseconds delay.
+#
+# use "activerehashing yes" if you don't have such hard requirements but
+# want to free memory asap when possible.
+activerehashing yes

data/lib/boned/api/base.rb

@@ -0,0 +1,163 @@
+require 'sinatra/reloader'  # consider big_band
+
+class Boned::APIBase < Sinatra::Base
+  
+  set :public => 'public/'
+  set :views => 'views/'
+  set :static => true
+  
+  configure :dev do
+    Bone.info "Environment: #{ENV['RACK_ENV']}"
+    register Sinatra::Reloader
+    dont_reload "lib/**/*.rb"
+    also_reload "lib/boned.rb"
+    before do
+      #Bone.debug = true
+      Bone.info $/, $/, "--> #{request_method} #{current_uri_path}"
+      content_type 'text/plain'
+      Boned.allow_register = true
+    end
+  end
+  
+  configure :prod do
+    Bone.debug = false
+    before do
+      content_type 'application/json'
+      Boned.allow_register = false
+    end
+  end
+  
+  helpers do
+    def carefully(ret='', &blk)
+      begin
+        ret = blk.call
+      rescue => ex
+        generic_error ex.class, ex
+      end
+      ret
+    end 
+    
+    def generic_error(event=nil, ex=nil)
+      Bone.info "#{event} #{request_token}:#{request_signature}" unless event.nil?
+      unless ex.nil?
+        Bone.info ex.message
+        Bone.ld ex.backtrace 
+      end
+      return error(404, "Bad bone rising")
+    end
+    
+    def error_message msg
+      Bone.info "[400] #{msg}"
+      return error(400, msg)
+    end
+    
+    def request_token
+      env['HTTP_X_BONE_TOKEN'] || params[:token] 
+    end
+    def request_signature
+      @request_signature ||= params[:sig] || env['HTTP_X_BONE_SIGNATURE']
+      @request_signature
+    end
+    def request_secret
+      # no leading/trail whitspace end
+      @request_secret ||= (body_content || env['HTTP_X_BONE_SECRET']).strip 
+      @request_secret
+    end
+    def body_content
+      @body_content ||= request.body.read 
+      @body_content
+    end
+    
+    def uri(*path)
+      [root_path, path].flatten.join('/')
+    end
+    def root_path
+      env['SCRIPT_NAME']
+    end
+    def current_uri_path
+      env['REQUEST_URI']
+    end
+    def request_method
+      env['REQUEST_METHOD'].to_s.downcase  # important to be downcase for signature check
+    end
+    def current_host
+      env['HTTP_HOST'].to_s.downcase
+    end
+    def secure?
+      (env['HTTP_X_SCHEME'] == "https")  # X-Scheme is set by nginx
+    end
+    
+    def local?
+      LOCAL_HOSTS.member?(env['SERVER_NAME']) && (client_ipaddress == '127.0.0.1')
+    end
+    
+    def assert_token
+      assert_exists request_token, "No token"
+    end
+    
+    def assert_secret
+      assert_exists request_secret, "No secret"
+    end
+    
+    def check_token
+      generic_error "[unknown-token]" if !Bone.token? request_token
+      true
+    end
+    
+    def check_signature
+      assert_exists request_signature, "No signature"
+      unless params[:sigversion] == Bone::API::HTTP::SIGVERSION
+        error_message "API must be version: #{Bone::API::HTTP::SIGVERSION}"
+      end
+      # We need to re-parse the query string b/c Sinatra or Rack is
+      # including the value of the POST body as a key with no value.
+      qs = Bone::API::HTTP.parse_query request.query_string
+      qs.delete 'sig' # Yo dawg, I put a signature in your signature
+      stamp, now = (qs['stamp'] || 0).to_i, Bone::API::HTTP.canonical_time
+      generic_error "[sig-expired] #{stamp}" if (now - stamp) > 30.seconds
+      tobj = Bone::API::Redis::Token.new request_token
+      secret = tobj.secret.value
+      path = current_uri_path.split('?').first
+      sig = Bone::API::HTTP.generate_signature secret, current_host, request_method, path, qs, body_content
+      generic_error "[sig-mismatch] #{sig}" if sig != request_signature
+      Bone.new request_token
+    end
+
+    
+    # +names+ One or more a required parameter names (Symbol)
+    def assert_params(*names)
+      names.each do |n|
+        return error_message("Missing param: %s" % n) if params[n].to_s.empty?
+      end
+      true
+    end
+    alias_method :assert_param, :assert_params
+    
+    def assert_exists(val, msg)
+      return error_message msg if val.to_s.empty?
+      true
+    end
+    
+    def assert_true(val, msg)
+      return error_message msg if val != true
+      true
+    end
+    
+    def assert_sha1(val)
+      return error_message("#{val} is not a sha1 digest") unless is_sha1?(val)
+    end
+    
+    def assert_sha256(val)
+      return error_message("#{val} is not a sha256 digest") unless is_sha256?(val)
+    end
+    
+    def is_sha1?(val)
+      val.to_s.match(/\A[0-9a-f]{40}\z/)
+    end
+    def is_sha256?(val)
+      val.to_s.match(/\A[0-9a-f]{64}\z/)
+    end
+    
+  end
+end
+

data/lib/boned/api.rb

@@ -1,94 +1,101 @@
-
 require 'boned'
+require 'boned/api/base'
 
+class Boned::API < Boned::APIBase
 
-class Boned::API < Sinatra::Base
+  # TODO: Remove these.
+  ##get '/all' do
+  ##  keys = Bone::API::Redis::Token.redis.keys '*'
+  ##  keys.join $/
+  ##end  
+  ##get "/:token/secret/?" do
+  ##  carefully do
+  ##    assert_token && check_token
+  ##    bone = check_signature
+  ##    bone.secret
+  ##  end
+  ##end
   
-  set :public => 'public/'
-  set :views => 'views/'
-  set :static => true
+  #get "/:token/:bucket/keys/?" do
+  #  Bone.info 
+  #  "poop"
+  #end
+  #
+  #get "/:token/:bucket/key/:key/?" do
+  #  Bone.info 
+  #  "poop"
+  #end
   
-  configure :development do
-    before do
-      Boned.enable_debug
-      Boned.ld '  --> ' << env['REQUEST_URI']
-      content_type 'text/plain'
+  get "/:token/key/:key/?" do
+    carefully do
+      assert_token && check_token
+      bone = check_signature
+      bone.key?(params[:key]) ? bone[params[:key]] : generic_error
     end
   end
   
-  configure :production do
-    Boned.disable_debug
-    before do
-      content_type 'application/json'
+  get "/:token/keys/?" do
+    carefully do
+      assert_token && check_token
+      bone = check_signature
+      list = bone.keys || []
+      list.join $/
     end
   end
   
-  helpers do
-    def carefully(ret='', &blk)
-      begin
-        ret = blk.call
-      rescue Boned::BadBone => ex
-        return error(404, ex.message)
-      rescue => ex
-        Boned.ld "#{current_token}:#{params[:key]}", ex.message
-        Boned.ld ex.backtrace
-        return error(400, "Bad bone rising")
-      end
-      ret
-    end 
-    
-    def current_token() @env['HTTP_X_BONE_TOKEN'] || params[:token] end
-    def current_sig() @env['HTTP_X_BONE_SIGNATURE'] || params[:sig] end
-  
-    def uri(*path)
-      [root_path, path].flatten.join('/')
+  get "/:token/?" do
+    carefully do
+      assert_token && check_token
+      bone = check_signature
+      # list of buckets, currently hardcoded to global
+      bone.token?(request_token) ? 'global' : generic_error
     end
-    def root_path
-      env['SCRIPT_NAME']
-    end
-    
-    # +names+ One or more a required parameter names (Symbol)
-    def assert_params(*names)
-      names.each do |n|
-        if params[n].nil? || params[n].empty?
-          return error(400, "Missing param: %s" % n)
-        end
-      end
-    end
-    alias_method :assert_param, :assert_params
-
-    def assert_exists(val, msg)
-      return error(400, msg) if val.nil? || 
-                                      (val.respond_to?(:empty?) && val.empty?)
-    end
-
-    def assert_true(val, msg)
-      return error(400, msg) if val == true
-    end
-    
-    def assert_sha1(val)
-      return error(400, "#{val} is not a sha1 digest") unless is_sha1?(val)
+  end
+  
+  post "/:token/key/:key/?" do
+    carefully do
+      assert_token && check_token
+      bone = check_signature
+      value = body_content # don't modify content in any way
+      bone.set params[:key], value
     end
-    
-    def assert_sha256(val)
-      return error(400, "#{val} is not a sha256 digest") unless is_sha256?(val)
+  end
+  
+  delete "/destroy/:token/?" do
+    carefully do
+      assert_token && check_token
+      bone = check_signature
+      Bone.destroy request_token
     end
-    
-    def is_sha1?(val)
-      val.match(/\A[0-9a-f]{40}\z/)
+  end
+  
+  post "/generate/?" do
+    carefully do
+      token, secret = *Bone.generate
+      token.nil? ? generic_error : [token, secret].join($/)
     end
-    def is_sha256?(val)
-      val.match(/\A[0-9a-f]{64}\z/)
+  end
+  
+  post "/register/:token/?" do
+    carefully do
+      generic_error '[register-disabled]' unless Boned.allow_register
+      assert_secret
+      generic_error "[rereg-attempt]" if Bone.token? request_token
+      token = Bone.register request_token, request_secret
+      token.nil? ? generic_error("[register-failed]") : token
     end
-    
+  end
+  
+  helpers do
+    #Bone.debug = true
   end
 end
 
-class Boned::API::Stub < Boned::API
+
+class Boned::API::Stub < Boned::APIBase
   get '/' do
     content_type 'text/plain'
     "Do you want to get bones?"
   end
 end
 
-require 'boned/api/service'