bone 0.2.6 → 0.3.0

data/config/redis-server.conf

@@ -0,0 +1,195 @@
+# BS REDIS 2.0 CONFIG (dev) -- 2010-11-17
+
+# NOTE: NOTE: auto-generated by delano on tucker at 2010-11-29 11:49:17 -0500
+
+dir /tmp
+
+pidfile boned-redis.pid
+logfile boned-redis.log
+dbfilename boned-redis.rdb
+
+port 8045
+bind 127.0.0.1
+daemonize yes
+
+timeout 300
+
+#loglevel debug
+#loglevel verbose
+loglevel warning
+
+databases 16
+
+save 900 100
+save 300 5000
+
+
+rdbcompression yes
+
+# requirepass foobared
+# maxclients 0
+
+appendonly no
+appendfilename redis.aof
+
+# TODO: Consider having separate configs when the usecase for Redis
+# changes. For example, one for production, another for batch processing. 
+#
+# Nothing is changed from here on out:
+
+################################## INCLUDES ###################################
+
+# Include one or more other config files here.  This is useful if you
+# have a standard template that goes to all redis server but also need
+# to customize a few per-server settings.  Include files can include
+# other files, so use this wisely.
+#
+# include /path/to/local.conf
+# include /path/to/other.conf
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+#
+# slaveof <masterip> <masterport>
+
+# If the master is password protected (using the "requirepass" configuration
+# directive below) it is possible to tell the slave to authenticate before
+# starting the replication synchronization process, otherwise the master will
+# refuse the slave request.
+#
+# masterauth <master-password>
+
+
+# The fsync() call tells the Operating System to actually write data on disk
+# instead to wait for more data in the output buffer. Some OS will really flush 
+# data on disk, some other OS will just try to do it ASAP.
+#
+# Redis supports three different modes:
+#
+# no: don't fsync, just let the OS flush the data when it wants. Faster.
+# always: fsync after every write to the append only log . Slow, Safest.
+# everysec: fsync only if one second passed since the last fsync. Compromise.
+#
+# The default is "everysec" that's usually the right compromise between
+# speed and data safety. It's up to you to understand if you can relax this to
+# "no" that will will let the operating system flush the output buffer when
+# it wants, for better performances (but if you can live with the idea of
+# some data loss consider the default persistence mode that's snapshotting),
+# or on the contrary, use "always" that's very slow but a bit safer than
+# everysec.
+#
+# If unsure, use "everysec".
+
+# appendfsync always
+appendfsync everysec
+# appendfsync no
+
+################################ VIRTUAL MEMORY ###############################
+
+# Virtual Memory allows Redis to work with datasets bigger than the actual
+# amount of RAM needed to hold the whole dataset in memory.
+# In order to do so very used keys are taken in memory while the other keys
+# are swapped into a swap file, similarly to what operating systems do
+# with memory pages.
+#
+# To enable VM just set 'vm-enabled' to yes, and set the following three
+# VM parameters accordingly to your needs.
+
+vm-enabled no
+# vm-enabled yes
+
+# This is the path of the Redis swap file. As you can guess, swap files
+# can't be shared by different Redis instances, so make sure to use a swap
+# file for every redis process you are running. Redis will complain if the
+# swap file is already in use.
+#
+# The best kind of storage for the Redis swap file (that's accessed at random) 
+# is a Solid State Disk (SSD).
+#
+# *** WARNING *** if you are using a shared hosting the default of putting
+# the swap file under /tmp is not secure. Create a dir with access granted
+# only to Redis user and configure Redis to create the swap file there.
+vm-swap-file /tmp/redis.swap
+
+# vm-max-memory configures the VM to use at max the specified amount of
+# RAM. Everything that deos not fit will be swapped on disk *if* possible, that
+# is, if there is still enough contiguous space in the swap file.
+#
+# With vm-max-memory 0 the system will swap everything it can. Not a good
+# default, just specify the max amount of RAM you can in bytes, but it's
+# better to leave some margin. For instance specify an amount of RAM
+# that's more or less between 60 and 80% of your free RAM.
+vm-max-memory 0
+
+# Redis swap files is split into pages. An object can be saved using multiple
+# contiguous pages, but pages can't be shared between different objects.
+# So if your page is too big, small objects swapped out on disk will waste
+# a lot of space. If you page is too small, there is less space in the swap
+# file (assuming you configured the same number of total swap file pages).
+#
+# If you use a lot of small objects, use a page size of 64 or 32 bytes.
+# If you use a lot of big objects, use a bigger page size.
+# If unsure, use the default :)
+vm-page-size 32
+
+# Number of total memory pages in the swap file.
+# Given that the page table (a bitmap of free/used pages) is taken in memory,
+# every 8 pages on disk will consume 1 byte of RAM.
+#
+# The total swap size is vm-page-size * vm-pages
+#
+# With the default of 32-bytes memory pages and 134217728 pages Redis will
+# use a 4 GB swap file, that will use 16 MB of RAM for the page table.
+#
+# It's better to use the smallest acceptable value for your application,
+# but the default is large in order to work in most conditions.
+vm-pages 134217728
+
+# Max number of VM I/O threads running at the same time.
+# This threads are used to read/write data from/to swap file, since they
+# also encode and decode objects from disk to memory or the reverse, a bigger
+# number of threads can help with big objects even if they can't help with
+# I/O itself as the physical device may not be able to couple with many
+# reads/writes operations at the same time.
+#
+# The special value of 0 turn off threaded I/O and enables the blocking
+# Virtual Memory implementation.
+vm-max-threads 4
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes
+
+# Hashes are encoded in a special way (much more memory efficient) when they
+# have at max a given numer of elements, and the biggest element does not
+# exceed a given threshold. You can configure this limits with the following
+# configuration directives.
+hash-max-zipmap-entries 64
+hash-max-zipmap-value 512
+
+# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
+# order to help rehashing the main Redis hash table (the one mapping top-level
+# keys to values). The hash table implementation redis uses (see dict.c)
+# performs a lazy rehashing: the more operation you run into an hash table
+# that is rhashing, the more rehashing "steps" are performed, so if the
+# server is idle the rehashing is never complete and some more memory is used
+# by the hash table.
+# 
+# The default is to use this millisecond 10 times every second in order to
+# active rehashing the main dictionaries, freeing memory when possible.
+#
+# If unsure:
+# use "activerehashing no" if you have hard latency requirements and it is
+# not a good thing in your environment that Redis can reply form time to time
+# to queries with 2 milliseconds delay.
+#
+# use "activerehashing yes" if you don't have such hard requirements but
+# want to free memory asap when possible.
+activerehashing yes

data/lib/bone.rb

@@ -1,180 +1,171 @@
-require 'uri'
-require 'net/http'
-require 'net/https'
 
 unless defined?(BONE_HOME)
   BONE_HOME = File.expand_path(File.join(File.dirname(__FILE__), '..') )
 end
 
-module Bone
-  extend self
-  VERSION = "0.2.6"
-  APIVERSION = 'v1'.freeze
-  
+local_libs = %w{familia}
+local_libs.each { |dir| 
+  a = File.join(BONE_HOME, '..', '..', 'opensource', dir, 'lib')
+  $:.unshift a
+}
+
+require 'familia'
+require 'base64'
+require 'openssl'
+require 'time'
+
+class Bone
+  module VERSION
+    def self.to_s
+      load_config
+      [@version[:MAJOR], @version[:MINOR], @version[:PATCH]].join('.')
+    end
+    alias_method :inspect, :to_s
+    def self.load_config
+      require 'yaml'
+      @version ||= YAML.load_file(File.join(BONE_HOME, '..', 'VERSION.yml'))
+    end
+  end
+end
+
+
+class Bone
+  unless defined?(Bone::APIVERSION)
+    APIVERSION = 'v2'.freeze 
+    SECRETCHAR = [('a'..'z'),('A'..'Z'),(0..9)].map(&:to_a).flatten.freeze
+  end
+  @source = URI.parse(ENV['BONE_SOURCE'] || 'redis://127.0.0.1:6379/')
+  @apis = {}
+  @digest_type = OpenSSL::Digest::SHA256
   class Problem < RuntimeError; end
-  class BadBone < Problem; end
-  
-  @digest_type = nil  # set at the end
-  @debug = false
+  class NoToken < Problem; end  
   class << self
-    attr_accessor :digest_type
-    def enable_debug()  @debug = true  end
-    def disable_debug() @debug = false end
-    def debug?()        @debug == true end
-    def ld(*msg)
-      return unless Bone.debug?
-      prefix = "D(#{Thread.current.object_id}):  "
-      STDERR.puts "#{prefix}" << msg.join("#{$/}#{prefix}")
+    attr_accessor :debug
+    attr_reader :apis, :api, :source, :digest_type
+    attr_writer :token, :secret
+    
+    def source= v
+      @source = URI.parse v
+      select_api
     end
-  end
-  
-  SOURCE = (ENV['BONE_SOURCE'] || "http://localhost:6043").freeze
-  TOKEN = ENV['BONE_TOKEN'].freeze
-  
-  # Get a key from the boned server. Same as `get!`  
-  # but does not raise an exception for an unknown key.
-  def get(key, opts={})
-    get! key, opts
-  rescue Bone::Problem
-    nil
-  end
-  
-  # Get a key from the boned server. Raises an exception 
-  # for an unknown key.
-  def get!(key, opts={})
-    token = opts[:token] || ENV['BONE_TOKEN'] || TOKEN
-    request(:get, token, key) # returns the response body
-  end
-  
-  def set(key, value, opts={})
-    set! key, value, opts
-  rescue Bone::Problem
-    nil
-  end
-  
-  def set!(key, value, opts={})
-    token = opts[:token] || ENV['BONE_TOKEN'] || TOKEN
-    opts[:value] = value
-    request(:set, token, key, opts)
-    key # return the key b/c it could be a binary file
-  end
-  
-  def del(key, opts={})
-    token = opts[:token] || ENV['BONE_TOKEN'] || TOKEN
-    request(:del, token, key, opts) # returns the response body
-  end
-  
-  def [](keyname)
-    get(keyname)
-  end
-  
-  def []=(keyname, value)
-    set(keyname, value)
-  end
-  
-  def keys(keyname=nil, opts={})
-    token = opts[:token] || ENV['BONE_TOKEN'] || TOKEN
-    k = request(:keys, token, keyname, opts)
-    k.split($/)
-  end
-  
-  # <tt>require</tt> a library from the vendor directory.
-  # The vendor directory should be organized such
-  # that +name+ and +version+ can be used to create
-  # the path to the library. 
-  #
-  # e.g.
-  # 
-  #     vendor/httpclient-2.1.5.2/httpclient
-  #
-  def require_vendor(name, version)
-    path = File.join(BONE_HOME, 'vendor', "#{name}-#{version}", 'lib')
-    $:.unshift path
-    Bone.ld "REQUIRE VENDOR: ", path
-    require name
-  end
-  
-  def valid_token?(val)
-    is_sha256? val
-  end
-  
-  def is_sha1?(val)
-    val.to_s.match /\A[0-9a-f]{40}\z/
-  end
-  
-  def is_sha256?(val)
-    val.to_s.match /\A[0-9a-f]{64}\z/
-  end
-  
-  def digest(val)
-    @digest_type.hexdigest val
-  end
-  
-  def generate_token
-    srand
-    digest [`hostname`, `w`, Time.now, rand].join(':')
-  end
-  alias_method :token, :generate_token
-  
-  private
-  
-  def request(action, token, key, params={})
-    params[:token] = token
-    path = "/bone/#{APIVERSION}/#{action}/#{key}"
-    source = SOURCE
-    unless source.match(/\Ahttp/)
-      source = ['http:', SOURCE].join('//')
-    end
-    uri = URI.parse source
-    host, port = uri.host, uri.port
-    port ||= (uri.scheme == 'https' ? 443 : 6043)
+    alias_method :src=, :source=
+    alias_method :src, :source
     
-    Bone.ld "SOURCE: #{uri.to_s}"
-    Bone.ld "URI: #{path}"
-    Bone.ld "PARAMS: " << params.inspect
+    # e.g.
+    #
+    #  Bone.cred = 'token:secret'
+    #
+    def credentials= token
+      @token, @secret = *token.split(':')
+    end
+    alias_method :cred=, :credentials=
     
-    case action
-    when :del
-      headers = { 'X-BONE_TOKEN' => token }
-      req = Net::HTTP::Delete.new(path, headers)
-    when :set
-      query = {}
-      params.each_pair {|n,v| query[n.to_s] = v } 
-      req = Net::HTTP::Post.new(path)
-      req.set_form_data query
-    when :get, :keys
-      args = []
-      params.each_pair {|n,v| args << "#{n}=#{URI.encode(v.to_s)}" }     
-      query = [path, args.join('&')].join('?') 
-      Bone.ld "GET: #{query}"
-      req = Net::HTTP::Get.new(query)
-    else
-      raise Bone::Problem, "Unknown action: #{action}"
-    end
-    http = Net::HTTP.new(host, port)
-    http.use_ssl = true if uri.scheme == 'https'
-    http.verify_mode = OpenSSL::SSL::VERIFY_PEER 
-    res = http.request(req) 
-    case res
-    when Net::HTTPSuccess, Net::HTTPRedirection
-      res.body
-    else
-      raise Bone::Problem, "#{res.body} (#{res.code} #{res.message})"
+    def token
+      @token || ENV['BONE_TOKEN']
     end
-  rescue Errno::ECONNREFUSED => ex
-    raise Bone::Problem, "No boned"
-  end
-  
-  def determine_digest_type
-    if RUBY_PLATFORM == "java"
-      require 'openssl'
-      Bone.digest_type = OpenSSL::Digest::SHA256
+    
+    def secret 
+      @secret || ENV['BONE_SECRET']
+    end
+    
+    def info *msg
+      STDERR.puts *msg
+    end
+    
+    def ld *msg
+      info *msg if debug
+    end
+    
+    # Stolen from Rack::Utils which stole it from Camping.
+    def uri_escape s
+      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+        '%'+$1.unpack('H2'*bytesize($1)).join('%').upcase
+      }.tr(' ', '+')
+    end
+    
+    # Stolen from Rack::Utils which stole it from Camping.
+    def uri_unescape s
+      s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
+        [$1.delete('%')].pack('H*')
+      }
+    end
+    
+    # Return the bytesize of String; uses String#size under Ruby 1.8 and
+    # String#bytesize under 1.9.
+    if ''.respond_to?(:bytesize)
+      def bytesize s
+        s.bytesize
+      end
     else
-      require 'digest'
-      Bone.digest_type = Digest::SHA256
+      def bytesize s
+        s.size
+      end
+    end
+    
+    def is_sha1? val
+      val.to_s.match /\A[0-9a-f]{40}\z/
+    end
+
+    def is_sha256? val
+      val.to_s.match /\A[0-9a-f]{64}\z/
+    end
+
+    def digest val, type=nil
+      type ||= @digest_type
+      type.hexdigest val
+    end
+    
+    def random_token
+      p1 = (0...21).map{ SECRETCHAR[rand(SECRETCHAR.length)] }.join
+      p2 = Bone.api.token_suffix
+      p3 = (0...2).map{ SECRETCHAR[rand(SECRETCHAR.length)] }.join
+      [p1,p2,p3].join.upcase
+    end
+    
+    def random_secret 
+      src = [SECRETCHAR, %w'* ^ $ ! / . - _ + %'].flatten
+      p1 = (0...2).map{ SECRETCHAR[rand(SECRETCHAR.length)] }.join
+      p2 = (0...60).map{ src[rand(src.length)] }.join
+      p3 = (0...2).map{ SECRETCHAR[rand(SECRETCHAR.length)] }.join      
+      [p1,p2,p3].join
+    end
+    
+    def select_api
+      begin
+        @api = Bone.apis[Bone.source.scheme.to_sym]
+        raise RuntimeError, "Bad source: #{Bone.source}" if api.nil?
+        @api.connect
+      rescue => ex
+        Bone.info "#{ex.class}: #{ex.message}", ex.backtrace
+        exit
+      end
+    end
+    
+    def register_api scheme, klass
+      Bone.apis[scheme.to_sym] = klass
+    end
+    
+    # <tt>require</tt> a library from the vendor directory.
+    # The vendor directory should be organized such
+    # that +name+ and +version+ can be used to create
+    # the path to the library. 
+    #
+    # e.g.
+    # 
+    #     vendor/httpclient-2.1.5.2/httpclient
+    #
+    def require_vendor name, version
+      path = File.join(BONE_HOME, 'vendor', "#{name}-#{version}", 'lib')
+      $:.unshift path
+      Bone.ld "REQUIRE VENDOR: ", path
+      require name
     end
   end
   
-  @digest_type = determine_digest_type
+  require 'bone/api'
+  include Bone::API::InstanceMethods
+  extend Bone::API::ClassMethods
+  select_api
 end
 
+