io_shuten 0.0.3.dev1 → 0.1.0.dev6

data/Gemfile

@@ -2,12 +2,19 @@ source "https://rubygems.org"
 
 gem "iobuffer", "~> 1.0.0"
 
+# REDIS STORE
 gem "hiredis", "~> 0.4.4"
 gem "redis", "~> 2.2.2", :require => ["redis/connection/hiredis", "redis"]
+gem "redis-namespace"
 
+# MONGO STORE
 gem "bson_ext", "~> 1.5.2"
 gem "mongo", "~> 1.5.2"
 
+# ZMQ "STORE"
+gem "ffi"
+gem "ffi-rzmq"
+
 group :development, :test do
   gem "rake"
   gem "jeweler"
@@ -22,6 +29,8 @@ group :development, :test do
   gem "yard-blame"
 
   gem "pry"
-  #gem "pry-doc"
-end
 
+  gem "alf"
+  gem "fastercsv"
+  gem "viiite"
+end

data/Gemfile.lock

@@ -1,31 +1,46 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    alf (0.10.1)
+      highline (~> 1.6.2)
+      myrrha (~> 1.2.1)
+      quickl (~> 0.4.1)
     bson (1.5.2)
     bson_ext (1.5.2)
       bson (= 1.5.2)
     coderay (0.9.8)
     diff-lcs (1.1.3)
+    fastercsv (1.5.4)
+    ffi (1.0.11)
+    ffi-rzmq (0.9.3)
+      ffi
     git (1.2.5)
+    gnuplot (2.3.6)
+    highline (1.6.11)
     hiredis (0.4.4)
     iobuffer (1.0.0)
     jeweler (1.6.4)
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
+    json (1.6.5)
     method_source (0.6.7)
       ruby_parser (>= 2.3.1)
     mongo (1.5.2)
       bson (= 1.5.2)
     multi_json (1.0.4)
+    myrrha (1.2.2)
     pry (0.9.7.4)
       coderay (~> 0.9.8)
       method_source (~> 0.6.7)
       ruby_parser (>= 2.3.1)
       slop (~> 2.1.0)
+    quickl (0.4.2)
     rake (0.9.2.2)
     redcarpet (2.0.1)
     redis (2.2.2)
+    redis-namespace (1.1.0)
+      redis (< 3.0.0)
     rspec (2.8.0)
       rspec-core (~> 2.8.0)
       rspec-expectations (~> 2.8.0)
@@ -46,6 +61,10 @@ GEM
     simplecov-rcov (0.2.3)
       simplecov (>= 0.4.1)
     slop (2.1.0)
+    viiite (0.2.0)
+      alf (~> 0.10.1)
+      gnuplot (~> 2.3.6)
+      json
     yard (0.7.4)
     yard-blame (0.0.1)
       yard
@@ -54,7 +73,11 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  alf
   bson_ext (~> 1.5.2)
+  fastercsv
+  ffi
+  ffi-rzmq
   hiredis (~> 0.4.4)
   iobuffer (~> 1.0.0)
   jeweler
@@ -63,9 +86,11 @@ DEPENDENCIES
   rake
   redcarpet
   redis (~> 2.2.2)
+  redis-namespace
   rspec (~> 2.8.0)
   simplecov
   simplecov-csv
   simplecov-rcov
+  viiite
   yard
   yard-blame

data/README.md

@@ -59,6 +59,13 @@ Stores content in a mongodb database.
 Provides basic methods like `#read`, `#write` and `#close`.
 A logger only needs write and close methods of an IO object.
 
+### IO_shuten::Zmq
+
+"Stores" content in a zmq socket.
+
+Provides basic methods like `#read`, `#write` and `#close`.
+A logger only needs write and close methods of an IO object.
+
 
 ## Not for production!
 

data/Rakefile

@@ -93,5 +93,11 @@ end
 
 
 
-task :default => [:envinfo,:spec]
+desc "Runs complex viiite benchmark suite"
+task :benchmark do
+  sh "./benchmarks.sh"
+end
+
 
+
+task :default => [:envinfo,:spec]

data/benchmark/viiite-template.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+require "rubygems"
+require "viiite"
+require "bundler/setup"
+require "io_shuten"
+require "logger"
+
+lorem_ipsum = <<LOREM
+Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod
+tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.
+At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,
+no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet,
+consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et
+dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo
+dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem
+ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
+diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
+voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd
+gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+LOREM
+
+tmp_path = File.expand_path("../../tmp", __FILE__)
+Dir.mkdir(tmp_path) unless File.exists?(tmp_path)
+
+ioc = IO_shuten.const_get $CUR_IMPL
+runs = 1..5
+internal_loops = 1_024
+
+Viiite.bench do |b|
+  b.variation_point :ruby, Viiite.which_ruby
+  b.variation_point :type, $CUR_IMPL
+
+  b.range_over(runs, :run) do |run|
+
+    ioc.purge_instances!
+    b.report :writes do
+      internal_loops.times do |i|
+        iob = ioc.new("wbuff-#{run}-#{i}")
+        iob.write lorem_ipsum
+      end
+    end
+
+    ioc.purge_instances!
+    b.report :logging do
+      internal_loops.times do |i|
+        logdev = ioc.new("logdev-#{run}-#{i}")
+        logger = Logger.new(logdev)
+        logger.info lorem_ipsum
+        logger.debug "counter: #{run}-#{i}"
+      end
+    end
+
+    ioc.purge_instances!
+    b.report :file_writes do
+      internal_loops.times do |i|
+        iob = ioc.new("#{tmp_path}/wbuff-#{run}-#{i}")
+        iob.write lorem_ipsum
+        iob.save_to_file
+      end
+    end
+
+    ioc.purge_instances!
+    b.report :file_reads do
+      internal_loops.times do |i|
+        iob = ioc.new("#{tmp_path}/wmem-#{run}-#{i}")
+        iob.write lorem_ipsum
+        iob.save_to_file
+      end
+    end
+
+  end
+
+  # File cleaning …
+  Dir["#{tmp_path}/**/*"].each do |f|
+    File.unlink f
+  end
+
+end

data/benchmarks/buffer.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+$LOAD_PATH.unshift File.expand_path("../../benchmark", __FILE__)
+
+$CUR_IMPL = "Buffer"
+
+require "viiite-template.rb"
+

data/benchmarks/memory.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+$LOAD_PATH.unshift File.expand_path("../../benchmark", __FILE__)
+
+$CUR_IMPL = "Memory"
+
+require "viiite-template.rb"
+

data/benchmarks.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+echo "Loading RVM ..."
+[[ -s $rvm_path/scripts/rvm ]] && source $rvm_path/scripts/rvm >/dev/null 2>&1
+
+RUBIES="ruby-1.8.7-p357 ree ruby-1.9.2-p290 ruby-1.9.3-p0"
+GEMSET="io_shuten"
+BENCHMARKS=`ls benchmarks`
+BENCH_OUT="benchmark/out.rash"
+BENCH_REP="benchmark/report"
+BENCH_RUN=10
+CUR_RAKE_VER=`gem list rake | awk '$1~/rake/{print $2}'`
+
+[[ -e "$BENCH_OUT" ]] && rm $BENCH_OUT && touch $BENCH_OUT
+
+for ruby in $RUBIES; do
+
+  echo
+  echo "Switching to: $ruby@$GEMSET"
+  rvm use --create $ruby@$GEMSET >/dev/null 2>&1
+
+  ruby_rake_ver=`gem list rake | awk '$1~/rake/{print $2}'`
+
+  if [[ "$ruby_rake_ver" != "$CUR_RAKE_VER" ]]; then
+    echo "Installing rake $CUR_RAKE_VER ..."
+    gem install rake -v=$CUR_RAKE_VER >/dev/null 2>&1
+  else
+    echo "rake $CUR_RAKE_VER already installed."
+  fi
+
+  bundler_installed=`gem list bundler | awk '$1~/bundler/ { print $1 }'`
+
+  if [[ -z "$bundler_installed" ]]; then
+    echo "Installing bundler ..."
+    gem install bundler --pre >/dev/null 2>&1
+  else
+    echo "bundler already installed."
+  fi
+
+  echo "Installing gems ..."
+  bundle install >/dev/null 2>&1
+
+  echo "Running benchmarks ..."
+  for bench in $BENCHMARKS; do
+    viiite run --runs=$BENCH_RUN $bench >> $BENCH_OUT 2>/dev/null
+  done
+
+done
+
+viiite report $BENCH_OUT -h --regroup=bench,ruby,type > ${BENCH_REP}.bench-ruby-type.txt
+viiite report $BENCH_OUT -h --regroup=bench,type,ruby > ${BENCH_REP}.bench-type-ruby.txt
+viiite report $BENCH_OUT -h --regroup=ruby,bench,type > ${BENCH_REP}.ruby-bench-type.txt
+viiite report $BENCH_OUT -h --regroup=ruby,type,bench > ${BENCH_REP}.ruby-type-bench.txt
+viiite report $BENCH_OUT -h --regroup=type,bench,ruby > ${BENCH_REP}.type-bench-ruby.txt
+viiite report $BENCH_OUT -h --regroup=type,ruby,bench > ${BENCH_REP}.type-ruby-bench.txt
+
+[[ -e "$BENCH_OUT" ]] && rm $BENCH_OUT
+
+echo
+echo "Reports created."

data/io_shuten.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = "io_shuten"
-  s.version = "0.0.3.dev1"
+  s.version = "0.1.0.dev6"
 
   s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
   s.authors = ["Christoph Grabo"]
-  s.date = "2012-01-25"
-  s.description = "IO::shuten \342\200\223 Use databases as IO handler like you would do with files and streams."
+  s.date = "2012-01-30"
+  s.description = "IO::shuten \u{2013} Use databases as IO handler like you would do with files and streams."
   s.email = ["chris@dinarrr.com"]
   s.extra_rdoc_files = [
     "LICENSE",
@@ -29,7 +29,10 @@ Gem::Specification.new do |s|
     "LICENSE.de",
     "README.md",
     "Rakefile",
-    "benchmark/compare_mem_w_buf.rb",
+    "benchmark/viiite-template.rb",
+    "benchmarks.sh",
+    "benchmarks/buffer.rb",
+    "benchmarks/memory.rb",
     "doc/IO_3A_3ABuffer.html",
     "doc/IO_shuten.html",
     "doc/IO_shuten/Base.html",
@@ -81,13 +84,20 @@ Gem::Specification.new do |s|
     "lib/io_shuten/mongo.rb",
     "lib/io_shuten/redis.rb",
     "lib/io_shuten/stores.rb",
+    "lib/io_shuten/stores/base_container.rb",
     "lib/io_shuten/stores/mongo.rb",
     "lib/io_shuten/stores/mongo/collection.rb",
     "lib/io_shuten/stores/mongo/gridfs.rb",
     "lib/io_shuten/stores/redis.rb",
+    "lib/io_shuten/stores/redis/container.rb",
     "lib/io_shuten/stores/redis/key_value.rb",
+    "lib/io_shuten/stores/redis/key_value/collection.rb",
+    "lib/io_shuten/stores/redis/key_value/single.rb",
     "lib/io_shuten/stores/redis/pub_sub.rb",
+    "lib/io_shuten/stores/redis/pub_sub/publisher.rb",
+    "lib/io_shuten/stores/redis/pub_sub/subscriber.rb",
     "lib/io_shuten/version.rb",
+    "lib/io_shuten/zmq.rb",
     "spec/examples/logger_spec.rb",
     "spec/lib/buffer_spec.rb",
     "spec/lib/memory_spec.rb",
@@ -100,13 +110,14 @@ Gem::Specification.new do |s|
     "spec/lib/stores/redis/pub_sub_spec.rb",
     "spec/lib/stores/redis_spec.rb",
     "spec/lib/stores_spec.rb",
+    "spec/lib/zmq_spec.rb",
     "spec/spec_helper.rb"
   ]
   s.homepage = "http://github.com/asaaki/io_shuten"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
-  s.summary = "IO::shuten \342\200\223 Use databases as IO handler. (NOT YET READY FOR PRODUCTION!)"
+  s.rubygems_version = "1.8.15"
+  s.summary = "IO::shuten \u{2013} Use databases as IO handler. (NOT YET READY FOR PRODUCTION!)"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
@@ -115,8 +126,11 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<iobuffer>, ["~> 1.0.0"])
       s.add_runtime_dependency(%q<hiredis>, ["~> 0.4.4"])
       s.add_runtime_dependency(%q<redis>, ["~> 2.2.2"])
+      s.add_runtime_dependency(%q<redis-namespace>, [">= 0"])
       s.add_runtime_dependency(%q<bson_ext>, ["~> 1.5.2"])
       s.add_runtime_dependency(%q<mongo>, ["~> 1.5.2"])
+      s.add_runtime_dependency(%q<ffi>, [">= 0"])
+      s.add_runtime_dependency(%q<ffi-rzmq>, [">= 0"])
       s.add_development_dependency(%q<rake>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
       s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
@@ -127,12 +141,18 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<yard>, [">= 0"])
       s.add_development_dependency(%q<yard-blame>, [">= 0"])
       s.add_development_dependency(%q<pry>, [">= 0"])
+      s.add_development_dependency(%q<alf>, [">= 0"])
+      s.add_development_dependency(%q<fastercsv>, [">= 0"])
+      s.add_development_dependency(%q<viiite>, [">= 0"])
     else
       s.add_dependency(%q<iobuffer>, ["~> 1.0.0"])
       s.add_dependency(%q<hiredis>, ["~> 0.4.4"])
       s.add_dependency(%q<redis>, ["~> 2.2.2"])
+      s.add_dependency(%q<redis-namespace>, [">= 0"])
       s.add_dependency(%q<bson_ext>, ["~> 1.5.2"])
       s.add_dependency(%q<mongo>, ["~> 1.5.2"])
+      s.add_dependency(%q<ffi>, [">= 0"])
+      s.add_dependency(%q<ffi-rzmq>, [">= 0"])
       s.add_dependency(%q<rake>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<rspec>, ["~> 2.8.0"])
@@ -143,13 +163,19 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<yard>, [">= 0"])
       s.add_dependency(%q<yard-blame>, [">= 0"])
       s.add_dependency(%q<pry>, [">= 0"])
+      s.add_dependency(%q<alf>, [">= 0"])
+      s.add_dependency(%q<fastercsv>, [">= 0"])
+      s.add_dependency(%q<viiite>, [">= 0"])
     end
   else
     s.add_dependency(%q<iobuffer>, ["~> 1.0.0"])
     s.add_dependency(%q<hiredis>, ["~> 0.4.4"])
     s.add_dependency(%q<redis>, ["~> 2.2.2"])
+    s.add_dependency(%q<redis-namespace>, [">= 0"])
     s.add_dependency(%q<bson_ext>, ["~> 1.5.2"])
     s.add_dependency(%q<mongo>, ["~> 1.5.2"])
+    s.add_dependency(%q<ffi>, [">= 0"])
+    s.add_dependency(%q<ffi-rzmq>, [">= 0"])
     s.add_dependency(%q<rake>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<rspec>, ["~> 2.8.0"])
@@ -160,6 +186,9 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<yard>, [">= 0"])
     s.add_dependency(%q<yard-blame>, [">= 0"])
     s.add_dependency(%q<pry>, [">= 0"])
+    s.add_dependency(%q<alf>, [">= 0"])
+    s.add_dependency(%q<fastercsv>, [">= 0"])
+    s.add_dependency(%q<viiite>, [">= 0"])
   end
 end
 

data/lib/io_shuten/redis.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require "redis"
+require "redis-namespace"
 
 module IO_shuten
   # Implementation of the Redis storage
@@ -9,8 +10,8 @@ module IO_shuten
   #
   # KeyValue
   # has different node types.
-  # Incremental single message store: each write will create a new key.
-  # Message collection store: all writes go to a single key.
+  #   Single: all writes go into a single key
+  #   Collection: every write will create a new key (namespaced)
   # Good for events or logs.
   #
   # PubSub
@@ -31,17 +32,19 @@ module IO_shuten
     # @param [Symbol] type for backend :key_value => [:single,:collection], for backend :pub_sub => [:publisher,:subscriber]
     # @param [Object] Redis an instance for this specific node, otherwise the Redis.redis instance will be used
     def initialize node_name, backend, type, redis_instance = nil
-      # based on the backend type it should instantiate the corresponding backend store module
-      node = :nil
-      case backend
-        when :key_value
-          Stores::Redis::KeyValue #.create(type)
-        when :pub_sub
-          Stores::Redis::PubSub #.create(type)
+      if [String, Symbol].include?(node_name.class)
+        unless Redis.instance_exists? node_name
+          @node_name      = node_name
+          redis_instance  ||= @@redis
+          @container      = Stores::Redis::Container.new(node_name, backend, type, redis_instance)
+          @@instances     << self unless @@instances.include?(self)
         else
-          raise ArgumentError, "Backend type unknown. Use :key_value or :pub_sub"
+          raise Errors::NodeExistsError, "Node already in pool, replacement is not allowed."
+        end
+      else
+        raise Errors::NodeNameError, "Name must be kind of String or Symbol and can't be nil."
       end
-      @@instances << node
+
     end
 
     ### class methods ###
@@ -56,19 +59,16 @@ module IO_shuten
       # Sets a new global redis client for the pool
       # @return [Object] new redis client
       def redis= new_redis
-        @redis = new_redis
+        @@redis = new_redis
       end
-    end
 
-    ### instance methods ###
-
-    def read
-    end
-
-    def write
-    end
+      def redis_clear!
+        keys = @@redis.keys
+        keys.each do |key|
+          @@redis.del key
+        end
+      end
 
-    def close
     end
 
   end

data/lib/io_shuten/stores/base_container.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+module IO_shuten::Stores
+  # Namespace for mongodb backends
+  module BaseContainer
+    def read
+      # dummy
+    end
+
+    def write data
+      # dummy
+    end
+
+    def close
+      # dummy
+      # we never close/quit the connection
+      # because the instance can be a shared one
+    end
+  end
+end

data/lib/io_shuten/stores/redis/container.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+
+module IO_shuten::Stores::Redis
+  # Redis Backend container
+  class Container
+    include IO_shuten::Stores::BaseContainer
+
+    attr_reader :redis
+    attr :backend, :backend_type
+
+    def initialize node_name, backend, type, redis_instance = nil
+      @node_name = node_name
+      use_backend backend
+      use_type    type
+      use_redis   redis_instance
+    end
+
+    def use_backend backend
+      backends = {:key_value => KeyValue, :pub_sub => PubSub}
+      if backends.keys.include?(backend)
+        self.extend backends[backend]
+        @backend = backend
+      else
+        raise ArgumentError, "Backend unknown. Use :key_value or :pub_sub"
+      end
+    end
+
+    def backend_spec
+      [@backend,@backend_type]
+    end
+
+    def use_redis redis
+      @redis = redis unless redis == nil
+    end
+
+  end
+end

data/lib/io_shuten/stores/redis/key_value/collection.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+module IO_shuten::Stores::Redis::KeyValue
+  # Redis Backend type for storing every data piece into its own key (namespace)
+  module Collection
+
+    def read
+      result = read_collection
+      result.join
+    end
+
+    def read_collection
+      coll = @redis.keys("#{@node_name}:*").sort
+      coll.inject([]) do |store, key|
+        store << (@redis.get(key))
+        store
+      end
+    end
+
+    def write data
+      counter = @redis.keys("#{@node_name}:*").size + 1
+      @redis.set "#{@node_name}:#{counter}", data
+    end
+
+    def clear!
+      coll = @redis.keys("#{@node_name}:*")
+      coll.each{ |key| @redis.del key }
+    end
+
+  end
+end

data/lib/io_shuten/stores/redis/key_value/single.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+module IO_shuten::Stores::Redis::KeyValue
+  # Redis Backend type for storing data into a single key (content => array/hash)
+  module Single
+
+    def read
+      result = read_raw
+      result.join
+    end
+
+    def read_raw
+      @redis.lrange @node_name, 0, -1
+    end
+
+    def write data
+      @redis.rpush @node_name, data
+    end
+
+    def clear!
+      @redis.del @node_name
+    end
+
+  end
+end

data/lib/io_shuten/stores/redis/key_value.rb

@@ -3,5 +3,19 @@
 module IO_shuten::Stores::Redis
   # Redis Backend for key-value based storage
   module KeyValue
+
+    def use_type type
+      types = {:single => KeyValue::Single, :collection => KeyValue::Collection}
+      if types.keys.include?(type)
+        self.extend types[type]
+        @backend_type = type
+      else
+        raise ArgumentError, "Type for backend unknown. Use :single or :collection"
+      end
+    end
+
   end
 end
+
+require "io_shuten/stores/redis/key_value/single"
+require "io_shuten/stores/redis/key_value/collection"

data/lib/io_shuten/stores/redis/pub_sub.rb

@@ -3,5 +3,17 @@
 module IO_shuten::Stores::Redis
   # Redis Backend for pub-sub (event/message) based storage
   module PubSub
+
+    def use_type type
+      if [:publisher,:subscriber].include?(type)
+        @backend_type = type
+      else
+        raise ArgumentError, "Type for backend unknown. Use :single or :collection"
+      end
+    end
+
   end
 end
+
+require "io_shuten/stores/redis/pub_sub/publisher"
+require "io_shuten/stores/redis/pub_sub/subscriber"

data/lib/io_shuten/stores/redis.rb

@@ -6,6 +6,9 @@ module IO_shuten::Stores
   end
 end
 
+require "redis"
+require "redis-namespace"
+
 require "io_shuten/stores/redis/key_value"
 require "io_shuten/stores/redis/pub_sub"
-
+require "io_shuten/stores/redis/container"

data/lib/io_shuten/stores.rb

@@ -6,6 +6,6 @@ module IO_shuten
   end
 end
 
+require "io_shuten/stores/base_container"
 require "io_shuten/stores/mongo"
 require "io_shuten/stores/redis"
-