io_shuten 0.1.0.dev7 → 0.1.1.dev1

data/Gemfile

@@ -1,34 +1,28 @@
 source "https://rubygems.org"
 
-gem "iobuffer", "~> 1.0.0"
+gem "iobuffer", "~> 1.1.2"
 
 # REDIS STORE
-gem "hiredis", "~> 0.4.4"
-gem "redis", "~> 2.2.2", :require => ["redis/connection/hiredis", "redis"]
-gem "redis-namespace"
+gem "hiredis", "~> 0.4.5"
+gem "redis", "~> 3.0.2", :require => ["redis/connection/hiredis", "redis"]
+gem "redis-namespace", "~> 1.2.1"
 
 # MONGO STORE
 gem "bson_ext", "~> 1.5.2"
 gem "mongo", "~> 1.5.2"
 
 # ZMQ "STORE"
-gem "ffi"
-gem "ffi-rzmq"
+gem "ffi", "~> 1.1.5"
+gem "ffi-rzmq", "~> 0.9.6"
 
 group :development, :test do
   gem "rake"
   gem "jeweler"
 
-  gem "rspec", "~> 2.8.0"
-  gem "simplecov", :require => false
-  gem "simplecov-rcov", :require => false
-  gem "simplecov-csv", :require => false
-
-  gem "redcarpet"
-  gem "yard"
-  gem "yard-blame"
+  gem "rspec", "~> 2.11.0"
 
   gem "pry"
+  gem "pry-doc"
 
   gem "alf"
   gem "fastercsv"

data/Gemfile.lock

@@ -8,66 +8,55 @@ GEM
     bson (1.5.2)
     bson_ext (1.5.2)
       bson (= 1.5.2)
-    coderay (0.9.8)
+    coderay (1.0.8)
     diff-lcs (1.1.3)
-    fastercsv (1.5.4)
-    ffi (1.0.11)
-    ffi-rzmq (0.9.3)
+    fastercsv (1.5.5)
+    ffi (1.1.5)
+    ffi-rzmq (0.9.6)
       ffi
     git (1.2.5)
     gnuplot (2.3.6)
-    highline (1.6.11)
-    hiredis (0.4.4)
-    iobuffer (1.0.0)
-    jeweler (1.6.4)
+    highline (1.6.15)
+    hiredis (0.4.5)
+    iobuffer (1.1.2)
+    jeweler (1.8.4)
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
-    json (1.6.5)
-    method_source (0.6.7)
-      ruby_parser (>= 2.3.1)
+      rdoc
+    json (1.7.5)
+    method_source (0.8.1)
     mongo (1.5.2)
       bson (= 1.5.2)
-    multi_json (1.0.4)
     myrrha (1.2.2)
-    pry (0.9.7.4)
-      coderay (~> 0.9.8)
-      method_source (~> 0.6.7)
-      ruby_parser (>= 2.3.1)
-      slop (~> 2.1.0)
-    quickl (0.4.2)
+    pry (0.9.10)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.3.1)
+    pry-doc (0.4.4)
+      pry (>= 0.9.9.6)
+      yard (~> 0.8.1)
+    quickl (0.4.3)
     rake (0.9.2.2)
-    redcarpet (2.0.1)
-    redis (2.2.2)
-    redis-namespace (1.1.0)
-      redis (< 3.0.0)
-    rspec (2.8.0)
-      rspec-core (~> 2.8.0)
-      rspec-expectations (~> 2.8.0)
-      rspec-mocks (~> 2.8.0)
-    rspec-core (2.8.0)
-    rspec-expectations (2.8.0)
-      diff-lcs (~> 1.1.2)
-    rspec-mocks (2.8.0)
-    ruby_parser (2.3.1)
-      sexp_processor (~> 3.0)
-    sexp_processor (3.0.10)
-    simplecov (0.5.4)
-      multi_json (~> 1.0.3)
-      simplecov-html (~> 0.5.3)
-    simplecov-csv (0.1.3)
-      simplecov
-    simplecov-html (0.5.3)
-    simplecov-rcov (0.2.3)
-      simplecov (>= 0.4.1)
-    slop (2.1.0)
+    rdoc (3.12)
+      json (~> 1.4)
+    redis (3.0.2)
+    redis-namespace (1.2.1)
+      redis (~> 3.0.0)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.3)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.3)
+    slop (3.3.3)
     viiite (0.2.0)
       alf (~> 0.10.1)
       gnuplot (~> 2.3.6)
       json
-    yard (0.7.4)
-    yard-blame (0.0.1)
-      yard
+    yard (0.8.3)
 
 PLATFORMS
   ruby
@@ -76,21 +65,16 @@ DEPENDENCIES
   alf
   bson_ext (~> 1.5.2)
   fastercsv
-  ffi
-  ffi-rzmq
-  hiredis (~> 0.4.4)
-  iobuffer (~> 1.0.0)
+  ffi (~> 1.1.5)
+  ffi-rzmq (~> 0.9.6)
+  hiredis (~> 0.4.5)
+  iobuffer (~> 1.1.2)
   jeweler
   mongo (~> 1.5.2)
   pry
+  pry-doc
   rake
-  redcarpet
-  redis (~> 2.2.2)
-  redis-namespace
-  rspec (~> 2.8.0)
-  simplecov
-  simplecov-csv
-  simplecov-rcov
+  redis (~> 3.0.2)
+  redis-namespace (~> 1.2.1)
+  rspec (~> 2.11.0)
   viiite
-  yard
-  yard-blame

data/README.md

@@ -2,6 +2,11 @@
 
 Use databases as IO handler like you would do with files and streams.
 
+Backends are (or will be): Redis, MongoDB.
+
+Special backend will be: ZeroMQ (not a database but cool message queue backend).
+
+Wishes? MySQL, Postgres, Sqlite? Other NoSQLs, memcache? Tell me!
 
 ## Motivation
 
@@ -9,16 +14,37 @@ One purpose for creating this project is to have an IO like interface to databas
 
 With this gem you should have the possibility to use redis or mongodb as endpoints for IO related tasks like logging.
 
-Example:
+And logging was my first idea for this project. I want to have the possibility to set my own logger (endpoint) in a Ruby/Rails app.
+
+Also you could use different logger endpoints in the same app for different purposes but without bloating your app/tmp dir with huge files.
+
+If you have more cool ideas, what to do with it, tell me (and I will put it in this README).
+
+### Example
 
 ```ruby
 require "io_shuten"
 require "logger"
 
-logdev = IO_shuten::Memory.new('log_node')
+# use redis namespace to separate and avoid naming conflicts
+IO_shuten::Redis.redis = Redis::Namespace.new("io_shuten:test", :redis => Redis.new)
+
+# instantiate logdev and logger
+logdev = IO_shuten::Redis.new(:app_log, :key_value, :collection)
 logger = Logger.new(logdev)
 
-logger.info "This message will be stored in memory."
+# log something
+logger.info "This message will be stored in redis as a single key."
+logger.debug "And this message will be also a single key"
+
+#check
+puts logdev.read #=> will return both messages as a single string
+```
+
+And check in redis for the keys:
+
+```bash
+redis-cli KEYS "io_shuten/test"
 ```
 
 
@@ -27,44 +53,137 @@ logger.info "This message will be stored in memory."
 In japanese `io` means *500* or *big amount*, `shū･ten` means *endpoint* (train terminus/last stop or mathematical endpoint).
 
 
-## Types
+## Storage Types
+
+
+### IO_shuten::Redis
 
+Stores content in a redis database.
 
-### IO_shuten::Memory
 
-An in-memory storage system (handles simple StringIO objects).
+#### Backends
 
-Offers write-to-disk for single instance and all instances.
+The redis storage system provides two different backends: `KeyValue` and `PubSub`.
 
-Provides all methods of StringIO.
 
-### IO_shuten::Buffer
+##### KeyValue
 
-Like IO_shuten::Memory but with IO::Buffer as container instead of StringIO.
-(Install it with `gem install iobuffer`.)
+Has two different storage types: `Single` and `Collection`.
 
-Performance depends on machine and ruby version. Almost same throughput, on 1.9.3 is the native StringIO better than the IO::Buffer cext.
 
-### IO_shuten::Redis
+##### KeyValue::Single
 
-Stores content in a redis database.
+Stores every message in a single key in redis (like a big document).
+
+Choose this, if you have only small amount of chunks to store.
+More reads, less writes.
+
+```ruby
+require "io_shuten"
+IO_shuten::Redis.redis = Redis::Namespace.new("io_shuten:key_value:single:test", :redis => Redis.new)
+
+io_object = IO_shuten::Redis.new(:my_node, :key_value, :single)
+
+io_object.write "my awesome data!\n"
+io_object.write "chunky bacon!!!\n"
+
+io_object.read
+```
+
+After this you will have a single key in redis like:
+
+```
+$ redis-cli KEYS "io_shuten:key_value:collection:test:*"
+1) "io_shuten:key_value:single:test:my_node"
+```
+
+The content is a LIST:
+
+```
+$ redis-cli LRANGE "io_shuten:key_value:single:test:my_node" 0 -1
+1) "my awesome data!\n"
+2) "chunky bacon!!!\n"
+```
 
-Provides basic methods like `#read`, `#write` and `#close`.
-A logger only needs write and close methods of an IO object.
+##### KeyValue::Collection
+
+Stores every single message in its own key. The provided node_name acts as namespace.
+
+Choose this if you have huge amounts of chunks, especially constantly coming in, like logging events.
+More writes, less reads.
+
+```ruby
+require "io_shuten"
+IO_shuten::Redis.redis = Redis::Namespace.new("io_shuten:key_value:collection:test", :redis => Redis.new)
+
+io_object = IO_shuten::Redis.new(:my_node, :key_value, :collection)
+
+io_object.write "my awesome data!\n"
+io_object.write "chunky bacon!!!\n"
+
+io_object.read
+```
+
+After this you will have two keys in redis like:
+
+```
+$ redis-cli KEYS "io_shuten:key_value:collection:test:*"
+1) "io_shuten:key_value:collection:test:my_node:0000000000000000"
+2) "io_shuten:key_value:collection:test:my_node:0000000000000001"
+3) "io_shuten:key_value:collection:test:my_node:0000000000000002"
+```
+The first key with 16 zeros is a counter (to avoid long running key searches to get the count).
+
+Numerical part is a 16 digit number, should be enough for 10 quadrillion (minus 1) messages per node.
+(The reason is for internal sorting of the keys)
+
+With messages of an average length of only 16 bytes this would end up in 160 petabytes of raw data.
+
+
+##### PubSub
+
+**Not yet implemented!**
+
+Will have two types: `Publisher` and `Subscriber`.
+
+As logger backend only the `PubSub::Publisher` will be usable,
+but you can use a redis subscriber on the other side to receive the messages and do something with them.
 
 ### IO_shuten::Mongo
 
-Stores content in a mongodb database.
+**Not yet implemented!**
 
-Provides basic methods like `#read`, `#write` and `#close`.
-A logger only needs write and close methods of an IO object.
 
 ### IO_shuten::Zmq
 
-"Stores" content in a zmq socket.
+**Not yet implemented!**
+
+
+### IO_shuten::Memory and IO_shuten::Buffer
+
+**Maybe only useful as temporary storage for a session.**
+
+_These implementations were the first tests and aren't intended to be used in production._
 
-Provides basic methods like `#read`, `#write` and `#close`.
-A logger only needs write and close methods of an IO object.
+> An in-memory storage system (handles simple StringIO/IOBuffer objects).
+>
+> Offers write-to-disk for single instance and all instances.
+>
+> Provides all methods of StringIO respectively IOBuffer.
+>
+> _IOBuffer usage:_ Install it with `gem install iobuffer` first.
+>
+> Performance of both are nearly the same.
+>
+> Read the _documentation_ if you really want to use them.
+
+
+## [Documentation](http://asaaki.github.com/io_shuten/doc/index.html)
+
+
+## Future Plans
+
+Maybe a special Logger, especially for Redis (to get benefits of the key search).
 
 
 ## Not for production!
@@ -78,15 +197,20 @@ Feel free to contribute, write issues if you want to support me with your ideas
 
 ## git flowed
 
-Development will happen in the `develop` branch now, master should be in a deployable state.
+Development will happen in the `develop` branch, `master` should be in a deployable state.
 
-Read more about `git flow` and the branchin model here:
+Read more about `git flow` and the branching model here:
 
 * http://nvie.com/posts/a-successful-git-branching-model/
 * https://github.com/nvie/gitflow
 
 I like this approach and try to use it whenever possible, also in one-man-development.
 
+### Contribute
+
+Fork it, create your feature in a `feature/branch` based on `develop`(!), make a pull request.
+
+Hotfixes for master can be made out of master branch, of course. ;o)
 
 ## License
 

data/Rakefile

@@ -7,18 +7,12 @@ require "jeweler"
 require "rspec"
 require "rspec/core/rake_task"
 
-require "yard"
-
 
 $LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
 require "io_shuten/version"
 
-
-
 RUBY_ENGINE = '(no engine)' unless defined? RUBY_ENGINE
 
-
-
 Jeweler::Tasks.new do |gem|
   gem.name        = "io_shuten"
   gem.version     = IO_shuten::VERSION
@@ -31,8 +25,6 @@ Jeweler::Tasks.new do |gem|
   gem.description = "IO::shuten – Use databases as IO handler like you would do with files and streams."
 end
 
-
-
 task :gem => :build
 
 desc "Release current gem version to rubygems.org"
@@ -49,21 +41,6 @@ task RSpec::Core::RakeTask.new(:spec) do |t|
   t.verbose = false
 end
 
-desc "Run all specs + code coverage"
-task "spec:cov" do
-  ENV["COV"] = 'true'
-  Rake::Task["spec"].invoke
-end
-
-
-
-YARD::Rake::YardocTask.new do |t|
-  t.files   = ['lib/**/*.rb']
-  #t.options = ['--any', '--extra', '--opts']
-end
-
-
-
 desc "Starts IRB with env"
 task :irb do
   sh "irb -I lib -r io_shuten"
@@ -74,15 +51,11 @@ task :pry do
   sh "pry -I lib -r io_shuten --no-pager"
 end
 
-
-
 desc "Prints current environment"
 task :envinfo do
   puts ['RUBY:',RUBY_PLATFORM,RUBY_ENGINE,RUBY_VERSION].join(" ")
 end
 
-
-
 desc "RBX ONLY: Clean up rbc and .rbx"
 task :rbx_clean do
   Dir["./**/*.rbc","./**/.*.rbc"].each do |f|
@@ -91,8 +64,6 @@ task :rbx_clean do
   sh "rm -rf .rbx" if File.exists?(".rbx") && File.directory?(".rbx")
 end
 
-
-
 desc "Runs complex viiite benchmark suite"
 task :benchmark do
   sh "./benchmarks.sh"