redisarray 0.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in redisarray.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    redisarray (0.0.1)
+      redis (~> 2.2.2)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    rake (0.9.2.2)
+    redis (2.2.2)
+    rspec (2.7.0)
+      rspec-core (~> 2.7.0)
+      rspec-expectations (~> 2.7.0)
+      rspec-mocks (~> 2.7.0)
+    rspec-core (2.7.1)
+    rspec-expectations (2.7.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.7.0)
+
+PLATFORMS
+  x86-mingw32
+
+DEPENDENCIES
+  rake (~> 0.9)
+  redisarray!
+  rspec (~> 2.7.0)

data/README.md

@@ -0,0 +1,96 @@
+redisarray
+==========
+
+[RedisArray] Implements memory efficient algorithm allowing to store tables or two dimensional arrays inside Redis.
+
+Introduction
+============
+
+Often we need to store a table with headers and rows or just an adjacent 2D array inside our Redis key-value store. We could quickly design our own custom solution but it probably would not be optimized for memory usage.
+The purpose of this project is to design general solution for storing two dimensional arrays inside Redis with memory optimization in mind. 
+How is that accomplished? All the data that is going to be written is just sliced on pieces behind the scenes and stored in hashes with 100 of fields each which is memory efficient. To read more about this technique follow this page: http://redis.io/topics/memory-optimization
+
+
+Compatibility
+=============
+
+So far compatible with ruby 1.8.7. Does not work with 1.9.2. There are a few adjustments needed concerning the use of CSV class which has significantly changed in 1.9.2.
+	
+Install
+=======
+
+    gem install redisarray
+	
+Usage
+=====
+
+There are two classes worth noting:
+- RedisTable - where most of the action takes place
+- RedisWorkbook - a wrapper for RedisTable that simplifies storage of array of arrays or a workbook of worksheets with headers which is itself an array of arrays
+
+You can start using redisarray gem like this:
+
+	require 'rubygems'
+	require 'redisarray'
+	include RedisArray
+
+	group_name = RedisHashGroup.new.name
+    @workbook = RedisWorkbook.new group_name, 'my_workbook'
+	@workbook.set_sheet_data "my_sheet_name",
+                               rows = [['row 1 cell 1','row 1 cell 2'], ['row 2 cell 1','row 2 cell 2']],
+                               :start_from_row => 3
+	p @workbook.get_sheet_data("my_sheet_name")	
+
+By default it will assume redis is listening on localhost port 6379, to change it do this:
+
+	require 'rubygems'
+	require 'redisarray'
+	require 'redis'
+	include RedisArray
+	
+	RedisTable.set_redis Redis.new(:host => 'localhost', :port => 6379)
+	
+	group_name = RedisHashGroup.new.name
+	...
+	
+For more examples take a look at spec directory especially in redis_workbook_spec.rb 
+
+Developer Instructions
+======================
+
+The dependencies for the gem and for developing the gem are managed by Bundler.
+
+    gem install bundler
+    git clone http://github.com/ksob/redisarray.git
+    cd ./redisarray
+	bundle install
+
+Specs can be run with (they require some preconditions to be met like running Redis on localhost port 6379):
+
+	bundle exec rspec spec
+
+License
+=======
+
+(The MIT License)
+
+Copyright (c) 2011 Kamil Sobieraj, ksobej@gmail.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/lib/redisarray.rb

@@ -0,0 +1,12 @@
+module RedisArray
+  require 'redisarray/core_ext/hash/reverse_merge'
+  require 'redisarray/core_ext/class/attribute_accessors'
+  require 'csv'
+  require 'redis'
+
+  require 'redisarray/version'
+  require 'redisarray/redis_table'
+  require 'redisarray/redis_hash_group'
+  require 'redisarray/redis_workbook'
+end
+

data/lib/redisarray/core_ext/array/extract_options.rb

@@ -0,0 +1,29 @@
+class Hash
+  # By default, only instances of Hash itself are extractable.
+  # Subclasses of Hash may implement this method and return
+  # true to declare themselves as extractable. If a Hash
+  # is extractable, Array#extract_options! pops it from
+  # the Array when it is the last element of the Array.
+  def extractable_options?
+    instance_of?(Hash)
+  end
+end
+
+class Array
+  # Extracts options from a set of arguments. Removes and returns the last
+  # element in the array if it's a hash, otherwise returns a blank hash.
+  #
+  #   def options(*args)
+  #     args.extract_options!
+  #   end
+  #
+  #   options(1, 2)           # => {}
+  #   options(1, 2, :a => :b) # => {:a=>:b}
+  def extract_options!
+    if last.is_a?(Hash) && last.extractable_options?
+      pop
+    else
+      {}
+    end
+  end
+end

data/lib/redisarray/core_ext/class/attribute_accessors.rb

@@ -0,0 +1,79 @@
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "array/extract_options"))
+
+# Extends the class object with class and instance accessors for class attributes,
+# just like the native attr* accessors for instance attributes.
+#
+# Note that unlike +class_attribute+, if a subclass changes the value then that would
+# also change the value for parent class. Similarly if parent class changes the value
+# then that would change the value of subclasses too.
+#
+#  class Person
+#    cattr_accessor :hair_colors
+#  end
+#
+#  Person.hair_colors = [:brown, :black, :blonde, :red]
+#  Person.hair_colors     # => [:brown, :black, :blonde, :red]
+#  Person.new.hair_colors # => [:brown, :black, :blonde, :red]
+#
+# To opt out of the instance writer method, pass :instance_writer => false.
+# To opt out of the instance reader method, pass :instance_reader => false.
+#
+#   class Person
+#     cattr_accessor :hair_colors, :instance_writer => false, :instance_reader => false
+#   end
+#
+#   Person.new.hair_colors = [:brown]  # => NoMethodError
+#   Person.new.hair_colors             # => NoMethodError
+class Class
+  def cattr_reader(*syms)
+    options = syms.extract_options!
+    syms.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+        unless defined? @@#{sym}
+          @@#{sym} = nil
+        end
+
+        def self.#{sym}
+          @@#{sym}
+        end
+      EOS
+
+      unless options[:instance_reader] == false
+        class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+          def #{sym}
+            @@#{sym}
+          end
+        EOS
+      end
+    end
+  end
+
+  def cattr_writer(*syms)
+    options = syms.extract_options!
+    syms.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+        unless defined? @@#{sym}
+          @@#{sym} = nil
+        end
+
+        def self.#{sym}=(obj)
+          @@#{sym} = obj
+        end
+      EOS
+
+      unless options[:instance_writer] == false
+        class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+          def #{sym}=(obj)
+            @@#{sym} = obj
+          end
+        EOS
+      end
+      self.send("#{sym}=", yield) if block_given?
+    end
+  end
+
+  def cattr_accessor(*syms, &blk)
+    cattr_reader(*syms)
+    cattr_writer(*syms, &blk)
+  end
+end

data/lib/redisarray/core_ext/hash/reverse_merge.rb

@@ -0,0 +1,23 @@
+class Hash
+  # Merges the caller into +other_hash+. For example,
+  #
+  #   options = options.reverse_merge(:size => 25, :velocity => 10)
+  #
+  # is equivalent to
+  #
+  #   options = {:size => 25, :velocity => 10}.merge(options)
+  #
+  # This is particularly useful for initializing an options hash
+  # with default values.
+  def reverse_merge(other_hash)
+    other_hash.merge(self)
+  end
+
+  # Destructive +reverse_merge+.
+  def reverse_merge!(other_hash)
+    # right wins if there is no left
+    merge!(other_hash) { |key, left, right| left }
+  end
+
+  alias_method :reverse_update, :reverse_merge!
+end

data/lib/redisarray/redis_hash_group.rb

@@ -0,0 +1,22 @@
+module RedisArray
+  class RedisHashGroup
+    attr_reader :name
+
+    def initialize options={}
+      options.reverse_merge!(:existing_group_name => nil)
+      if options[:prefix]
+        @name = options[:prefix] + "-" + rand(100000).to_s
+        while not RedisTable.get_redis.keys("#{@name}:*").empty? do
+          @name = options[:prefix] + "-" + rand(100000).to_s
+        end
+      elsif options[:existing_group_name].nil?
+        @name = rand(100000).to_s
+        while not RedisTable.get_redis.keys("#{@name}:*").empty? do
+          @name = rand(100000).to_s
+        end
+      else
+        @name = options[:existing_group_name].to_s
+      end
+    end
+  end
+end

data/lib/redisarray/redis_table.rb

@@ -0,0 +1,187 @@
+require 'redis'
+require 'csv'
+
+module RedisArray
+  class RedisTable
+    cattr_accessor :logger
+    @silence = false
+
+    def self.silence?
+      @silence
+    end
+
+    # Silence the logger.
+    def self.silence!
+      @silence = true
+      self
+    end
+
+    @redis = Redis.new(:host => 'localhost', :port => 6379)
+
+    def self.get_redis
+      @redis
+    end
+
+    def self.set_redis redis
+      @redis = redis
+    end
+
+    def self.get_table_names main_key_part
+      res = []
+      @redis.keys("#{main_key_part}:*").each do |key|
+        res << key.scan(/^#{Regexp.escape(main_key_part)}\:(.+):\d+/)[0][0]
+      end
+      res
+    end
+
+    def self.get_row_ranges main_key_part, table_name
+      row_ranges = []
+      @redis.keys("#{main_key_part}:#{table_name}:*").each do |key|
+        row_ranges << key.scan(/^#{Regexp.escape(main_key_part)}\:#{Regexp.escape(table_name)}\:(\d+)/)[0][0]
+      end
+      row_ranges
+    end
+
+    # Parameters:
+    # +rows+ - specifies a range of rows to return,
+    # it is header row agnostic in meaning that its up to you
+    # to track if you supplied a header or not
+    # As to the place of the header:
+    # if you supplied headers in +header_row+ parameter for +set_table+
+    # then it is stored at index 0 and you can retrieve it usign +get_table_data+
+    # in two ways:
+    # 1. specify +rows+ with value of :all
+    # 2. specify +rows+ with value of 0
+    def self.get_table_data main_key_part, table_name, rows = 0..-1
+      table = {}
+      if rows == (0..-1)
+        get_row_ranges(main_key_part, table_name).each do |kri|
+          @redis.hgetall("#{main_key_part}:#{table_name}:#{kri}").each_pair do |k, v|
+            table[(kri.to_i*100) + k.to_i] = v
+          end
+        end
+      elsif rows.kind_of? Integer
+        get_row_ranges(main_key_part, table_name).each do |kri|
+          next if kri.to_i != rows / 100
+          table[(kri.to_i*100) + rows.to_i] = @redis.hget("#{main_key_part}:#{table_name}:#{kri}", rows)
+        end
+      elsif rows.kind_of? Range
+        # TODO: handle case where rows is range like 15..20
+        raise "TODO: handle case where rows is range like 15..20"
+      else
+        raise "Unsupported type of 'rows' parameter!"
+      end
+
+      table
+    end
+
+    def self.get_table_data_as_array main_key_part, table_name, rows = 0..-1
+      hash = get_table_data main_key_part, table_name, rows
+      res = hash.inject([]) do |array, (k, v)|
+        array[k] = CSV.parse(v+"\r\n"+v)[0].collect { |cell| cell.to_s } #if not cell.nil?}
+        array
+      end
+      if rows.kind_of? Integer
+        res.compact
+      else
+        res
+      end
+    end
+
+    def self.append_table_data main_key_part, table_name, rows, options={}
+
+      kri = get_row_ranges(main_key_part, table_name).sort[-1].to_i
+      res = @redis.hgetall("#{main_key_part}:#{table_name}:#{kri}").sort do |p1, p2|
+        p1[0].to_i <=> p2[0].to_i
+      end
+      first_empty_ri = 0 # the nearest (to the beginning) empty row index
+      if res.length > 0
+        first_empty_ri = res[-1][0].to_i + 1
+      else
+        # it is empty so start from the beginning
+      end
+
+      start_from_row = kri.to_i * 100 + first_empty_ri
+      if start_from_row == 0
+        set_table_data main_key_part, table_name, rows
+      else
+        set_table_data main_key_part, table_name, rows, options={:start_from_row => start_from_row}
+      end
+    end
+
+    # options:
+    # :start_from_row - i.e. the output index or in other words an offset of the output data
+    #       it cannot be greater than 99
+    # :header_row - array of strings - if specified they will be written at index 0 and the data rows will start at index 1
+    # :skip_headers - boolean - if specified the :header_row will be skipped and the data rows will start at index 0
+    # (unless the :start_from_row is specified at the same time that would overwrite this option,
+    # that is when :start_from_row is specified the :skip_headers will not cause shifting the data)
+    def self.set_table_data main_key_part, table_name, rows, options={}
+      if options[:skip_headers]
+        options.reverse_merge!(:start_from_row => 1)
+      else
+        options.reverse_merge!(:start_from_row => 0)
+      end
+      new_rows = Marshal::load(Marshal.dump(rows))
+      new_rows.unshift(options[:header_row]) if options[:header_row] and not options[:skip_headers]
+      source_row_index = 0 # the index in the source array of data (i.e. the one we are copying data from to put them to redis)
+      (0).upto((new_rows.count + options[:start_from_row]) / 100) do |kri|
+        next if kri < options[:start_from_row] / 100
+        (options[:start_from_row] / 100 == kri ? options[:start_from_row] % 100 : 0).upto(99).each do |ri|
+          break if source_row_index >= new_rows.count #+ options[:start_from_row]
+          curr_key = "#{main_key_part}:#{table_name}:#{kri}", ri # + (kri==0 ? options[:start_from_row] : 0)
+          log("HGET", "#{curr_key.to_s}  ==  #{@redis.hget(*curr_key).to_s}")
+          log("HSET", "#{curr_key.to_s}  ==  #{CSV.generate_line(new_rows[source_row_index]).to_s}")
+          if RUBY_VERSION < '1.9'
+            @redis.hset(*(curr_key << CSV.generate_line(new_rows[source_row_index])))
+          else
+            @redis.hset(*(curr_key << CSV.generate_line(new_rows[source_row_index], :row_sep => '')))
+          end
+          source_row_index += 1
+        end
+      end
+    end
+
+    def self.delete_all_table main_key_part, table_name
+      self.get_row_ranges(main_key_part, table_name).each do |kri|
+        curr_key = "#{main_key_part}:#{table_name}:#{kri}"
+        @redis.del(curr_key)
+      end
+    end
+
+    def self.delete_table_data main_key_part, table_name, options={}
+      if options[:skip_headers]
+        options.reverse_merge!(:start_from_row => 1)
+      else
+        options.reverse_merge!(:start_from_row => 0)
+      end
+
+      # TODO: start_from_row is not handled yet
+
+      if options[:skip_headers]
+        header = self.get_table_data main_key_part, table_name, rows = 0
+        self.delete_all_table main_key_part, table_name
+        self.set_table_data main_key_part, table_name, [], :header_row => header
+      else
+        self.delete_all_table main_key_part, table_name
+      end
+    end
+
+    def set_rows table_name, rows
+      rows.each_with_index do |row, idx|
+        @redis.zadd("matrix.#{table_name}.rows", idx, row)
+      end
+    end
+
+    def append_row table_name, row
+      @redis.zadd("matrix.#{table_name}.rows", idx, row)
+    end
+
+    private
+    def self.log(operation, message)
+      return unless logger && !silence?
+      logger.debug("RedisTable: #{operation}: #{message}")
+    end
+
+  end
+end