bigrecord-driver 0.0.3

data/bin/cassandra-driver

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+launcher = File.join(File.dirname(__FILE__), 'launcher')
+args = ["cassandra"] + ARGV
+exec "bash", launcher, *args

data/bin/hbase-driver

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+launcher = File.join(File.dirname(__FILE__), 'launcher')
+args = ["hbase"] + ARGV
+exec "bash", launcher, *args

data/bin/launcher

@@ -0,0 +1,155 @@
+#! /bin/bash
+DRIVERNAME=$1
+# Fix for Amazon EC2 instances
+unset RUBYLIB
+
+# Needs the $JRUBY_HOME environment variable to be set
+if [ "$JRUBY_HOME" == "" ]; then
+  echo "\$JRUBY_HOME is not set. Please set it in your ~/.bashrc file."
+  exit 1
+fi
+
+VENDOR_DIR=`dirname "$0"`
+VENDOR_DIR=`cd "$VENDOR_DIR/../vendor"; pwd`
+
+CONF_DIR=`dirname "$0"`
+CONF_DIR=`cd "$CONF_DIR/../conf"; pwd`
+
+# Ruby file that contains the drb server
+DRIVER=$VENDOR_DIR/../lib/big_record_driver/"$DRIVERNAME"_driver/server.rb
+
+# location where the pids are stored
+PIDS_DIR=/tmp/$DRIVERNAME-driver/run
+mkdir -p $PIDS_DIR
+
+# startup delay
+STARTUP_TIMEOUT=60
+
+# location where the log files are stored
+LOGS_DIR=/tmp/$DRIVERNAME-driver/log
+mkdir -p $LOGS_DIR
+
+# set the environment to use jruby by default
+PATH=$JRUBY_HOME/bin:$PATH
+for f in $VENDOR_DIR/java/$DRIVERNAME/*.jar; do
+  if [ -f $f ]; then
+    CLASSPATH=${CLASSPATH}:$f;
+  fi
+done
+#CLASSPATH=$CLASSPATH:$VENDOR_DIR/java/hadoop-0.19.1-core.jar:$VENDOR_DIR/java/hbase-0.19.1.jar:$VENDOR_DIR/java/commons-logging-1.0.4.jar:$VENDOR_DIR/java/commons-logging-api-1.0.4.jar:$VENDOR_DIR/java/log4j-1.2.13.jar:$CONF_DIR/log4j.properties
+
+export PATH CLASSPATH
+
+# TODO: find a way to use the return value of this function instead of
+# manipulating the global variable
+PIDS="" 
+
+refresh_pids() {
+		if [ -f $PIDS_DIR/$PORT.pid ]; then
+        	PID=`cat $PIDS_DIR/$PORT.pid`
+        else
+        	PID=""
+        fi
+}
+
+start() {
+        echo -n "Starting $DRIVERNAME driver on port $PORT."
+        
+        refresh_pids
+
+        if [ -f "$PIDS_DIR/$PORT.pid" -a "$PID" != "" ] ; then
+                echo -e "\nAlready running (pid="$PID")." 
+                exit 1
+        else
+            rm -f $LOGS_DIR/$PORT.log
+            nohup jruby $DRIVER $PORT >> $LOGS_DIR/$PORT.log 2>&1 < /dev/null &
+            PID=$!
+            if [ "$PID" != "" ] ; then
+                # monitor the log file for the message saying that the server is started
+                for ((i=0; i<$STARTUP_TIMEOUT; i+=1)); do
+                  sleep 1
+                  echo -n "."
+                  if [ "$(cat $LOGS_DIR/$PORT.log | grep 'Started drb server')" != "" ] ; then
+                    break
+                  fi
+                done
+
+                if [ "$i" == $STARTUP_TIMEOUT ] ; then
+                  echo -e "\nStartup timeout: couldn't start the DRb server."
+                else
+                  echo $PID > $PIDS_DIR/$PORT.pid
+                fi
+                echo ""
+            else
+            	echo -e "\nAn error occured while starting the DRb server."
+            fi
+        fi
+}
+
+stop() {
+        echo "Stopping $DRIVERNAME driver on port $PORT."
+
+        refresh_pids
+
+        if [ -f "$PIDS_DIR/$PORT.pid" -a "$PID" != "" ] ; then
+        		echo "Stopping driver (pid = $PID)." >> $LOGS_DIR/$PORT.log
+                kill $PID
+                rm $PIDS_DIR/$PORT.pid
+        else
+                echo "No $DRIVERNAME driver to kill."
+        fi
+}
+
+status() {
+		refresh_pids
+		
+		if [ -f "$PIDS_DIR/$PORT.pid" -a "$PID" != "" ] ; then
+				echo "Running."
+		else
+				echo "Stopped."
+		fi
+}
+
+start_debug() {
+  jruby $DRIVER $PORT
+}
+
+print_usage() {
+	echo "Usage: $DRIVERNAME-driver {start|stop|restart|status|start_debug} [-p <port>]"
+}
+
+set_port() {
+  if [ "$3" == "-p" -a "$4" != "" ]; then
+    PORT="$4"
+  else
+    PORT=40000
+  fi  
+}
+
+case "$2" in
+    start)
+    	set_port $@
+   		start
+        ;;
+    stop)
+    	set_port $@
+   		stop
+        ;;
+    restart)
+    	set_port $@
+   		stop
+   		start
+        ;;
+    status)
+        set_port $@
+        status
+        ;;
+    start_debug)
+    	set_port $@
+    	start_debug
+    	;;
+    *)
+        print_usage 
+        exit 1
+esac
+exit 0

data/conf/log4j.properties

@@ -0,0 +1,48 @@
+# Define some default values that can be overridden by system properties
+bigrecord.root.logger=INFO,DRFA
+bigrecord.log.dir=/tmp/hbase-driver/log
+bigrecord.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${bigrecord.root.logger}
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${bigrecord.log.dir}/${bigrecord.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this 
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+
+#log4j.logger.com.bigrecord=TRACE
+#log4j.logger.com.bigrecord.util.hbaserecord=DEBUG
+#log4j.logger.com.bigrecord.util.jactiverecord=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+#log4j.logger.org.apache.hadoop.dfs=DEBUG

data/lib/big_record_driver/bigrecord_server.rb

@@ -0,0 +1,119 @@
+require File.dirname(__FILE__) + '/exceptions'
+require File.dirname(__FILE__) + '/column_descriptor'
+require 'drb'
+# The name of the java String class conflicts with ruby's String class.
+module Java
+  include_class "java.lang.String"
+  include_class "java.lang.Exception"
+
+end
+
+class String
+  def to_bytes
+    Java::String.new(self).getBytes
+  end
+end
+
+
+module BigRecordDriver
+
+class BigRecordServer
+  include_class "java.io.IOException"
+
+  def configure(config = {})
+    raise NotImplementedError
+  end
+
+  def update(table_name, row, values, timestamp=nil)
+    raise NotImplementedError
+  end
+
+  def get(table_name, row, column, options={})
+    raise NotImplementedError
+  end
+
+  def get_columns(table_name, row, columns, options={})
+    raise NotImplementedError
+  end
+
+  def get_consecutive_rows(table_name, start_row, limit, columns, stop_row = nil)
+    raise NotImplementedError
+  end
+
+  def delete(table_name, row)
+    raise NotImplementedError
+  end
+
+  def create_table(table_name, column_descriptors)
+    raise NotImplementedError
+  end
+
+  def drop_table(table_name)
+    raise NotImplementedError
+  end
+
+  def truncate_table(table_name)
+    raise NotImplementedError
+  end
+
+  def ping
+    raise NotImplementedError
+  end
+
+  def table_exists?(table_name)
+    raise NotImplementedError
+  end
+
+  def table_names
+    raise NotImplementedError
+  end
+
+  def method_missing(method, *args)
+    super
+  rescue NoMethodError
+    raise NoMethodError, "undefined method `#{method}' for \"#{self}\":#{self.class}"
+  end
+
+  def respond_to?(method)
+    super
+  end
+
+  protected
+
+    def to_ruby_string(cell)
+      Java::String.new(cell.getValue).to_s
+    end
+    # Try to recover from network related exceptions. e.g. hbase has been restarted and the
+    # cached connections in @tables are no longer valid. Every method in this class (except connect_table)
+    # should have its code wrapped by a call to this method.
+    def safe_exec
+      yield
+    rescue IOException => e
+      puts "A network error occured: #{e.message}. Trying to recover..."
+      init_connection
+      begin
+        yield
+      rescue Exception, Java::Exception => e2
+        if e2.class == e.class
+          puts "Failed to recover the connection."
+        else
+          puts "Failed to recover the connection but got a different error this time: #{e2.message}."
+        end
+        puts "Stack trace:"
+        puts e2.backtrace.join("\n")
+
+        if e2.kind_of?(NativeException)
+          raise BigRecordDriver::JavaError, e2.message
+        else
+          raise e2
+        end
+      end
+      puts "Connection recovered successfully..."
+    rescue Exception => e
+      puts "\n#{e.class.name}: #{e.message}"
+      puts e.backtrace.join("\n")
+      raise e
+    end
+end
+
+end

data/lib/big_record_driver/cassandra_driver/server.rb

@@ -0,0 +1,135 @@
+require File.dirname(__FILE__) + '/../column_descriptor'
+require File.dirname(__FILE__) + '/../exceptions'
+require File.dirname(__FILE__) + '/../bigrecord_server'
+
+module BigRecordDriver
+class CassandraServer < BigRecordServer
+  include_class "org.apache.cassandra.service.Cassandra"
+  include_class "org.apache.cassandra.service.InvalidRequestException"
+  include_class "org.apache.cassandra.service.NotFoundException"
+  include_class "org.apache.cassandra.service.UnavailableException"
+  include_class "org.apache.cassandra.service.column_t"
+  include_class "org.apache.thrift.TException"
+  include_class "org.apache.thrift.protocol.TBinaryProtocol"
+  include_class "org.apache.thrift.transport.TSocket"
+  include_class "org.apache.thrift.transport.TTransport"
+
+  def configure(config = {})
+    config[:adr]        ||= 'localhost'
+    config[:port]       ||= 9160
+    @config = config
+    init_connection
+  end
+  
+  def update(table_name, row, values, timestamp=nil)
+    safe_exec do
+      return nil unless row
+      timestamp = 0 unless timestamp
+      values.each do |column, value|  
+      @cassandraClient.insert(table_name.to_s, row, column, value.to_bytes, timestamp, true)
+      end
+      row
+    end
+  end
+
+  def get(table_name, row, column, options={})
+    safe_exec do
+      return nil unless row
+      # Retreive only the last version by default
+      options[:num_versions] ||= 1
+
+      # validate the arguments
+      raise ArgumentError, "num_versions must be >= 1" unless options[:num_versions] >= 1
+      begin
+        if options[:timestamp]
+          raw_data = @cassandraClient.get_columns_since(table_name.to_s, row, column, options[:timestamp])
+        else
+          raw_data = @cassandraClient.get_column(table_name.to_s, row, column)
+        end
+      rescue NotFoundException => e2
+        puts e2.message
+        puts e2.class
+      end
+      # Return either a single value or an array, depending on the number of version that have been requested
+      if options[:timestamp]
+        return [] unless raw_data
+        max_index = raw_data.length > options[:num_versions] || raw_data.length
+        0..max_index.each do |i|
+          arr[i] = Java::String.new(raw_data[i].value).to_s
+        end
+        arr
+      else
+        return nil unless raw_data
+        Java::String.new(raw_data.value).to_s
+      end
+    end
+  end
+
+  def get_columns(table_name, row, columns, options={})
+    safe_exec do
+      return nil unless row
+      raise ArgumentError, "timestamp on get_columns is not currently supported with cassandra" if options[:timestamp]
+      arr = []
+      columns.each_with_index do |col, i|
+        begin
+          if col[-1,1] == ':'
+            arr + @cassandraClient.get_slice(table_name.to_s, row, col, -1, -1).to_a
+          else
+            arr + @cassandraClient.get_column(table_name.to_s, row, col)
+          end
+        rescue NotFoundException => e2
+          puts e2.message
+          puts e2.class
+        end
+      end
+      unless !result or result.isEmpty
+        values = {}
+        arr.each do |column_t|
+          values[column_t.getColumnName.to_s] = Java::String.new(column_t.value).to_s
+        end
+        values["attribute:id"] = row
+        values
+      end
+      
+    end
+  end
+
+## It's currently impossible to have compliant delete with cassandra,
+## you would have to do it famiyl by family
+#  def delete(table_name, row)
+#    safe_exec do
+#      table.remove(table_name, row, ??, ??, true)
+#    end
+#  end
+
+  def ping
+    safe_exec do
+      @socket.isOpen
+    end
+  end
+
+  def table_names
+    safe_exec do
+      @cassandraClient.getStringListProperty("tables") #.collect{|td| Java::String.new(td.getName).to_s}
+    end
+  end
+
+  def table_exists?(table_name)
+    !@cassandraClient.describeTable(table_name.to_s).include?("not found.")
+  end
+
+  private
+    def init_connection
+      @socket = TSocket.new(@config[:adr], @config[:port]);
+      binary_protocol = TBinaryProtocol.new(@socket, false, false);
+      @cassandraClient = Cassandra::Client.new(binary_protocol);
+      @socket.open;
+    end
+end
+end
+
+port = ARGV[0]
+port ||= 45000
+DRb.start_service("druby://:#{port}", BigRecordDriver::CassandraServer.new)
+puts "Started drb server on port #{port}."
+DRb.thread.join

data/lib/big_record_driver/client.rb

@@ -0,0 +1,36 @@
+require 'rubygems'
+require 'activesupport' 
+require 'set'
+require 'drb'
+
+module BigRecordDriver
+  class Client
+    
+    def initialize(config={}) # :nodoc:
+      config = config.symbolize_keys
+      config[:drb_host]     ||= '127.0.0.1'
+      config[:drb_port]     ||= 40000
+      
+      @config = config
+
+      DRb.start_service('druby://127.0.0.1:0')
+      begin
+        @server = DRbObject.new(nil, "druby://#{@config[:drb_host]}:#{@config[:drb_port]}")
+      rescue DRb::DRbConnError
+        raise ConnectionError, "Failed to connect to the DRb server (jruby) " +
+                                      "at #{@config[:drb_host]}:#{@config[:drb_port]}."
+      end
+      @server.configure(@config)
+    end
+    
+    # Delegate the methods to the server
+    def method_missing(method, *args)
+      @server.send(method, *args)
+    end
+    
+    def respond_to?(method)
+      super
+    end
+    
+  end
+end

data/lib/big_record_driver/column_descriptor.rb

@@ -0,0 +1,23 @@
+module BigRecordDriver
+
+  class ColumnDescriptor
+
+    attr_accessor :name
+    attr_accessor :versions
+    attr_accessor :in_memory
+    attr_accessor :bloom_filter
+    attr_accessor :compression
+
+    def initialize(name, options={})
+      raise ArgumentError, "name is mandatory" unless name
+
+      @name = name.to_s
+      @versions         = options[:versions]
+      @in_memory        = options[:in_memory]
+      @bloom_filter     = options[:bloom_filter]
+      @compression      = options[:compression]
+    end
+
+  end
+
+end

data/lib/big_record_driver/driver_manager.rb

@@ -0,0 +1,34 @@
+module BigRecordDriver
+
+  class DriverManager
+    class << self
+
+      def set_cmd(db = 'hbase')
+        @@CMD = File.dirname(__FILE__) + "/../../bin/#{db}-driver"
+      end
+      DriverManager.set_cmd
+      def start(port = 40005)
+        `ruby #{@@CMD} start -p #{port.to_s}`
+      end
+
+      def restart(port = 40005)
+        `ruby #{@@CMD} restart -p #{port.to_s}`
+      end
+
+      def stop(port = 40005)
+        `ruby #{@@CMD} stop -p #{port.to_s}`
+      end
+
+      def running?(port = 40005)
+        status = `ruby #{@@CMD} status -p #{port.to_s}`
+        status == "Running.\n"
+      end
+
+      def silent_start(port = 40005)
+        start(port) unless running?(port)
+      end
+    end
+
+  end
+
+end

data/lib/big_record_driver/exceptions.rb

@@ -0,0 +1,12 @@
+module BigRecordDriver
+  class BigDBError < StandardError
+  end
+  class TableNotFound < BigDBError
+  end
+  class TableAlreadyExists < BigDBError
+  end
+  class JavaError < BigDBError
+  end
+  class ConnectionError < BigDBError
+  end
+end