hipster_sql_to_hbase 0.1.2

data/lib/adapter/hbase/hbase_constants.rb

@@ -0,0 +1,14 @@
+#
+# Autogenerated by Thrift Compiler (0.9.1)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+module Apache
+  module Hadoop
+    module Hbase
+      module Thrift
+      end
+    end
+  end
+end

data/lib/adapter/hbase/hbase_types.rb

@@ -0,0 +1,282 @@
+#
+# Autogenerated by Thrift Compiler (0.9.1)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+module Apache
+  module Hadoop
+    module Hbase
+      module Thrift
+        # TCell - Used to transport a cell value (byte[]) and the timestamp it was
+# stored with together as a result for get and getRow methods. This promotes
+# the timestamp of a cell to a first-class value, making it easy to take
+# note of temporal data. Cell is used all the way from HStore up to HTable.
+        class TCell
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          VALUE = 1
+          TIMESTAMP = 2
+
+          FIELDS = {
+            VALUE => {:type => ::Thrift::Types::STRING, :name => 'value', :binary => true},
+            TIMESTAMP => {:type => ::Thrift::Types::I64, :name => 'timestamp'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An HColumnDescriptor contains information about a column family
+# such as the number of versions, compression settings, etc. It is
+# used as input when creating a table or adding a column.
+        class ColumnDescriptor
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          NAME = 1
+          MAXVERSIONS = 2
+          COMPRESSION = 3
+          INMEMORY = 4
+          BLOOMFILTERTYPE = 5
+          BLOOMFILTERVECTORSIZE = 6
+          BLOOMFILTERNBHASHES = 7
+          BLOCKCACHEENABLED = 8
+          TIMETOLIVE = 9
+
+          FIELDS = {
+            NAME => {:type => ::Thrift::Types::STRING, :name => 'name', :binary => true},
+            MAXVERSIONS => {:type => ::Thrift::Types::I32, :name => 'maxVersions', :default => 3},
+            COMPRESSION => {:type => ::Thrift::Types::STRING, :name => 'compression', :default => %q"NONE"},
+            INMEMORY => {:type => ::Thrift::Types::BOOL, :name => 'inMemory', :default => false},
+            BLOOMFILTERTYPE => {:type => ::Thrift::Types::STRING, :name => 'bloomFilterType', :default => %q"NONE"},
+            BLOOMFILTERVECTORSIZE => {:type => ::Thrift::Types::I32, :name => 'bloomFilterVectorSize', :default => 0},
+            BLOOMFILTERNBHASHES => {:type => ::Thrift::Types::I32, :name => 'bloomFilterNbHashes', :default => 0},
+            BLOCKCACHEENABLED => {:type => ::Thrift::Types::BOOL, :name => 'blockCacheEnabled', :default => false},
+            TIMETOLIVE => {:type => ::Thrift::Types::I32, :name => 'timeToLive', :default => -1}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A TRegionInfo contains information about an HTable region.
+        class TRegionInfo
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          STARTKEY = 1
+          ENDKEY = 2
+          ID = 3
+          NAME = 4
+          VERSION = 5
+          SERVERNAME = 6
+          PORT = 7
+
+          FIELDS = {
+            STARTKEY => {:type => ::Thrift::Types::STRING, :name => 'startKey', :binary => true},
+            ENDKEY => {:type => ::Thrift::Types::STRING, :name => 'endKey', :binary => true},
+            ID => {:type => ::Thrift::Types::I64, :name => 'id'},
+            NAME => {:type => ::Thrift::Types::STRING, :name => 'name', :binary => true},
+            VERSION => {:type => ::Thrift::Types::BYTE, :name => 'version'},
+            SERVERNAME => {:type => ::Thrift::Types::STRING, :name => 'serverName', :binary => true},
+            PORT => {:type => ::Thrift::Types::I32, :name => 'port'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A Mutation object is used to either update or delete a column-value.
+        class Mutation
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ISDELETE = 1
+          COLUMN = 2
+          VALUE = 3
+          WRITETOWAL = 4
+
+          FIELDS = {
+            ISDELETE => {:type => ::Thrift::Types::BOOL, :name => 'isDelete', :default => false},
+            COLUMN => {:type => ::Thrift::Types::STRING, :name => 'column', :binary => true},
+            VALUE => {:type => ::Thrift::Types::STRING, :name => 'value', :binary => true},
+            WRITETOWAL => {:type => ::Thrift::Types::BOOL, :name => 'writeToWAL', :default => true}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A BatchMutation object is used to apply a number of Mutations to a single row.
+        class BatchMutation
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ROW = 1
+          MUTATIONS = 2
+
+          FIELDS = {
+            ROW => {:type => ::Thrift::Types::STRING, :name => 'row', :binary => true},
+            MUTATIONS => {:type => ::Thrift::Types::LIST, :name => 'mutations', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Apache::Hadoop::Hbase::Thrift::Mutation}}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # For increments that are not incrementColumnValue
+# equivalents.
+        class TIncrement
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          TABLE = 1
+          ROW = 2
+          COLUMN = 3
+          AMMOUNT = 4
+
+          FIELDS = {
+            TABLE => {:type => ::Thrift::Types::STRING, :name => 'table', :binary => true},
+            ROW => {:type => ::Thrift::Types::STRING, :name => 'row', :binary => true},
+            COLUMN => {:type => ::Thrift::Types::STRING, :name => 'column', :binary => true},
+            AMMOUNT => {:type => ::Thrift::Types::I64, :name => 'ammount'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # Holds row name and then a map of columns to cells.
+        class TRowResult
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ROW = 1
+          COLUMNS = 2
+
+          FIELDS = {
+            ROW => {:type => ::Thrift::Types::STRING, :name => 'row', :binary => true},
+            COLUMNS => {:type => ::Thrift::Types::MAP, :name => 'columns', :key => {:type => ::Thrift::Types::STRING, :binary => true}, :value => {:type => ::Thrift::Types::STRUCT, :class => ::Apache::Hadoop::Hbase::Thrift::TCell}}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A Scan object is used to specify scanner parameters when opening a scanner.
+        class TScan
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          STARTROW = 1
+          STOPROW = 2
+          TIMESTAMP = 3
+          COLUMNS = 4
+          CACHING = 5
+          FILTERSTRING = 6
+
+          FIELDS = {
+            STARTROW => {:type => ::Thrift::Types::STRING, :name => 'startRow', :binary => true, :optional => true},
+            STOPROW => {:type => ::Thrift::Types::STRING, :name => 'stopRow', :binary => true, :optional => true},
+            TIMESTAMP => {:type => ::Thrift::Types::I64, :name => 'timestamp', :optional => true},
+            COLUMNS => {:type => ::Thrift::Types::LIST, :name => 'columns', :element => {:type => ::Thrift::Types::STRING, :binary => true}, :optional => true},
+            CACHING => {:type => ::Thrift::Types::I32, :name => 'caching', :optional => true},
+            FILTERSTRING => {:type => ::Thrift::Types::STRING, :name => 'filterString', :binary => true, :optional => true}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An IOError exception signals that an error occurred communicating
+# to the Hbase master or an Hbase region server.  Also used to return
+# more general Hbase error conditions.
+        class IOError < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An IllegalArgument exception indicates an illegal or invalid
+# argument was passed into a procedure.
+        class IllegalArgument < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An AlreadyExists exceptions signals that a table with the specified
+# name already exists
+        class AlreadyExists < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+      end
+    end
+  end
+end

data/lib/datatype_extras.rb

@@ -0,0 +1,18 @@
+begin
+  " ".squish
+rescue
+  class String
+
+    def squish
+      dup.squish!
+    end
+
+    def squish!
+      gsub!(/\A[[:space:]]+/, '')
+      gsub!(/[[:space:]]+\z/, '')
+      gsub!(/[[:space:]]+/, ' ')
+      self
+    end
+    
+  end
+end

data/lib/executor.rb

@@ -0,0 +1,91 @@
+module HipsterSqlToHbase
+
+  # This class takes the magic provided by the ThriftCallGroup and
+  # turns it into an HBase reality by opening a connection to a 
+  # specified host and port and executing the pertinent Thrift calls.
+  class Executor
+    @@host=nil
+    @@port=nil
+    
+    # Set class variable <b>host</b> when instantiated.
+    def host=(host_s)
+      self.class.host=host_s
+    end
+    
+    # Get class variable <b>host</b> when instantiated.
+    def host
+      self.class.host
+    end
+    
+    # Set class variable <b>host</b> when NOT instantiated.
+    def self.host=(host_s)
+      @@host=host_s
+    end
+    
+    # Get class variable <b>host</b> when NOT instantiated.
+    def self.host
+      @@host
+    end
+    
+    # Set class variable <b>port</b> when instantiated.
+    def port=(port_n)
+      self.class.port=port_n
+    end
+    
+    # Get class variable <b>port</b> when instantiated.
+    def port
+      self.class.port
+    end
+    
+    # Set class variable <b>port</b> when NOT instantiated.
+    def self.port=(port_n)
+      @@port=port_n
+    end
+    
+    # Get class variable <b>port</b> when NOT instantiated.
+    def self.port
+      @@port
+    end
+    
+    # Initialize a Thrift connection to the specified host and port
+    # and execute the provided ThriftCallGroup object.
+    def execute(thrift_call_group,host_s=nil,port_n=nil,incr=false)
+      @@host = host_s if !host_s.nil?
+      @@port = port_n if !port_n.nil?
+      socket = Thrift::Socket.new(@@host, @@port)
+
+      transport = Thrift::BufferedTransport.new(socket)
+      transport.open
+
+      protocol = Thrift::BinaryProtocol.new(transport)
+      client = HBase::Client.new(protocol)
+      
+      results = []
+      
+      if incr
+        not_incr = true
+        c_row = 0
+      end
+      
+      thrift_call_group.each do |thrift_call|
+        if incr
+          if not_incr
+            c_row = increment_table_row_index(thrift_call[:arguments][0],thrift_call_group.length,client) 
+            not_incr = false
+          end
+          c_row += 1
+          thrift_call[:arguments][1] = c_row.to_s 
+        end
+        results << client.send(thrift_call[:method],*thrift_call[:arguments])
+      end
+      
+      results
+    end
+    
+    private 
+    
+    def increment_table_row_index(table_name,amount,client)
+      client.incrementAndReturn("index:#{table_name}",amount)
+    end
+  end
+end

data/lib/hipster_sql_to_hbase.rb

@@ -0,0 +1,167 @@
+require "date"
+require "time"
+require "treetop"
+require_relative "datatype_extras"
+require_relative "sql_treetop_load"
+require_relative "result_tree_to_hbase_converter"
+require_relative "result_tree_to_json_converter"
+
+# This module provides the methods necessary to parse valid SQL
+# sentences into:
+#
+# * Treetop Syntax Trees
+# * Hash Trees
+# * JSON Trees
+# * HBase (Thrift)
+#
+# It also allows to directly execute valid SQL sentences as Thrift methods.
+module HipsterSqlToHbase
+  
+  # Treetop base SQL parser.
+  #
+  # Forked from Scott Taylor's guillotine and perfected by Jean Lescure.
+  #
+  # https://www.github.com/smtlaissezfaire/guillotine/tree/master/lib/guillotine/parser
+  #
+  # https://www.github.com/jeanlescure/hipster_sql_to_hbase
+  class SyntaxParser < SQLParser
+    class ItemsNode < Treetop::Runtime::SyntaxNode
+      def values
+        items.values.unshift(item.value)
+      end
+    end
+
+    class ItemNode < Treetop::Runtime::SyntaxNode
+      def values
+        [value]
+      end
+      
+      def value
+        text_value.to_sym
+      end
+    end
+  end
+  
+  # This class provides the initial parsed result with the methods
+  # necessary to be transformed into HBase (Thrift) or JSON, as well
+  # as to be executed as a Thrift method.
+  class ResultTree < Hash
+    def initialize(hash)
+      hash.each do |k,v|
+        self[k] = v
+      end
+    end
+    
+    # Transforms itself into an HBase (Thrift) method.
+    # 
+    # === example:
+    #   rt = HipsterSqlToHbase::ResultTree.new({...})
+    #   rt.to_hbase #=> {:method => 'mutateRow', :arguments => {...}}
+    def to_hbase
+      HipsterSqlToHbase::ResultTreeToHbaseConverter.new().convert self
+    end
+    
+    # Transforms itself into a JSON object.
+    # 
+    # === example:
+    #   user = HipsterSqlToHbase::ResultTree.new({...})
+    #   user.to_json #=> {:user => {:user_name => 'bob', :pass => 'bob1234', ...}}
+    def to_json
+      HipsterSqlToHbase::ResultTreeToJsonConverter.new().convert self
+    end
+    
+    # Executes itself as an HBase (Thrift) method.
+    # Short for running .to_hbase and then sending the result to the executor.
+    # 
+    # === example:
+    #   my_call = HipsterSqlToHbase::ResultTree.new({...})
+    #   my_call.execute 
+    #
+    # <em>if no arguments are passed the executor will use any previously set host 
+    # and port.</em>
+    def execute(host=nil,port=nil)
+      self.to_hbase.execute(host,port)
+    end
+  end
+  
+  class << self
+    # Generate a Treetop syntax tree from a valid, SQL string.
+    #
+    # === example:
+    #   HipsterSqlToHbase.parse_syntax "INSERT INTO users (user,password) VALUES ('user1','pass123'),('user2','2girls1pass')"
+    #
+    # === outputs:
+    #   #<Class:#<Treetop::Runtime::SyntaxNode:0x3274c98>>
+    def parse_syntax(string)
+      HipsterSqlToHbase::SyntaxParser.new.parse(string.squish)
+    end
+    
+    # Generate a <b>HipsterSqlToHbase</b>::<b>ResultTree</b> from a valid, SQL string.
+    #
+    # === example:
+    #   HipsterSqlToHbase.parse_tree "SELECT user,password FROM users WHERE id=1"
+    #
+    # === outputs:
+    #   {
+    #    :query_type=>:select,
+    #    :query_hash=>{
+    #      :select=>["user", "password"],
+    #      :from=>"users",
+    #      :where=>[{:column=>"id", :condition=>:"=", :value=>1}],
+    #      :limit=>nil,
+    #      :order_by=>nil
+    #    }
+    #   }
+    def parse_tree(string)
+      syntax_tree = parse_syntax(string)
+      return nil if syntax_tree.nil?
+      HipsterSqlToHbase::ResultTree.new({:query_type => syntax_tree.query_type, :query_hash => syntax_tree.tree})
+    end
+    
+    # Generate a <b>HipsterSqlToHbase</b>::<b>ThriftCallGroup</b> from a valid, SQL string.
+    #
+    # === example:
+    #    HipsterSqlToHbase.parse "INSERT INTO `users` (`user`,`pass`) VALUES ('andy','w00dy'),('zaphod','b33bl3br0x')"
+    #
+    # === outputs:
+    #    [
+    #      {
+    #        :method=>"mutateRow", 
+    #        :arguments=>[
+    #          "users", 
+    #          "c6af1d5b-01d7-477c-9539-f35a1e0758e2", 
+    #          [<Apache::Hadoop::Hbase::Thrift::Mutation>, <Apache::Hadoop::Hbase::Thrift::Mutation>], 
+    #          {}
+    #        ]
+    #      }, 
+    #      {
+    #        :method=>"mutateRow", 
+    #        :arguments=>[
+    #          "users",
+    #          "b630cf3c-c969-420e-afd9-b646466a6743", 
+    #          [<Apache::Hadoop::Hbase::Thrift::Mutation>, <Apache::Hadoop::Hbase::Thrift::Mutation>], 
+    #          {}
+    #        ]
+    #      }
+    #    ]
+    #
+    # === note:
+    #
+    # The resulting ThriftCallGroup can be executed simply by calling its <tt>.execute()</tt> method.
+    def parse(string)
+      result_tree = parse_tree(string)
+      return nil if result_tree.nil?
+      result_tree.to_hbase
+    end
+    
+    # Generates and automatically executes a <b>HipsterSqlToHbase</b>::<b>ThriftCallGroup</b> 
+    # from a valid, SQL string.
+    #
+    # The returned value varies depending on the SQL query type (i.e. SELECT, INSERT, etc).
+    def execute(string,host=nil,port=nil)
+      parsed_tree = parse_tree(string)
+      return nil if parsed_tree.nil?
+      parsed_tree.execute(host,port)
+    end
+  end
+end