hipster_sql_to_hbase 0.1.2

data/lib/result_tree_to_hbase_converter.rb

@@ -0,0 +1,119 @@
+require 'securerandom'
+require 'thrift'
+require_relative "executor"
+
+require File.join(File.dirname(__FILE__), 'adapter', 'hbase')
+
+module HipsterSqlToHbase
+  
+  # This class provides the method necessary to execute the Thrift result
+  # generated after parsing the SQL sentence.
+  class ThriftCallGroup < Array
+    @incr = false
+    def initialize(arr,incr=false)
+      arr.each do |v|
+        self << v
+      end
+      @incr = incr
+    end
+    def execute(host=nil,port=nil)
+      HipsterSqlToHbase::Executor.new().execute(self,host,port,@incr)
+    end
+  end
+  
+  # This class takes care of all HBase (Thrift) conversion magic by transforming
+  # the ResultTree objects into ThriftCallGroup objects.
+  class ResultTreeToHbaseConverter
+  
+    # Depending on the SQL sentence type, call the appropriate function.
+    def convert(result_tree)
+      send("#{result_tree[:query_type].to_s}_sentence",result_tree[:query_hash])
+    end
+    
+    # When SQL sentence is an INSERT query generate the Thrift mutations according
+    # to the specified query values.
+    def insert_sentence(hash)
+      thrift_method = "mutateRow"
+      thrift_table = hash[:into]
+      thrift_calls = []
+      hash[:values].each do |value_set|
+        thrift_row = SecureRandom.uuid
+        thrift_mutations = []
+        i = 0
+        hash[:columns].each do |col|
+          thrift_mutations << HBase::Mutation.new(column: col, value: value_set[i].to_s)
+          i += 1
+        end
+        thrift_calls << {:method => thrift_method,:arguments => [thrift_table,thrift_row,thrift_mutations,{}]}
+      end
+      HipsterSqlToHbase::ThriftCallGroup.new(thrift_calls,true)
+    end
+    
+    # When SQL sentence is a SELECT query generate the Thrift filters according
+    # to the specified query values.
+    def select_sentence(hash)
+      thrift_method = "getRowsByScanner"
+      thrift_table = hash[:from]
+      thrift_columns = hash[:select]
+      thrift_filters = recurse_where(hash[:where] || [])
+      
+      HipsterSqlToHbase::ThriftCallGroup.new([{:method => thrift_method,:arguments => [thrift_table,thrift_columns,thrift_filters,{}]}])
+    end
+    
+    # When SQL sentence is a CREATE TABLE query generate the Thrift column descriptors/families
+    # in accordance to the specified query values.
+    def create_table_sentence(hash)
+      thrift_method = "createTable"
+      thrift_table = hash[:table]
+      thrift_columns = []
+      hash[:columns].each do |col_name|
+        col_descriptor = Hbase::ColumnDescriptor.new
+        col_descriptor.name = col_name
+        thrift_columns << col_descriptor
+      end
+      
+      HipsterSqlToHbase::ThriftCallGroup.new([{:method => thrift_method,:arguments => [thrift_table,thrift_columns]}])
+    end
+    
+    private
+    
+    # Format the scanner filter for thrift based on the where clause(s)
+    # of a SELECT query.
+    def recurse_where(where_arr)
+      result_arr = []
+      where_arr.each do |val|
+        if val.is_a? Hash
+          result_arr << filters_from_key_value_pair(val)
+        elsif val.is_a? Array
+          result_arr << "(#{recurse_where(val)})"
+        elsif val.is_a? String
+          result_arr << val
+        else
+          raise "Recursive where undefined error."
+        end
+      end
+      result_arr.join(" ")
+    end
+    
+    # Generate a Thrift QualifierFilter and ValueFilter from key value pair.
+    def filters_from_key_value_pair(kvp)
+      if (kvp[:condition].to_s != "LIKE")
+        "(ValueFilter(#{kvp[:condition]},'binary:#{kvp[:value]}') AND DependentColumnFilter('#{kvp[:column]}',''))"
+      else
+        kvp[:value] = Regexp.escape(kvp[:value])
+        kvp[:value].sub!(/^%/,"^.*")
+        kvp[:value].sub!(/%$/,".*$")
+        while kvp[:value].match(/([^\\]{1,1})%/)
+          kvp[:value].sub!(/([^\\]{1,1})%/,"#{$1}.*?")
+        end
+        kvp[:value].sub!(/^_/,"^.")
+        kvp[:value].sub!(/_$/,".$")
+        while kvp[:value].match(/([^\\]{1,1})_/)
+          kvp[:value].sub!(/([^\\]{1,1})_/,"#{$1}.")
+        end
+        "(ValueFilter(=,'regexstring:#{kvp[:value]}') AND DependentColumnFilter('#{kvp[:column]}',''))"
+      end
+    end
+    
+  end
+end

data/lib/result_tree_to_json_converter.rb

@@ -0,0 +1,40 @@
+require 'json'
+
+require File.join(File.dirname(__FILE__), 'adapter', 'hbase')
+
+module HipsterSqlToHbase
+  class ResultTreeToJsonConverter
+    def convert(result_tree)
+      send("#{result_tree[:query_type].to_s}_sentence",result_tree[:query_hash])
+    end
+    def insert_sentence(hash)
+      table = hash[:into]
+      objects = []
+      hash[:values].each do |value_set|
+        object = {}
+        i = 0
+        hash[:columns].each do |col|
+          object[col.to_sym] = value_set[i]
+          i += 1
+        end
+        objects << object
+      end
+      JSON.generate({:write=>{:table=>table,:objects=>objects}})
+    end
+    def select_sentence
+      
+    end
+    def create_table_sentence
+      
+    end
+    
+    private
+    def qualifier_filters_from_cols(cols)
+      
+    end
+    def value_filters_from_vals(vals)
+      
+    end
+    
+  end
+end

data/lib/sql_parser/sql.treetop

@@ -0,0 +1,21 @@
+grammar SQL
+  include SQLSelect
+  include SQLDelete
+  include SQLTruncate
+  include DropTable #TODO: Fix my name
+  include SQLTransaction
+  include SQLInsert
+  include SQLShowTables
+  include SQLCreateTable
+
+  rule sql_statement
+    select_expression     /
+    delete                /
+    truncate              /
+    drop_table            /
+    transaction_statement /
+    insert                /
+    show_tables           /
+    create_table
+  end
+end

data/lib/sql_parser/sql_chars.treetop

@@ -0,0 +1,5 @@
+grammar SQLChars
+  rule sql_embeded_language_char
+    "[" / "]"
+  end
+end

data/lib/sql_parser/sql_create_table.treetop

@@ -0,0 +1,47 @@
+grammar SQLCreateTable
+  include SQLRowSupport
+  include SQLDataTypes
+
+  rule create_table
+    "CREATE" SPACE+ "TABLE" SPACE+ table_name SPACE+ OPEN_PARENS columns_and_datatypes CLOSE_PARENS {
+      def eval
+        options = {
+          :columns    => columns_and_datatypes.eval,
+          :table_name => table_name.eval
+        }
+        
+        options
+      end
+      def query_type
+        :create_table
+      end
+      def tree
+        values = eval
+        {
+          :table => values[:table_name],
+          :columns => values[:columns]
+        }
+      end
+    }
+  end
+
+  rule columns_and_datatypes
+    column_with_datatype COMMA columns_and_datatypes {
+      def eval
+        all = column_with_datatype.eval + columns_and_datatypes.eval
+        all.flatten!
+        all
+      end
+    }
+    /
+    column_with_datatype
+  end
+
+  rule column_with_datatype
+    SPACE* column_name SPACE+ datatype SPACE* {
+      def eval
+        [column_name.eval]
+      end
+    }
+  end
+end

data/lib/sql_parser/sql_datatypes.treetop

@@ -0,0 +1,71 @@
+grammar SQLDataTypes
+  include SQLPrimitives
+
+  rule datatype
+    bit_field /
+    numeric_type /
+    char_type
+  end
+  
+  rule char_type
+    char_field field_size_with_space?
+  end
+  
+  rule char_field
+    VARCHAR_KEYWORD
+  end
+
+  rule numeric_type
+    int_type / decimal_type
+  end
+
+  rule decimal_type
+    decimal_field field_size_with_space? unsigned? zerofill?
+  end
+
+  rule decimal_field
+    REAL_KEYWORD    /
+    DOUBLE_KEYWORD  /
+    FLOAT_KEYWORD   /
+    DECIMAL_KEYWORD /
+    NUMERIC_KEYWORD
+  end
+
+  rule int_type
+    int_field int_options
+  end
+
+  rule int_field
+    TINY_INT_KEYWORD     /
+    SMALL_INT_KEYWORD    /
+    INTEGER_KEYWORD      /
+    INT_KEYWORD          /
+    BIG_INT_KEYWORD
+  end
+
+  rule int_options
+    field_size_with_space? unsigned? zerofill?
+  end
+
+  rule bit_field
+    "BIT" field_size?
+  end
+
+  rule unsigned
+    SPACE UNSIGNED_KEYWORD
+  end
+
+  rule zerofill
+    SPACE ZEROFILL_KEYWORD
+  end
+
+  rule field_size_with_space
+    SPACE* field_size
+  end
+
+  rule field_size
+    OPEN_PARENS SPACE* integer SPACE* CLOSE_PARENS /
+    OPEN_PARENS SPACE* integer SPACE* COMMA SPACE* integer SPACE* CLOSE_PARENS
+  end
+end
+  

data/lib/sql_parser/sql_delete.treetop

@@ -0,0 +1,64 @@
+grammar SQLDelete
+  include SQLRowSupport
+  include SQLWhereCondition
+  include SQLOrderByClause
+  include SQLLimit
+  include SQLHelpers
+
+  rule delete
+    single_table_delete
+  end
+
+  rule single_table_delete
+    common_delete_clause
+    table_name
+    where_condition_or_empty
+    order_by_condition_or_empty
+    limit_condition_or_empty {
+      def eval
+        DeleteStatement.new(
+          table_name.eval, 
+          where_condition_or_empty.eval, 
+          order_by_condition_or_empty.eval, 
+          limit_condition_or_empty.eval
+        )
+      end
+    }
+  end
+
+  rule common_delete_clause
+    "DELETE" SPACE
+    optional_delete_directives
+    "FROM" SPACE
+  end
+
+  rule optional_delete_directives
+    optional_low_priority
+    optional_quick
+    optional_ignore
+  end
+
+  rule optional_low_priority
+    low_priority / EMPTY_STRING
+  end
+
+  rule optional_quick
+    quick / EMPTY_STRING
+  end
+
+  rule optional_ignore
+    ignore / EMPTY_STRING
+  end
+
+  rule ignore
+    "IGNORE" SPACE
+  end
+
+  rule quick
+    "QUICK" SPACE
+  end
+
+  rule low_priority
+    "LOW_PRIORITY" SPACE
+  end
+end

data/lib/sql_parser/sql_drop_table.treetop

@@ -0,0 +1,26 @@
+grammar DropTable
+  include SQLPrimitives
+  include SQLRowSupport
+
+  rule drop_table
+    "DROP" (temporary)? SPACE "TABLE" SPACE
+    table_name
+    (restrict / cascade)? {
+      def eval
+        DataStore.drop_table(table_name.eval.to_sym)
+      end
+    }
+  end
+
+  rule temporary
+    SPACE "TEMPORARY"
+  end
+
+  rule restrict
+    SPACE "RESTRICT"
+  end
+
+  rule cascade
+    SPACE "CASCADE"
+  end
+end

data/lib/sql_parser/sql_from_clause.treetop

@@ -0,0 +1,12 @@
+grammar SQLFromClause
+  include SQLPrimitives
+  include SQLRowSupport
+
+  rule from
+    "FROM" SPACE one_or_more_table_names {
+      def tables
+        one_or_more_table_names.eval
+      end
+    }
+  end
+end

data/lib/sql_parser/sql_group_by_clause.treetop

@@ -0,0 +1,15 @@
+grammar SQLGroupByClause
+  include SQLPrimitives
+  include SQLRowSupport
+  include SQLHelpers
+
+  rule group_by
+    "GROUP BY " one_or_more_column_names {
+      def eval
+        columns = one_or_more_column_names.eval
+        columns_as_syms = columns.map { |column| column.to_sym }
+        GroupBy.new(*columns_as_syms)
+      end
+    }
+  end
+end

data/lib/sql_parser/sql_helpers.treetop

@@ -0,0 +1,19 @@
+grammar SQLHelpers
+  rule where_condition_or_empty
+    SPACE where_condition {
+      def eval
+        where_condition.eval
+      end
+    }
+    /  
+    EMPTY_STRING
+  end
+
+  rule limit_condition_or_empty
+    SPACE limit { def eval; limit.eval; end } / EMPTY_STRING
+  end
+
+  rule order_by_condition_or_empty
+    SPACE order_by { def eval; order_by.eval; end } / EMPTY_STRING
+  end
+end