rbhive 0.1.17 → 0.2.1

data/lib/rbhive/connection.rb

@@ -71,7 +71,7 @@ module RBHive
     
     def fetch(query)
       execute(query)
-      ResultSet.new(client.fetchAll)
+      ResultSet.new(client.fetchAll, client.getSchema)
     end
     
     def fetch_in_batch(query, batch_size=100)
@@ -107,26 +107,4 @@ module RBHive
       client.send(meth, *args)
     end
   end
-  
-  class ResultSet < Array
-    def initialize(rows)
-      super(rows.map {|r| r.split("\t") })
-    end
-    
-    def to_csv(out_file=nil)
-      output(",", out_file)
-    end
-    
-    def to_tsv(out_file=nil)
-      output("\t", out_file)
-    end
-    
-    private
-    
-    def output(sep, out_file)
-      sv = self.map { |r| r.join(sep) }.join("\n")
-      return sv if out_file.nil?
-      File.open(out_file, 'w') { |f| f << sv }
-    end
-  end
 end

data/lib/rbhive/result_set.rb

@@ -0,0 +1,33 @@
+module RBHive
+  class ResultSet < Array
+    def initialize(rows, schema=[])
+      @schema = SchemaDefinition.new(schema, rows.first)
+      super(rows.map {|r| @schema.coerce_row(r) })
+    end
+    
+    def column_names
+      @schema.column_names
+    end
+    
+    def column_type_map
+      @schema.column_type_map
+    end
+    
+    def to_csv(out_file=nil)
+      output(",", out_file)
+    end
+    
+    def to_tsv(out_file=nil)
+      output("\t", out_file)
+    end
+    
+    private
+    
+    def output(sep, out_file)
+      rows = self.map { |r| @schema.coerce_row_to_array(r).join(sep) }
+      sv = rows.join("\n")
+      return sv if out_file.nil?
+      File.open(out_file, 'w+') { |f| f << sv }
+    end
+  end
+end

data/lib/rbhive/schema_definition.rb

@@ -0,0 +1,60 @@
+module RBHive
+  class SchemaDefinition
+    attr_reader :schema
+  
+    TYPES = { 
+      :boolean => :to_s, 
+      :string => :to_s, 
+      :bigint => :to_i, 
+      :float => :to_f, 
+      :double => :to_f, 
+      :int => :to_i, 
+      :smallint => :to_i, 
+      :tinyint => :to_i
+    }
+  
+    def initialize(schema, example_row)
+      @schema = schema
+      @example_row = example_row.split("\t")
+    end
+  
+    def column_names
+      @column_names ||= begin
+        schema_names = @schema.fieldSchemas.map {|c| c.name.to_sym }
+        # Lets fix the fact that Hive doesn't return schema data for partitions on SELECT * queries
+        # For now we will call them :_p1, :_p2, etc. to avoid collisions.
+        offset = 0
+        while schema_names.length < @example_row.length
+          schema_names.push(:"_p#{offset+=1}")
+        end
+        schema_names
+      end
+    end
+  
+    def column_type_map
+      @column_type_map ||= column_names.inject({}) do |hsh, c| 
+        definition = @schema.fieldSchemas.find {|s| s.name.to_sym == c }
+        # If the column isn't in the schema (eg partitions in SELECT * queries) assume they are strings
+        hsh[c] = definition ? definition.type.to_sym : :string
+        hsh
+      end
+    end
+  
+    def coerce_row(row)
+      column_names.zip(row.split("\t")).inject({}) do |hsh, (column_name, value)|
+        hsh[column_name] = coerce_column(column_name, value)
+        hsh
+      end
+    end
+  
+    def coerce_column(column_name, value)
+      type = column_type_map[column_name]
+      conversion_method = TYPES[type]
+      conversion_method ? value.send(conversion_method) : value
+    end
+  
+    def coerce_row_to_array(row)
+      column_names.map { |n| row[n] }
+    end
+  end
+end

data/lib/rbhive/table_schema.rb

@@ -0,0 +1,88 @@
+module RBHive
+  class TableSchema
+    attr_accessor :name
+    attr_reader :columns, :partitions
+    def initialize(name, comment=nil, options={}, &blk)
+      @name, @comment = name, comment
+      @location = options[:location] || nil
+      @field_sep = options[:field_sep] || "\t"
+      @line_sep = options[:line_sep] || "\n"
+      @collection_sep = options[:collection_sep] || "|"
+      @columns = []
+      @partitions = []
+      instance_eval(&blk) if blk
+    end
+  
+    def column(name, type, comment=nil)
+      @columns << Column.new(name, type, comment)
+    end
+  
+    def partition(name, type, comment=nil)
+      @partitions << Column.new(name, type, comment)
+    end
+  
+    def create_table_statement()
+      %[CREATE #{external}TABLE #{table_statement}
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '#{@field_sep}'
+  LINES TERMINATED BY '#{@line_sep}'
+  COLLECTION ITEMS TERMINATED BY '#{@collection_sep}'
+  STORED AS TEXTFILE
+  #{location}]
+    end
+  
+    def replace_columns_statement
+      alter_columns_statement("REPLACE")
+    end
+  
+    def add_columns_statement
+      alter_columns_statement("ADD")
+    end
+  
+    def to_s
+      table_statement
+    end
+  
+    private
+
+    def external
+      @location.nil? ? '' : 'EXTERNAL '
+    end
+  
+    def table_statement
+      comment_string = (@comment.nil? ? '' : " COMMENT '#{@comment}'")
+      %[`#{@name}` #{column_statement}#{comment_string}\n#{partition_statement}]
+    end
+
+    def location
+      @location.nil? ? '' : "LOCATION '#{@location}'"
+    end
+  
+    def alter_columns_statement(add_or_replace)
+      %[ALTER TABLE `#{name}` #{add_or_replace} COLUMNS #{column_statement}]
+    end
+  
+    def column_statement
+      cols = @columns.join(",\n")
+      "(\n#{cols}\n)"
+    end
+  
+    def partition_statement
+      return "" if @partitions.nil? || @partitions.empty?
+      cols = @partitions.join(",\n")
+      "PARTITIONED BY (\n#{cols}\n)"
+    end
+  
+    class Column
+      attr_reader :name, :type, :comment
+      def initialize(name, type, comment=nil)
+        @name, @type, @comment = name, type, comment
+      end
+    
+      def to_s
+        comment_string = @comment.nil? ? '' : " COMMENT '#{@comment}'"
+        "`#{@name}` #{@type.to_s.upcase}#{comment_string}"
+      end
+    end
+  end
+end

data/lib/rbhive.rb

@@ -1,2 +1,4 @@
 require File.join(File.dirname(__FILE__), 'rbhive', 'connection')
-require File.join(File.dirname(__FILE__), 'rbhive', 'schema')
+require File.join(File.dirname(__FILE__), 'rbhive', 'table_schema')
+require File.join(File.dirname(__FILE__), 'rbhive', 'result_set')
+require File.join(File.dirname(__FILE__), 'rbhive', 'schema_definition')

metadata

@@ -1,49 +1,39 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: rbhive
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 1
-  - 17
-  version: 0.1.17
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Forward Internet Group
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-12-07 00:00:00 +00:00
+date: 2010-12-07 00:00:00.000000000 +00:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+dependencies:
+- !ruby/object:Gem::Dependency
   name: thrift
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2153944980 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 4
-        - 0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.4.0
   type: :runtime
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: *2153944980
 description: Simple lib for executing Hive queries
 email: andy@forward.co.uk
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - lib/rbhive.rb
 - lib/rbhive/connection.rb
-- lib/rbhive/schema.rb
+- lib/rbhive/table_schema.rb
+- lib/rbhive/result_set.rb
+- lib/rbhive/schema_definition.rb
 - lib/thrift/facebook_service.rb
 - lib/thrift/fb303_constants.rb
 - lib/thrift/fb303_types.rb
@@ -62,34 +52,26 @@ files:
 has_rdoc: true
 homepage: http://github.com/trfficbroker/rbhive
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Simple lib for executing Hive queries
 test_files: []
-

data/lib/rbhive/schema.rb

@@ -1,73 +0,0 @@
-class TableSchema
-  attr_accessor :name
-  attr_reader :columns, :partitions
-  def initialize(name, comment=nil, field_sep='\t', line_sep='\n', &blk)
-    @name, @comment, @field_sep, @line_sep = name, comment, field_sep, line_sep
-    @columns = []
-    @partitions = []
-    instance_eval(&blk) if blk
-  end
-  
-  def column(name, type, comment=nil)
-    @columns << Column.new(name, type, comment)
-  end
-  
-  def partition(name, type, comment=nil)
-    @partitions << Column.new(name, type, comment)
-  end
-  
-  def create_table_statement()
-    %[CREATE TABLE #{table_statement}
-ROW FORMAT DELIMITED
-FIELDS TERMINATED BY '#{@field_sep}'
-LINES TERMINATED BY '#{@line_sep}'
-STORED AS TEXTFILE]
-  end
-  
-  def replace_columns_statement
-    alter_columns_statement("REPLACE")
-  end
-  
-  def add_columns_statement
-    alter_columns_statement("ADD")
-  end
-  
-  def to_s
-    table_statement
-  end
-  
-  private
-  
-  def table_statement
-    comment_string = (@comment.nil? ? '' : " COMMENT '#{@comment}'")
-    %[`#{@name}` #{column_statement}#{comment_string}\n#{partition_statement}]
-  end
-  
-  def alter_columns_statement(add_or_replace)
-    %[ALTER TABLE `#{name}` #{add_or_replace} COLUMNS #{column_statement}]
-  end
-  
-  def column_statement
-    cols = @columns.join(",\n")
-    "(\n#{cols}\n)"
-  end
-  
-  def partition_statement
-    return "" if @partitions.nil? || @partitions.empty?
-    
-    cols = @partitions.join(",\n")
-    "PARTITIONED BY (\n#{cols}\n)"
-  end
-  
-  class Column
-    attr_reader :name, :type, :comment
-    def initialize(name, type, comment=nil)
-      @name, @type, @comment = name, type, comment
-    end
-    
-    def to_s
-      comment_string = @comment.nil? ? '' : " COMMENT '#{@comment}'"
-      "`#{@name}` #{@type.to_s.upcase}#{comment_string}"
-    end
-  end
-end