orcfile 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ee87da3cc76f9655fd8393da478578d57143210e
+  data.tar.gz: dcac0b20f2f932694b3eab4a1bc32bcf0e7037d1
+SHA512:
+  metadata.gz: 1a50f7b1a22662b433b213bf0e56d0b012e1c6c07c0773caadf6f6095307fee78ebfd2ac6d94f6d4e699cf1e75ef6c992075e68a51732a3fa010a33d770e5bc5
+  data.tar.gz: b9c8e252962f7215828636923ebead9ca8f2d122cadb5d559cac77d2e453ce7dafcf089f9d9ff05ef0cea9036f2663e97abf766295a9ea8c1c95269a38e4d3c9

data/README.rdoc

@@ -0,0 +1,99 @@
+=ORCFILE
+Ruby Gem for reading and writing Apache Optimized Row Columnar (ORC) files.
+This gem can also be paired using the factory_girl gem.
+
+==Installation
+Must use jruby.
+
+Add this line to your application's Gemfile:
+
+    gem 'orc_file'
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install orc_file
+
+==Usage
+===+OrcFileWriter+
+To write a file, you will need to initialize the OrcFileWriter class.
+This object needs a table schema, your dataset, the path to store the file, and an optional configuration hash.
+
+    OrcFileWriter.new(table_schema, data_set, path, *options={})
+====_table_schema_
+The table_schema must be a hash containing the column name and datatype as the key-value pair.
+
+Valid datatypes are:
+- integer
+- decimal
+- float
+- date
+- datetime
+- time
+- string
+
+
+    table_schema = {:id => :integer, :amount => :decimal, :rate => :float}
+
+====_data_set_
+The data_set must contain a hash with the column name and data value as the key-value pair.
+
+For one row in the dataset:
+
+    data_set = {:id => 1, :amount => 1000.01, :rate => 0.0005}
+
+For multiple rows in the dataset:
+
+    dataset = [{:id => 1, :amount => 1000.01, :rate => 0.0005},
+               {:id => 2, :amount => 2500.5, :rate => 0.1},
+               {:id => 3, :amount => 10.12, :rate => 10.0134}]
+
+====_path_
+The path should be the full file path or relative to your working directory. You must also specify the file name.
+
+    path = '/temp/orc_file.orc'
+
+====_options_
+Options is an optional hash parameter containing 5 configurable settings for writing an ORC file.
+
+   `:stripe_size` defines the size of the stripe, defaulted as 67,108,864 bytes <br>
+   `:row_index_stride` defines the number of rows between row index entries, defaulted as 10,000 <br>
+   `:buffer_size` defines the orc buffer size, defaulted as 262,144 bytes <br>
+   `:compression` defines the compression codec (NONE,ZLIB,SNAPPY,LZO), defaulted as ZLIB. <br>
+
+Define the options parameter has a hash
+
+    options = {:stripe_size => 70000000, :compression => 'SNAPPY'}
+
+===+write_to_orc+
+Once you have the OrcFileWriter object initialized you must call write_to_orc to write out the file
+
+      OrcFileWriter.new(table_schema, data_set, path, options).write_to_orc
+
+===+OrcFileReader+
+To read a file, you will need to initialize the OrcFileReader class.
+This object needs a table schema,  and the path of the file to be read.
+
+    OrcFileReader.new(table_schema, path)
+====_table_schema_
+The table_schema must be a hash containing the column name and datatype as the key-value pair.
+
+Valid datatypes are:
+- integer
+- decimal
+- float
+- date
+- datetime
+- time
+- string
+
+
+    table_schema = {:id => :integer, :amount => :decimal, :rate => :float}
+
+====_path_
+The path should be the full file path or relative to your working directory. You must also specify the file name.
+
+    path = '/temp/orc_file.orc'

data/lib/orc_file.rb

@@ -0,0 +1,25 @@
+require 'java'
+require 'bigdecimal'
+require 'time'
+require 'jars/slf4j-api-1.7.9.jar'
+require 'jars/commons-logging-1.2.jar'
+require 'jars/commons-configuration-1.10.jar'
+require 'jars/slf4j-simple-1.7.20.jar'
+require 'jars/hadoop-core-1.2.1.jar'
+require 'jars/hive-exec-2.1.1.jar'
+require 'orc_schema'
+require 'orc_options'
+require 'orc_reader_options'
+require 'orc_file_writer'
+require 'orc_file_reader'
+
+java_import 'org.slf4j.LoggerFactory'
+java_import 'org.apache.hadoop.hive.common.type.HiveDecimal'
+java_import 'org.apache.hadoop.conf.Configuration'
+java_import 'org.apache.hadoop.fs.Path'
+java_import 'org.apache.orc.CompressionKind'
+java_import 'org.apache.orc.TypeDescription'
+java_import 'org.apache.orc.OrcFile'
+
+
+

data/lib/orc_file_reader.rb

@@ -0,0 +1,49 @@
+class OrcFileReader
+  attr_reader :reader, :orc_options, :table_schema
+
+  def initialize(table_schema, path='orc_file.orc')
+    @orc_options = OrcReaderOptions.new
+    @table_schema = table_schema
+    path = Path.new(path)
+    @reader = OrcFile.createReader(path, @orc_options.orc)
+  end
+
+  def read_row(row_batch, row_index)
+    orc_row = {}
+    row_batch.cols.each_with_index do |column, index|
+      column_name = @table_schema.keys[index]
+      data_type = @table_schema[column_name]
+      case data_type
+        when :integer
+          orc_row[column_name] = column.vector[row_index]
+        when :decimal
+          orc_row[column_name] = column.vector[row_index].get_hive_decimal.to_s.to_d
+        when :float
+          #sets float value as 0.0005000000237487257 instead of 0.0005
+          orc_row[column_name] = column.vector[row_index]
+        when :datetime
+          orc_row[column_name] = DateTime.strptime(column.time[row_index].to_s, '%Q').to_time.to_datetime
+        when :time
+          orc_row[column_name] = Time.strptime(column.time[row_index].to_s, '%Q')
+        when :date
+          # orc_row[column_name] = Time.at(column.vector.first * 86400).to_date
+          orc_row[column_name] = Date.new(1970,1,1) + column.vector[row_index]
+        when :string
+          orc_row[column_name] = column.toString(row_index)
+      end
+    end
+    orc_row
+  end
+
+  def read_from_orc
+    rows = Array.new
+    row_batch = @reader.get_schema.createRowBatch()
+    @reader.rows.next_batch(row_batch)
+
+    @reader.number_of_rows.times do |row_index|
+      rows << read_row(row_batch, row_index)
+    end
+    rows
+  end
+
+end

data/lib/orc_file_writer.rb

@@ -0,0 +1,66 @@
+require 'orc_options'
+
+class OrcFileWriter
+  attr_reader :writer, :orc_options, :data_set, :table_schema, :output_path
+
+  def initialize(table_schema, data_set, path='orc_file.orc', options={})
+    @orc_options = OrcOptions.new
+    @orc_options.set_options(options)
+    @table_schema = table_schema
+    @data_set = data_set
+    @orc_options.define_table_schema(table_schema)
+    path.concat '.orc' unless path.include? '.orc'
+    path_object = Path.new(path)
+    @writer = OrcFile.createWriter(path_object, @orc_options.orc)
+  end
+
+  def create_row(row)
+    orc_row = @orc_options.orc_schema.schema.createRowBatch()
+    orc_row.size = 1
+    row.each_with_index do |(key, value), index|
+      data_type = @table_schema[key]
+      if value.nil?
+        orc_row.cols[index].noNulls = false
+        case data_type
+          when :datetime, :time, :decimal
+            orc_row.cols[index].set(0, value)
+          else
+            orc_row.cols[index].fill_with_nulls
+        end
+      else
+        case data_type
+          when :integer
+            data_for_column = value.to_java(:long)
+          when :decimal
+            data_for_column = HiveDecimal.create(value.to_d.to_java)
+          when :float
+            data_for_column = value.to_java(:double)
+          when :datetime, :time
+            data_for_column = value.to_time
+          when :date
+            # hive needs date formated as number of days since epoch (01/01/1970)
+            data_for_column = (value - Date.new(1970, 1, 1)).to_i
+          when :string
+            orc_row.cols[index].initBuffer(value.to_s.bytes.to_a.size)
+            data_for_column = value.to_s.bytes
+          else
+            raise ArgumentError, "column data type #{data_type} not defined"
+        end
+        orc_row.cols[index].fill(data_for_column)
+      end
+    end
+    orc_row
+  end
+
+  def write_to_orc
+    if @data_set.is_a? Array
+      @data_set.each do |row|
+        @writer.addRowBatch(create_row(row))
+      end
+    else
+      @writer.addRowBatch(create_row(@data_set))
+    end
+    @writer.close
+  end
+
+end

data/lib/orc_options.rb

@@ -0,0 +1,48 @@
+require 'orc_schema'
+
+class OrcOptions
+  attr_reader :orc_schema, :orc
+
+  def initialize()
+    conf = Configuration.new
+    @orc_schema = OrcSchema.new
+    @orc = OrcFile.writerOptions(conf)
+  end
+
+  def define_table_schema(table_schema)
+    raise TypeError, 'table_schema must be a Hash of {column_name: data_type}' unless table_schema.is_a? Hash
+    raise ArgumentError, 'table_schema cannot be an empty hash' if table_schema.empty?
+    table_schema.each do |column_name, data_type|
+      @orc_schema.add_column(column_name, data_type)
+    end
+    @orc.setSchema(@orc_schema.schema)
+  end
+
+  def define_stripe_size(stripe_size)
+    @orc.stripeSize(stripe_size)
+  end
+
+  def define_row_index_stride(row_index_stride)
+    @orc.rowIndexStride(row_index_stride)
+  end
+
+  def define_buffer_size(buffer_size)
+    @orc.bufferSize(buffer_size)
+  end
+
+  def define_compression(compression)
+    begin
+      @orc.compress(CompressionKind.valueOf(compression))
+    rescue java.lang.IllegalArgumentException
+      raise ArgumentError, "#{compression} is not a valid CompressionKind. Must be one of the following: \n#{CompressionKind.constants}"
+    end
+  end
+
+  def set_options(opts)
+    define_stripe_size(opts[:stripe_size]) unless opts[:stripe_size].nil?
+    define_row_index_stride(opts[:row_index_stride]) unless opts[:row_index_stride].nil?
+    define_buffer_size(opts[:buffer_size]) unless opts[:buffer_size].nil?
+    define_compression(opts[:compression]) unless opts[:compression].nil?
+  end
+
+end

data/lib/orc_reader_options.rb

@@ -0,0 +1,9 @@
+class OrcReaderOptions
+  attr_reader :orc, :orc_schema
+
+  def initialize()
+    conf = Configuration.new
+    @orc_schema = OrcSchema.new
+    @orc = OrcFile.readerOptions(conf)
+  end
+end

data/lib/orc_schema.rb

@@ -0,0 +1,28 @@
+class OrcSchema
+  attr_reader :schema
+  def initialize
+    @schema = TypeDescription.createStruct()
+  end
+
+  def add_column(column_name, data_type)
+    case data_type.downcase.to_sym
+      when :integer
+        type = TypeDescription.createLong()
+      when :datetime, :time
+        type = TypeDescription.createTimestamp()
+      when :date
+        type = TypeDescription.createDate()
+      when :decimal
+        type = TypeDescription.createDecimal()
+      when :float
+        type = TypeDescription.createFloat()
+      when :double
+        type = TypeDescription.createDouble()
+      when :string
+        type = TypeDescription.createString()
+      else
+        raise ArgumentError, "column data type #{data_type} not defined"
+    end
+    @schema.addField(column_name.to_s, type)
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: orcfile
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Andrew Shane
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-01-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: java
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem allows for the creation and reading of Apache Hive Optimized
+  Row Columnar (ORC) files.
+email:
+- ashane9@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.rdoc
+- lib/jars/commons-configuration-1.10.jar
+- lib/jars/commons-logging-1.2.jar
+- lib/jars/hadoop-core-1.2.1.jar
+- lib/jars/hive-exec-2.1.1.jar
+- lib/jars/slf4j-api-1.7.9.jar
+- lib/jars/slf4j-simple-1.7.20.jar
+- lib/orc_file.rb
+- lib/orc_file_reader.rb
+- lib/orc_file_writer.rb
+- lib/orc_options.rb
+- lib/orc_reader_options.rb
+- lib/orc_schema.rb
+homepage:
+licenses: []
+metadata:
+  source_code_uri: https://github.com/ashane9/orc_file
+  allowed_push_host: https://rubygems.org
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.14
+signing_key:
+specification_version: 4
+summary: Reader/writer of Hive ORC files
+test_files: []