avro_utils 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 138c585168f97b5d9786dfcfda80f7b5f25caa61
-  data.tar.gz: 1011ffd72b61545233da0a573608a2692d8d194c
+  metadata.gz: af0e71b643b4c4299c0f2da4540c2877047fc5e2
+  data.tar.gz: 78f6e118f3795b92d8e6079d176308d34aa2e8e5
 SHA512:
-  metadata.gz: ccd802a2f71f5a48a723dd457ed210662e6dae1a80a44d1500593cb909e5481981f2b40a6da95707f528fbadb24c4e668e80e4d0e753a5e2292da7424db30989
-  data.tar.gz: a5c4af7e7f56a3694fce9f78b0cdbd8780b726ce81f837ad0d6c3b256c6bd6a5d355f8c36c48786fab532ad0819b82850a84cf17a39ec1eadf3c5410571c59a3
+  metadata.gz: 36536604fa62db13e89d6df4ffd08876556718431fb22334b4ac1f9eeebec21b79dffb0d06b88ecae4b6f3a6619fc9d05cdc4da44b26f2959a1709c38c14309d
+  data.tar.gz: b94fb042627c71eb4dbee94ea8297775de0f082e9c328253f694f4f1efa85214a879fe63548c254610ad545b1f282e881c82496f0e85e61ec75cbf25aed85440

data/avro_utils.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'avro_utils'
-  s.version     = '0.1.0'
+  s.version     = '0.2.0'
   s.summary     = "A gem to convert different data format to avro"
   s.description = "A gem to convert different data format to avro"
   s.authors     = ['Linh Chau']

data/lib/utils/avro_utils.rb

@@ -2,53 +2,85 @@ class AvroUtils
 
   class << self
     def avro_schema_hash(hash_data, record_type_name)
-      result = { 
-        type: 'record',
-        name: record_type_name,
-        fields: []
-      }
-
-      hash_data.each do |key, value|
-        if value.is_a? Hash 
-          result[:fields] << { name: key, type: avro_schema_hash(value, key.to_s.singularize.camelize) }
-        elsif value.is_a? Array
-          result[:fields] << { name: key, type: { type: 'array', items: RUBY_AVRO_TYPE_MAPPING[value[0].class.name.to_sym] } }
-        else
-          result[:fields] << { name: key, type: RUBY_AVRO_TYPE_MAPPING[value.class.name.to_sym] }
+      process_data do  
+        result = { 
+          type: 'record',
+          name: record_type_name,
+          fields: []
+        }
+
+        hash_data.each do |key, value|
+          if value.is_a? Hash 
+            result[:fields] << { name: key, type: avro_schema_hash(value, key.to_s.singularize.camelize) }
+          elsif value.is_a? Array
+            result[:fields] << { name: key, type: { type: 'array', items: RUBY_AVRO_TYPE_MAPPING[value[0].class.name.to_sym] } }
+          else
+            result[:fields] << { name: key, type: RUBY_AVRO_TYPE_MAPPING[value.class.name.to_sym] }
+          end
         end
+        
+        result
       end
-      
-      result
-    rescue StandardError => error
-      puts("Error: #{error.message}\n#{error.backtrace.join("\n")}")
-      raise InvalidDataException.new(error.message)
     end
 
     def avro_schema(hash_data, record_type_name)
       Avro::Schema.parse(avro_schema_hash(hash_data, record_type_name).to_json)
     end
 
-    def json_to_avro(json_data, record_type_name)
-      buffer = StringIO.new
+    def json_to_avro(json_data, record_type_name, filename = nil)
+      process_data do  
+        buffer = (filename.nil?)? StringIO.new : File.new(filename, 'wb')
+        hash_data = hash_with_string_keys(json_data)
+
+        schema = avro_schema(hash_data, record_type_name)
 
-      # Avro assumes that all the hashes use strings for keys. It does not accept Indifferent Hash.
-      # So the conversion bellow ensures that the hash data uses strings for keys.
-      json_data = json_data.to_json if json_data.is_a? Hash
-      hash_data = JSON.parse(json_data)
+        file_writer  = Avro::DataFile::Writer.new(buffer, Avro::IO::DatumWriter.new(schema), schema)
+        file_writer << hash_data
+        file_writer.flush
 
-      schema = avro_schema(hash_data, record_type_name)
+        buffer.rewind
+        buffer.close if buffer.is_a? File
+        buffer
+      end
+    end
+    
+    def bulk_json_to_avro(collection, record_type_name, filename = nil)
+      process_data do
+        buffer = (filename.nil?)? StringIO.new : File.new(filename, 'wb')
+
+        schema = avro_schema(hash_with_string_keys(collection.first), record_type_name)
+        file_writer  = Avro::DataFile::Writer.new(buffer, Avro::IO::DatumWriter.new(schema), schema)
+
+        collection.each do |json_data|
+          hash_data = hash_with_string_keys(json_data)
+          file_writer << hash_data
+        end
 
-      datum_writer = Avro::IO::DatumWriter.new(schema)
-      file_writer  = Avro::DataFile::Writer.new(buffer, datum_writer, schema)
-      file_writer << hash_data
-      file_writer.flush
+        file_writer.flush
 
-      buffer.rewind
-      buffer
+        buffer.rewind
+        buffer.close if buffer.is_a? File
+        buffer
+      end
+    end
+
+    # Avro assumes that all the hashes use strings for keys. It does not accept Indifferent Hash.
+    # So the conversion bellow ensures that the hash data uses strings for keys.
+    def hash_with_string_keys(json_data)
+      process_data do
+        json_data = json_data.to_json if json_data.is_a? Hash
+        hash_data = JSON.parse(json_data)
+      end
+    end
+  
+    private
+
+    def process_data
+      yield
     rescue StandardError => error
       puts("Error: #{error.message}\n#{error.backtrace.join("\n")}")
       raise InvalidDataException.new(error.message)
     end
-  
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: avro_utils
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Linh Chau